From 8ddaca1c4bfd33ef345882e11455a6bf5f837532 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 10 Dec 2024 12:57:24 +0100 Subject: [PATCH 01/23] Add utils handle->info-struct to AddressTracker, check build-sizes and warn --- framework/decode/vulkan_address_replacer.cpp | 239 ++++++++++-------- framework/decode/vulkan_address_replacer.h | 33 +-- .../decode/vulkan_device_address_tracker.cpp | 40 +++ .../decode/vulkan_device_address_tracker.h | 20 ++ framework/decode/vulkan_object_info.h | 3 + .../decode/vulkan_replay_consumer_base.cpp | 8 +- framework/graphics/test/main.cpp | 32 +++ 7 files changed, 257 insertions(+), 118 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index a68f901617..d340dcffb0 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -108,33 +108,35 @@ decode::VulkanAddressReplacer::buffer_context_t::~buffer_context_t() VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* device_info, const encode::VulkanDeviceTable* device_table, const decode::CommonObjectInfoTable& object_table) : - device_table_(device_table) + _device_table(device_table) { GFXRECON_ASSERT(device_info != nullptr && device_table != nullptr) const VulkanPhysicalDeviceInfo* physical_device_info = object_table.GetVkPhysicalDeviceInfo(device_info->parent_id); - device_ = device_info->handle; - resource_allocator_ = device_info->allocator.get(); - get_device_address_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_2 - ? device_table->GetBufferDeviceAddress - : device_table->GetBufferDeviceAddressKHR; + _device = device_info->handle; + _resource_allocator = device_info->allocator.get(); + _get_device_address_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_2 + ? device_table->GetBufferDeviceAddress + : device_table->GetBufferDeviceAddressKHR; + + _get_acceleration_build_sizes_fn = device_table->GetAccelerationStructureBuildSizesKHR; if (physical_device_info != nullptr && physical_device_info->capture_raytracing_properties && physical_device_info->replay_device_info->raytracing_properties) { - capture_ray_properties_ = *physical_device_info->capture_raytracing_properties; - replay_ray_properties_ = *physical_device_info->replay_device_info->raytracing_properties; + _capture_ray_properties = *physical_device_info->capture_raytracing_properties; + _replay_ray_properties = *physical_device_info->replay_device_info->raytracing_properties; - if (capture_ray_properties_.shaderGroupHandleSize != replay_ray_properties_.shaderGroupHandleSize || - capture_ray_properties_.shaderGroupHandleAlignment != replay_ray_properties_.shaderGroupHandleAlignment || - capture_ray_properties_.shaderGroupBaseAlignment != replay_ray_properties_.shaderGroupBaseAlignment) + if (_capture_ray_properties.shaderGroupHandleSize != _replay_ray_properties.shaderGroupHandleSize || + _capture_ray_properties.shaderGroupHandleAlignment != _replay_ray_properties.shaderGroupHandleAlignment || + _capture_ray_properties.shaderGroupBaseAlignment != _replay_ray_properties.shaderGroupBaseAlignment) { - valid_sbt_alignment_ = false; + _valid_sbt_alignment = false; } GFXRECON_ASSERT(physical_device_info->replay_device_info != nullptr); GFXRECON_ASSERT(physical_device_info->replay_device_info->memory_properties.has_value()); - memory_properties_ = *physical_device_info->replay_device_info->memory_properties; + _memory_properties = *physical_device_info->replay_device_info->memory_properties; } } @@ -148,21 +150,21 @@ VulkanAddressReplacer::~VulkanAddressReplacer() mark_injected_commands_helper_t mark_injected_commands_helper; // explicitly free resources here, in order to mark destruction API-calls as injected - pipeline_context_sbt_ = {}; - pipeline_context_bda_ = {}; - shadow_sbt_map_ = {}; + _pipeline_context_sbt = {}; + _pipeline_context_bda = {}; + _shadow_sbt_map = {}; - if (pipeline_bda_ != VK_NULL_HANDLE) + if (_pipeline_bda != VK_NULL_HANDLE) { - device_table_->DestroyPipeline(device_, pipeline_bda_, nullptr); + _device_table->DestroyPipeline(_device, _pipeline_bda, nullptr); } - if (pipeline_sbt_ != VK_NULL_HANDLE) + if (_pipeline_sbt != VK_NULL_HANDLE) { - device_table_->DestroyPipeline(device_, pipeline_sbt_, nullptr); + _device_table->DestroyPipeline(_device, _pipeline_sbt, nullptr); } - if (pipeline_layout_ != VK_NULL_HANDLE) + if (_pipeline_layout != VK_NULL_HANDLE) { - device_table_->DestroyPipelineLayout(device_, pipeline_layout_, nullptr); + _device_table->DestroyPipelineLayout(_device, _pipeline_layout, nullptr); } } @@ -175,7 +177,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( const decode::VulkanDeviceAddressTracker& address_tracker, const std::unordered_map& group_handle_map) { - GFXRECON_ASSERT(device_table_ != nullptr); + GFXRECON_ASSERT(_device_table != nullptr); // NOTE: we expect this map to be populated here, but not for older captures (before #1844) using trimming. if (group_handle_map.empty()) @@ -200,12 +202,12 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // valid_sbt_alignment_ = false; // valid_group_handles = false; - if (!valid_sbt_alignment_ || !valid_group_handles) + if (!_valid_sbt_alignment || !valid_group_handles) { // mark injected commands mark_injected_commands_helper_t mark_injected_commands_helper; - if (pipeline_sbt_ == VK_NULL_HANDLE) + if (_pipeline_sbt == VK_NULL_HANDLE) { if (!init_pipeline()) { @@ -246,27 +248,27 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( address_remap(callable_sbt); // prepare linear hashmap - hashmap_sbt_.clear(); + _hashmap_sbt.clear(); for (const auto& [lhs, rhs] : group_handle_map) { - hashmap_sbt_.put(lhs, rhs); + _hashmap_sbt.put(lhs, rhs); } - if (!create_buffer(hashmap_sbt_.get_storage(nullptr), pipeline_context_sbt_.hashmap_storage)) + if (!create_buffer(_hashmap_sbt.get_storage(nullptr), _pipeline_context_sbt.hashmap_storage)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); } - hashmap_sbt_.get_storage(pipeline_context_sbt_.hashmap_storage.mapped_data); + _hashmap_sbt.get_storage(_pipeline_context_sbt.hashmap_storage.mapped_data); // input-handles constexpr uint32_t max_num_handles = 4; - if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), pipeline_context_sbt_.input_handle_buffer)) + if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), _pipeline_context_sbt.input_handle_buffer)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); } auto input_addresses = - reinterpret_cast(pipeline_context_sbt_.input_handle_buffer.mapped_data); + reinterpret_cast(_pipeline_context_sbt.input_handle_buffer.mapped_data); uint32_t num_addresses = 0; for (const auto& region : { raygen_sbt, miss_sbt, hit_sbt, callable_sbt }) @@ -278,14 +280,14 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } replacer_params_t replacer_params = {}; - replacer_params.hashmap.storage = pipeline_context_sbt_.hashmap_storage.device_address; - replacer_params.hashmap.size = hashmap_sbt_.size(); - replacer_params.hashmap.capacity = hashmap_sbt_.capacity(); + replacer_params.hashmap.storage = _pipeline_context_sbt.hashmap_storage.device_address; + replacer_params.hashmap.size = _hashmap_sbt.size(); + replacer_params.hashmap.capacity = _hashmap_sbt.capacity(); - replacer_params.input_handles = pipeline_context_sbt_.input_handle_buffer.device_address; + replacer_params.input_handles = _pipeline_context_sbt.input_handle_buffer.device_address; replacer_params.num_handles = num_addresses; - if (valid_sbt_alignment_) + if (_valid_sbt_alignment) { GFXRECON_LOG_INFO_ONCE("Replay adjusted mismatching raytracing shader-group-handles"); @@ -309,20 +311,20 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( "Replay adjusted a mismatching raytracing shader-binding-table using a shadow-buffer"); // output-handles - if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), pipeline_context_sbt_.output_handle_buffer)) + if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), _pipeline_context_sbt.output_handle_buffer)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); return; } // output to shadow-sbt-buffer - replacer_params.output_handles = pipeline_context_sbt_.output_handle_buffer.device_address; + replacer_params.output_handles = _pipeline_context_sbt.output_handle_buffer.device_address; // find/create shadow-SBT-buffer uint32_t sbt_offset = 0; - auto& shadow_buf_context = shadow_sbt_map_[command_buffer_info->handle]; + auto& shadow_buf_context = _shadow_sbt_map[command_buffer_info->handle]; - const uint32_t handle_size_aligned = aligned_size(replay_ray_properties_.shaderGroupHandleSize, - replay_ray_properties_.shaderGroupHandleAlignment); + const uint32_t handle_size_aligned = aligned_size(_replay_ray_properties.shaderGroupHandleSize, + _replay_ray_properties.shaderGroupHandleAlignment); for (auto& region : { raygen_sbt, miss_sbt, hit_sbt, callable_sbt }) { @@ -330,7 +332,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( { uint32_t num_handles_limit = region->size / region->stride; uint32_t group_size = aligned_size(num_handles_limit * handle_size_aligned, - replay_ray_properties_.shaderGroupBaseAlignment); + _replay_ray_properties.shaderGroupBaseAlignment); sbt_offset += group_size; // adjust group-size @@ -339,7 +341,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } } // raygen: stride == size - raygen_sbt->size = raygen_sbt->stride = replay_ray_properties_.shaderGroupBaseAlignment; + raygen_sbt->size = raygen_sbt->stride = _replay_ray_properties.shaderGroupBaseAlignment; if (!create_buffer(sbt_offset, shadow_buf_context, VK_BUFFER_USAGE_SHADER_BINDING_TABLE_BIT_KHR)) { @@ -348,7 +350,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } auto output_addresses = - reinterpret_cast(pipeline_context_sbt_.output_handle_buffer.mapped_data); + reinterpret_cast(_pipeline_context_sbt.output_handle_buffer.mapped_data); uint32_t out_index = 0; sbt_offset = 0; for (auto& region : { raygen_sbt, miss_sbt, hit_sbt, callable_sbt }) @@ -363,18 +365,18 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } } - device_table_->CmdBindPipeline(command_buffer_info->handle, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline_sbt_); + _device_table->CmdBindPipeline(command_buffer_info->handle, VK_PIPELINE_BIND_POINT_COMPUTE, _pipeline_sbt); // NOTE: using push-constants here requires us to re-establish the previous data, if any - device_table_->CmdPushConstants(command_buffer_info->handle, - pipeline_layout_, + _device_table->CmdPushConstants(command_buffer_info->handle, + _pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(replacer_params_t), &replacer_params); // run a single workgroup constexpr uint32_t wg_size = 32; - device_table_->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); + _device_table->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); // post memory-barrier for (const auto& buf : buffer_set) @@ -390,7 +392,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // set previous push-constant data if (!command_buffer_info->push_constant_data.empty()) { - device_table_->CmdPushConstants(command_buffer_info->handle, + _device_table->CmdPushConstants(command_buffer_info->handle, command_buffer_info->push_constant_pipeline_layout, command_buffer_info->push_constant_stage_flags, 0, @@ -407,7 +409,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( VkAccelerationStructureBuildRangeInfoKHR** build_range_infos, const VulkanDeviceAddressTracker& address_tracker) { - GFXRECON_ASSERT(device_table_ != nullptr); + GFXRECON_ASSERT(_device_table != nullptr); // TODO: testing only -> remove when closing issue #1526 constexpr bool force_replace = false; @@ -443,6 +445,38 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // check/correct scratch-address address_remap(build_geometry_info.scratchData.deviceAddress); + // check capture/replay acceleration-structure buffer-sizes + { + VkAccelerationStructureBuildSizesInfoKHR build_size_info = {}; + build_size_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR; + + std::vector primitive_counts(build_geometry_info.geometryCount); + for (uint32_t j = 0; j < build_geometry_info.geometryCount; ++j) + { + primitive_counts[j] = range_info->primitiveCount; + } + _get_acceleration_build_sizes_fn(_device, + VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, + &build_geometry_info, + primitive_counts.data(), + &build_size_info); + + // retrieve VkAccelerationStructureKHR -> VkBuffer -> check/correct size + auto* acceleration_structure_info = + address_tracker.GetAccelerationStructureByHandle(build_geometry_info.dstAccelerationStructure); + GFXRECON_ASSERT(acceleration_structure_info != nullptr) + auto* buffer_info = address_tracker.GetBufferByHandle(acceleration_structure_info->buffer); + GFXRECON_ASSERT(buffer_info != nullptr) + + if (buffer_info != nullptr && buffer_info->size < build_size_info.accelerationStructureSize) + { + GFXRECON_LOG_WARNING("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: buffer-size is " + "too small (%d < %d)", + buffer_info->size, + build_size_info.accelerationStructureSize); + } + } + for (uint32_t j = 0; j < build_geometry_info.geometryCount; ++j) { auto geometry = const_cast(build_geometry_info.pGeometries != nullptr @@ -490,23 +524,23 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (!addresses_to_replace.empty()) { // prepare linear hashmap - hashmap_bda_.clear(); + _hashmap_bda.clear(); auto acceleration_structure_map = address_tracker.GetAccelerationStructureDeviceAddressMap(); for (const auto& [capture_address, replay_address] : acceleration_structure_map) { if (force_replace || capture_address != replay_address) { // store addresses we will need to replace - hashmap_bda_.put(capture_address, replay_address); + _hashmap_bda.put(capture_address, replay_address); } } - if (!hashmap_bda_.empty()) + if (!_hashmap_bda.empty()) { // mark injected commands mark_injected_commands_helper_t mark_injected_commands_helper; - if (pipeline_bda_ == VK_NULL_HANDLE) + if (_pipeline_bda == VK_NULL_HANDLE) { if (!init_pipeline()) { @@ -516,43 +550,43 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } } - if (!create_buffer(hashmap_bda_.get_storage(nullptr), pipeline_context_bda_.hashmap_storage)) + if (!create_buffer(_hashmap_bda.get_storage(nullptr), _pipeline_context_bda.hashmap_storage)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); } - hashmap_bda_.get_storage(pipeline_context_bda_.hashmap_storage.mapped_data); + _hashmap_bda.get_storage(_pipeline_context_bda.hashmap_storage.mapped_data); uint32_t num_bytes = addresses_to_replace.size() * sizeof(VkDeviceAddress); - if (!create_buffer(num_bytes, pipeline_context_bda_.input_handle_buffer)) + if (!create_buffer(num_bytes, _pipeline_context_bda.input_handle_buffer)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); } - memcpy(pipeline_context_bda_.input_handle_buffer.mapped_data, addresses_to_replace.data(), num_bytes); + memcpy(_pipeline_context_bda.input_handle_buffer.mapped_data, addresses_to_replace.data(), num_bytes); replacer_params_t replacer_params = {}; - replacer_params.hashmap.storage = pipeline_context_bda_.hashmap_storage.device_address; - replacer_params.hashmap.size = hashmap_bda_.size(); - replacer_params.hashmap.capacity = hashmap_bda_.capacity(); + replacer_params.hashmap.storage = _pipeline_context_bda.hashmap_storage.device_address; + replacer_params.hashmap.size = _hashmap_bda.size(); + replacer_params.hashmap.capacity = _hashmap_bda.capacity(); // in-place - replacer_params.input_handles = pipeline_context_bda_.input_handle_buffer.device_address; - replacer_params.output_handles = pipeline_context_bda_.input_handle_buffer.device_address; + replacer_params.input_handles = _pipeline_context_bda.input_handle_buffer.device_address; + replacer_params.output_handles = _pipeline_context_bda.input_handle_buffer.device_address; replacer_params.num_handles = addresses_to_replace.size(); - device_table_->CmdBindPipeline(command_buffer_info->handle, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline_bda_); + _device_table->CmdBindPipeline(command_buffer_info->handle, VK_PIPELINE_BIND_POINT_COMPUTE, _pipeline_bda); // NOTE: using push-constants here requires us to re-establish the previous data, if any - device_table_->CmdPushConstants(command_buffer_info->handle, - pipeline_layout_, + _device_table->CmdPushConstants(command_buffer_info->handle, + _pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(replacer_params_t), &replacer_params); // run a single workgroup constexpr uint32_t wg_size = 32; - device_table_->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); + _device_table->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); // post memory-barrier for (const auto& buf : buffer_set) @@ -568,7 +602,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // set previous push-constant data if (!command_buffer_info->push_constant_data.empty()) { - device_table_->CmdPushConstants(command_buffer_info->handle, + _device_table->CmdPushConstants(command_buffer_info->handle, command_buffer_info->push_constant_pipeline_layout, command_buffer_info->push_constant_stage_flags, 0, @@ -581,7 +615,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( bool VulkanAddressReplacer::init_pipeline() { - if (pipeline_sbt_ != VK_NULL_HANDLE) + if (_pipeline_sbt != VK_NULL_HANDLE) { // assume already initialized return true; @@ -600,7 +634,7 @@ bool VulkanAddressReplacer::init_pipeline() pipeline_layout_info.pushConstantRangeCount = 1; pipeline_layout_info.pPushConstantRanges = &push_constant_range; - VkResult result = device_table_->CreatePipelineLayout(device_, &pipeline_layout_info, nullptr, &pipeline_layout_); + VkResult result = _device_table->CreatePipelineLayout(_device, &pipeline_layout_info, nullptr, &_pipeline_layout); if (result != VK_SUCCESS) { @@ -617,7 +651,7 @@ bool VulkanAddressReplacer::init_pipeline() shader_module_create_info.pCode = reinterpret_cast(spirv.data()); VkResult result = - device_table_->CreateShaderModule(device_, &shader_module_create_info, nullptr, &compute_module); + _device_table->CreateShaderModule(_device, &shader_module_create_info, nullptr, &compute_module); if (result != VK_SUCCESS) { @@ -638,8 +672,8 @@ bool VulkanAddressReplacer::init_pipeline() pipeline_create_info.layout = layout; pipeline_create_info.stage = stage_info; - result = device_table_->CreateComputePipelines( - device_, VK_NULL_HANDLE, 1, &pipeline_create_info, VK_NULL_HANDLE, &out_pipeline); + result = _device_table->CreateComputePipelines( + _device, VK_NULL_HANDLE, 1, &pipeline_create_info, VK_NULL_HANDLE, &out_pipeline); if (result != VK_SUCCESS) { @@ -648,19 +682,19 @@ bool VulkanAddressReplacer::init_pipeline() if (compute_module != VK_NULL_HANDLE) { - device_table_->DestroyShaderModule(device_, compute_module, nullptr); + _device_table->DestroyShaderModule(_device, compute_module, nullptr); } return result; }; // create SBT pipeline - if (create_pipeline(pipeline_layout_, g_replacer_sbt_comp, pipeline_sbt_) != VK_SUCCESS) + if (create_pipeline(_pipeline_layout, g_replacer_sbt_comp, _pipeline_sbt) != VK_SUCCESS) { return false; } // create BDA pipeline - if (create_pipeline(pipeline_layout_, g_replacer_bda_comp, pipeline_bda_) != VK_SUCCESS) + if (create_pipeline(_pipeline_layout, g_replacer_bda_comp, _pipeline_bda) != VK_SUCCESS) { return false; } @@ -679,7 +713,7 @@ bool VulkanAddressReplacer::create_buffer(size_t // free previous resources buffer_context = {}; - buffer_context.resource_allocator = resource_allocator_; + buffer_context.resource_allocator = _resource_allocator; buffer_context.num_bytes = num_bytes; VkBufferCreateInfo buffer_create_info = {}; @@ -690,7 +724,7 @@ bool VulkanAddressReplacer::create_buffer(size_t buffer_create_info.queueFamilyIndexCount = 0; buffer_create_info.size = num_bytes; - VkResult result = resource_allocator_->CreateBufferDirect( + VkResult result = _resource_allocator->CreateBufferDirect( &buffer_create_info, nullptr, &buffer_context.buffer, &buffer_context.allocator_data); if (result != VK_SUCCESS) { @@ -698,10 +732,10 @@ bool VulkanAddressReplacer::create_buffer(size_t } VkMemoryRequirements memory_requirements; - device_table_->GetBufferMemoryRequirements(device_, buffer_context.buffer, &memory_requirements); + _device_table->GetBufferMemoryRequirements(_device, buffer_context.buffer, &memory_requirements); uint32_t memory_type_index = - get_memory_type_index(memory_properties_, + get_memory_type_index(_memory_properties, memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); @@ -709,7 +743,7 @@ bool VulkanAddressReplacer::create_buffer(size_t { /* fallback to coherent */ memory_type_index = - get_memory_type_index(memory_properties_, + get_memory_type_index(_memory_properties, memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); } @@ -726,7 +760,7 @@ bool VulkanAddressReplacer::create_buffer(size_t alloc_flags_info.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT; alloc_info.pNext = &alloc_flags_info; - result = resource_allocator_->AllocateMemoryDirect( + result = _resource_allocator->AllocateMemoryDirect( &alloc_info, nullptr, &buffer_context.device_memory, &buffer_context.memory_data); if (result != VK_SUCCESS) @@ -735,7 +769,7 @@ bool VulkanAddressReplacer::create_buffer(size_t } VkMemoryPropertyFlags memory_flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - result = resource_allocator_->BindBufferMemory(buffer_context.buffer, + result = _resource_allocator->BindBufferMemory(buffer_context.buffer, buffer_context.device_memory, 0, buffer_context.allocator_data, @@ -750,10 +784,10 @@ bool VulkanAddressReplacer::create_buffer(size_t VkBufferDeviceAddressInfo address_info = {}; address_info.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO; address_info.buffer = buffer_context.buffer; - buffer_context.device_address = get_device_address_fn_(device_, &address_info); + buffer_context.device_address = _get_device_address_fn_(_device, &address_info); // map buffer - result = resource_allocator_->MapResourceMemoryDirect( + result = _resource_allocator->MapResourceMemoryDirect( VK_WHOLE_SIZE, 0, &buffer_context.mapped_data, buffer_context.allocator_data); return result == VK_SUCCESS; } @@ -774,28 +808,29 @@ void VulkanAddressReplacer::barrier(VkCommandBuffer command_buffer, barrier.srcAccessMask = src_access; barrier.dstAccessMask = dst_access; - device_table_->CmdPipelineBarrier( + _device_table->CmdPipelineBarrier( command_buffer, src_stage, dst_stage, VkDependencyFlags(0), 0, nullptr, 1, &barrier, 0, nullptr); } void swap(VulkanAddressReplacer& lhs, VulkanAddressReplacer& rhs) noexcept { - std::swap(lhs.device_table_, rhs.device_table_); - std::swap(lhs.memory_properties_, rhs.memory_properties_); - std::swap(lhs.capture_ray_properties_, rhs.capture_ray_properties_); - std::swap(lhs.replay_ray_properties_, rhs.replay_ray_properties_); - std::swap(lhs.valid_sbt_alignment_, rhs.valid_sbt_alignment_); - std::swap(lhs.device_, rhs.device_); - std::swap(lhs.get_device_address_fn_, rhs.get_device_address_fn_); - std::swap(lhs.resource_allocator_, rhs.resource_allocator_); - std::swap(lhs.pipeline_layout_, rhs.pipeline_layout_); - std::swap(lhs.pipeline_sbt_, rhs.pipeline_sbt_); - std::swap(lhs.pipeline_bda_, rhs.pipeline_bda_); - std::swap(lhs.pipeline_context_sbt_, rhs.pipeline_context_sbt_); - std::swap(lhs.pipeline_context_bda_, rhs.pipeline_context_bda_); - std::swap(lhs.hashmap_sbt_, rhs.hashmap_sbt_); - std::swap(lhs.hashmap_bda_, rhs.hashmap_bda_); - std::swap(lhs.shadow_sbt_map_, rhs.shadow_sbt_map_); + std::swap(lhs._device_table, rhs._device_table); + std::swap(lhs._memory_properties, rhs._memory_properties); + std::swap(lhs._capture_ray_properties, rhs._capture_ray_properties); + std::swap(lhs._replay_ray_properties, rhs._replay_ray_properties); + std::swap(lhs._valid_sbt_alignment, rhs._valid_sbt_alignment); + std::swap(lhs._device, rhs._device); + std::swap(lhs._resource_allocator, rhs._resource_allocator); + std::swap(lhs._pipeline_layout, rhs._pipeline_layout); + std::swap(lhs._pipeline_sbt, rhs._pipeline_sbt); + std::swap(lhs._pipeline_bda, rhs._pipeline_bda); + std::swap(lhs._pipeline_context_sbt, rhs._pipeline_context_sbt); + std::swap(lhs._pipeline_context_bda, rhs._pipeline_context_bda); + std::swap(lhs._hashmap_sbt, rhs._hashmap_sbt); + std::swap(lhs._hashmap_bda, rhs._hashmap_bda); + std::swap(lhs._shadow_sbt_map, rhs._shadow_sbt_map); + std::swap(lhs._get_device_address_fn_, rhs._get_device_address_fn_); + std::swap(lhs._get_acceleration_build_sizes_fn, rhs._get_acceleration_build_sizes_fn); } GFXRECON_END_NAMESPACE(decode) diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index eca38e0476..0d56e5854d 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -137,30 +137,33 @@ class VulkanAddressReplacer VkPipelineStageFlags dst_stage, VkAccessFlags dst_access); - const encode::VulkanDeviceTable* device_table_ = nullptr; - VkPhysicalDeviceMemoryProperties memory_properties_ = {}; - VkPhysicalDeviceRayTracingPipelinePropertiesKHR capture_ray_properties_{}, replay_ray_properties_{}; - bool valid_sbt_alignment_ = true; + const encode::VulkanDeviceTable* _device_table = nullptr; + VkPhysicalDeviceMemoryProperties _memory_properties = {}; + VkPhysicalDeviceRayTracingPipelinePropertiesKHR _capture_ray_properties{}, _replay_ray_properties{}; + bool _valid_sbt_alignment = true; - VkDevice device_ = VK_NULL_HANDLE; - PFN_vkGetBufferDeviceAddress get_device_address_fn_ = nullptr; - decode::VulkanResourceAllocator* resource_allocator_ = nullptr; + VkDevice _device = VK_NULL_HANDLE; + decode::VulkanResourceAllocator* _resource_allocator = nullptr; // common layout used for all pipelines - VkPipelineLayout pipeline_layout_ = VK_NULL_HANDLE; + VkPipelineLayout _pipeline_layout = VK_NULL_HANDLE; // pipeline dealing with shader-binding-table (SBT), replacing group-handles - VkPipeline pipeline_sbt_ = VK_NULL_HANDLE; + VkPipeline _pipeline_sbt = VK_NULL_HANDLE; // pipeline dealing with buffer-device-addresses (BDA), replacing addresses - VkPipeline pipeline_bda_ = VK_NULL_HANDLE; + VkPipeline _pipeline_bda = VK_NULL_HANDLE; - pipeline_context_t pipeline_context_sbt_; - pipeline_context_t pipeline_context_bda_; + pipeline_context_t _pipeline_context_sbt; + pipeline_context_t _pipeline_context_bda; - util::linear_hashmap hashmap_sbt_; - util::linear_hashmap hashmap_bda_; - std::unordered_map shadow_sbt_map_; + util::linear_hashmap _hashmap_sbt; + util::linear_hashmap _hashmap_bda; + std::unordered_map _shadow_sbt_map; + + // acquired function pointers + PFN_vkGetBufferDeviceAddress _get_device_address_fn_ = nullptr; + PFN_vkGetAccelerationStructureBuildSizesKHR _get_acceleration_build_sizes_fn = nullptr; }; GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_device_address_tracker.cpp b/framework/decode/vulkan_device_address_tracker.cpp index bab13b5993..0200daa5aa 100644 --- a/framework/decode/vulkan_device_address_tracker.cpp +++ b/framework/decode/vulkan_device_address_tracker.cpp @@ -35,6 +35,7 @@ void decode::VulkanDeviceAddressTracker::TrackBuffer(const decode::VulkanBufferI if (buffer_info != nullptr && buffer_info->capture_address != 0) { _buffer_capture_addresses[buffer_info->capture_address] = buffer_info->capture_id; + _buffer_handles[buffer_info->handle] = buffer_info->capture_id; } } @@ -51,8 +52,12 @@ void VulkanDeviceAddressTracker::TrackAccelerationStructure( { if (acceleration_structure_info != nullptr && acceleration_structure_info->capture_address != 0) { + // track device address _acceleration_structure_capture_addresses[acceleration_structure_info->capture_address] = acceleration_structure_info->capture_id; + + // track vulkan-handle + _acceleration_structure_handles[acceleration_structure_info->handle] = acceleration_structure_info->capture_id; } } @@ -71,6 +76,22 @@ decode::VulkanDeviceAddressTracker::GetBufferByCaptureDeviceAddress(VkDeviceAddr return GetBufferInfo(capture_address, _buffer_capture_addresses); } +const VulkanBufferInfo* VulkanDeviceAddressTracker::GetBufferByHandle(VkBuffer handle) const +{ + auto handle_it = _buffer_handles.find(handle); + if (handle_it != _buffer_handles.end()) + { + const auto& [h, handle_id] = *handle_it; + const VulkanBufferInfo* found_buffer_info = _object_info_table.GetVkBufferInfo(handle_id); + + if (found_buffer_info != nullptr) + { + return found_buffer_info; + } + } + return nullptr; +} + const VulkanBufferInfo* VulkanDeviceAddressTracker::GetBufferInfo(VkDeviceAddress device_address, const VulkanDeviceAddressTracker::buffer_address_map_t& address_map) const @@ -123,6 +144,25 @@ VulkanDeviceAddressTracker::GetAccelerationStructureByCaptureDeviceAddress(VkDev } return nullptr; } + +[[nodiscard]] const VulkanAccelerationStructureKHRInfo* +VulkanDeviceAddressTracker::GetAccelerationStructureByHandle(VkAccelerationStructureKHR handle) const +{ + auto handle_it = _acceleration_structure_handles.find(handle); + if (handle_it != _acceleration_structure_handles.end()) + { + const auto& [h, handle_id] = *handle_it; + const VulkanAccelerationStructureKHRInfo* found_acceleration_structure_info = + _object_info_table.GetVkAccelerationStructureKHRInfo(handle_id); + + if (found_acceleration_structure_info != nullptr) + { + return found_acceleration_structure_info; + } + } + return nullptr; +} + std::unordered_map VulkanDeviceAddressTracker::GetAccelerationStructureDeviceAddressMap() const { diff --git a/framework/decode/vulkan_device_address_tracker.h b/framework/decode/vulkan_device_address_tracker.h index c73857a4a2..c35fd09665 100644 --- a/framework/decode/vulkan_device_address_tracker.h +++ b/framework/decode/vulkan_device_address_tracker.h @@ -81,6 +81,14 @@ class VulkanDeviceAddressTracker */ [[nodiscard]] const VulkanBufferInfo* GetBufferByCaptureDeviceAddress(VkDeviceAddress capture_address) const; + /** + * @brief Retrieve a buffer info-struct by providing its vulkan-handle. + * + * @param handle a capture-time VkBuffer handle. + * @return a const-pointer to a found BufferInfo or nullptr. + */ + [[nodiscard]] const VulkanBufferInfo* GetBufferByHandle(VkBuffer handle) const; + /** * @brief Retrieve an acceleration-structure by providing a capture-time VkDeviceAddress. * @@ -90,6 +98,15 @@ class VulkanDeviceAddressTracker [[nodiscard]] const VulkanAccelerationStructureKHRInfo* GetAccelerationStructureByCaptureDeviceAddress(VkDeviceAddress capture_address) const; + /** + * @brief Retrieve an acceleration-structure info-struct by providing its vulkan-handle. + * + * @param handle a replay-time VkAccelerationStructureKHR handle. + * @return a const-pointer to a found AccelerationStructureKHRInfo or nullptr. + */ + [[nodiscard]] const VulkanAccelerationStructureKHRInfo* + GetAccelerationStructureByHandle(VkAccelerationStructureKHR handle) const; + /** * @brief Create and return a lookup-table containing all internally stored acceleration-structure addresses. * @@ -107,6 +124,9 @@ class VulkanDeviceAddressTracker const VulkanObjectInfoTable& _object_info_table; buffer_address_map_t _buffer_capture_addresses; std::unordered_map _acceleration_structure_capture_addresses; + + std::unordered_map _buffer_handles; + std::unordered_map _acceleration_structure_handles; }; GFXRECON_END_NAMESPACE(decode) diff --git a/framework/decode/vulkan_object_info.h b/framework/decode/vulkan_object_info.h index 4d18a647d3..047d167282 100644 --- a/framework/decode/vulkan_object_info.h +++ b/framework/decode/vulkan_object_info.h @@ -707,6 +707,9 @@ struct VulkanAccelerationStructureKHRInfo : public VulkanObjectInfoGetPointer(); VkDevice device = device_info->handle; + // keep track of associated buffer + auto* acceleration_structure_info = + reinterpret_cast(pAccelerationStructureKHR->GetConsumerData(0)); + GFXRECON_ASSERT(acceleration_structure_info); + acceleration_structure_info->buffer = replay_create_info->buffer; + if (device_info->property_feature_info.feature_accelerationStructureCaptureReplay) { // Set opaque device address @@ -7765,7 +7771,7 @@ void VulkanReplayConsumerBase::OverrideCmdBuildAccelerationStructuresKHR( VkAccelerationStructureBuildGeometryInfoKHR* build_geometry_infos = pInfos->GetPointer(); VkAccelerationStructureBuildRangeInfoKHR** build_range_infos = ppBuildRangeInfos->GetPointer(); - if (!device_info->allocator->SupportsOpaqueDeviceAddresses()) +// if (!device_info->allocator->SupportsOpaqueDeviceAddresses()) { auto& address_tracker = GetDeviceAddressTracker(device_info); auto& address_replacer = GetDeviceAddressReplacer(device_info); diff --git a/framework/graphics/test/main.cpp b/framework/graphics/test/main.cpp index d7f8c7c0ee..52b9cbed33 100644 --- a/framework/graphics/test/main.cpp +++ b/framework/graphics/test/main.cpp @@ -25,3 +25,35 @@ #define CATCH_CONFIG_MAIN #include + +#include "graphics/vulkan_shader_group_handle.h" + +TEST_CASE("vulkan_shader_group_handle - create empty handles", "[]") +{ + gfxrecon::graphics::shader_group_handle_t one, two; + + // check for all zeros + uint8_t data[gfxrecon::graphics::shader_group_handle_t::MAX_HANDLE_SIZE] = {}; + REQUIRE(memcmp(one.data, data, gfxrecon::graphics::shader_group_handle_t::MAX_HANDLE_SIZE) == 0); + + REQUIRE(one == two); + REQUIRE_FALSE(one != two); + + auto three = one; + REQUIRE(one == three); +} + +TEST_CASE("vulkan_shader_group_handle - create handles", "[]") +{ + std::vector data(32); + std::iota(data.begin(), data.end(), 0); + gfxrecon::graphics::shader_group_handle_t one(data.data(), data.size()); + + data[31] = 99; + gfxrecon::graphics::shader_group_handle_t two(data.data(), data.size()); + REQUIRE(one != two); + + // check hashing via std::hash + std::hash hasher; + REQUIRE(hasher(one) != hasher(two)); +} \ No newline at end of file From b4667adee2bb4756370d59a2d5d1cf51a5630385 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 10 Dec 2024 18:38:15 +0100 Subject: [PATCH 02/23] cleanup refactor, stubs for replacement AS --- framework/decode/screenshot_handler.cpp | 36 +++-------- framework/decode/screenshot_handler.h | 4 -- framework/decode/vulkan_address_replacer.cpp | 67 +++++--------------- framework/decode/vulkan_address_replacer.h | 21 ++++-- framework/graphics/vulkan_device_util.cpp | 19 ++++++ framework/graphics/vulkan_device_util.h | 4 ++ 6 files changed, 62 insertions(+), 89 deletions(-) diff --git a/framework/decode/screenshot_handler.cpp b/framework/decode/screenshot_handler.cpp index 27c7165c37..e0dbb35893 100644 --- a/framework/decode/screenshot_handler.cpp +++ b/framework/decode/screenshot_handler.cpp @@ -539,25 +539,6 @@ VkDeviceSize ScreenshotHandler::GetCopyBufferSize(VkDevice return memory_requirements.size; } -uint32_t ScreenshotHandler::GetMemoryTypeIndex(const VkPhysicalDeviceMemoryProperties& memory_properties, - uint32_t type_bits, - VkMemoryPropertyFlags property_flags) const -{ - uint32_t memory_type_index = std::numeric_limits::max(); - - for (uint32_t i = 0; i < memory_properties.memoryTypeCount; ++i) - { - if ((type_bits & (1 << i)) && - ((memory_properties.memoryTypes[i].propertyFlags & property_flags) == property_flags)) - { - memory_type_index = i; - break; - } - } - - return memory_type_index; -} - VkResult ScreenshotHandler::CreateCopyResource(VkDevice device, const encode::VulkanDeviceTable* device_table, const VkPhysicalDeviceMemoryProperties& memory_properties, @@ -597,17 +578,17 @@ VkResult ScreenshotHandler::CreateCopyResource(VkDevice device_table->GetBufferMemoryRequirements(device, copy_resource->buffer, &memory_requirements); uint32_t memory_type_index = - GetMemoryTypeIndex(memory_properties, - memory_requirements.memoryTypeBits, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + graphics::GetMemoryTypeIndex(memory_properties, + memory_requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); if (memory_type_index == std::numeric_limits::max()) { /* fallback to coherent */ - memory_type_index = - GetMemoryTypeIndex(memory_properties, - memory_requirements.memoryTypeBits, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + memory_type_index = graphics::GetMemoryTypeIndex(memory_properties, + memory_requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); } assert(memory_type_index != std::numeric_limits::max()); @@ -660,9 +641,8 @@ VkResult ScreenshotHandler::CreateCopyResource(VkDevice VkMemoryRequirements memory_requirements; device_table->GetImageMemoryRequirements(device, copy_resource->convert_image, &memory_requirements); - uint32_t memory_type_index = GetMemoryTypeIndex( + uint32_t memory_type_index = graphics::GetMemoryTypeIndex( memory_properties, memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - assert(memory_type_index != std::numeric_limits::max()); VkMemoryAllocateInfo allocate_info = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO }; diff --git a/framework/decode/screenshot_handler.h b/framework/decode/screenshot_handler.h index e7c82236ab..af56b6fc2b 100644 --- a/framework/decode/screenshot_handler.h +++ b/framework/decode/screenshot_handler.h @@ -100,10 +100,6 @@ class ScreenshotHandler : public ScreenshotHandlerBase uint32_t width, uint32_t height) const; - uint32_t GetMemoryTypeIndex(const VkPhysicalDeviceMemoryProperties& memory_properties, - uint32_t type_bits, - VkMemoryPropertyFlags property_flags) const; - VkResult CreateCopyResource(VkDevice device, const encode::VulkanDeviceTable* device_table, const VkPhysicalDeviceMemoryProperties& memory_properties, diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index d340dcffb0..a74a08db20 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -55,25 +55,6 @@ inline uint32_t div_up(uint32_t nom, uint32_t denom) return (nom + denom - 1) / denom; } -uint32_t get_memory_type_index(const VkPhysicalDeviceMemoryProperties& memory_properties, - uint32_t type_bits, - VkMemoryPropertyFlags property_flags) -{ - uint32_t memory_type_index = std::numeric_limits::max(); - - for (uint32_t i = 0; i < memory_properties.memoryTypeCount; ++i) - { - if ((type_bits & (1 << i)) && - ((memory_properties.memoryTypes[i].propertyFlags & property_flags) == property_flags)) - { - memory_type_index = i; - break; - } - } - - return memory_type_index; -} - struct hashmap_t { VkDeviceAddress storage; @@ -140,11 +121,6 @@ VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* } } -VulkanAddressReplacer::VulkanAddressReplacer(VulkanAddressReplacer&& other) noexcept : VulkanAddressReplacer() -{ - swap(*this, other); -} - VulkanAddressReplacer::~VulkanAddressReplacer() { mark_injected_commands_helper_t mark_injected_commands_helper; @@ -474,6 +450,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( "too small (%d < %d)", buffer_info->size, build_size_info.accelerationStructureSize); + + // TODO: create replacement AS } } @@ -735,17 +713,17 @@ bool VulkanAddressReplacer::create_buffer(size_t _device_table->GetBufferMemoryRequirements(_device, buffer_context.buffer, &memory_requirements); uint32_t memory_type_index = - get_memory_type_index(_memory_properties, - memory_requirements.memoryTypeBits, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + graphics::GetMemoryTypeIndex(_memory_properties, + memory_requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); if (memory_type_index == std::numeric_limits::max()) { /* fallback to coherent */ memory_type_index = - get_memory_type_index(_memory_properties, - memory_requirements.memoryTypeBits, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + graphics::GetMemoryTypeIndex(_memory_properties, + memory_requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); } GFXRECON_ASSERT(memory_type_index != std::numeric_limits::max()); @@ -792,6 +770,14 @@ bool VulkanAddressReplacer::create_buffer(size_t return result == VK_SUCCESS; } +std::optional +VulkanAddressReplacer::retrieve_acceleration_structure_asset(VkAccelerationStructureKHR) +{ + + // TODO: implement + return {}; +} + void VulkanAddressReplacer::barrier(VkCommandBuffer command_buffer, VkBuffer buffer, VkPipelineStageFlags src_stage, @@ -812,26 +798,5 @@ void VulkanAddressReplacer::barrier(VkCommandBuffer command_buffer, command_buffer, src_stage, dst_stage, VkDependencyFlags(0), 0, nullptr, 1, &barrier, 0, nullptr); } -void swap(VulkanAddressReplacer& lhs, VulkanAddressReplacer& rhs) noexcept -{ - std::swap(lhs._device_table, rhs._device_table); - std::swap(lhs._memory_properties, rhs._memory_properties); - std::swap(lhs._capture_ray_properties, rhs._capture_ray_properties); - std::swap(lhs._replay_ray_properties, rhs._replay_ray_properties); - std::swap(lhs._valid_sbt_alignment, rhs._valid_sbt_alignment); - std::swap(lhs._device, rhs._device); - std::swap(lhs._resource_allocator, rhs._resource_allocator); - std::swap(lhs._pipeline_layout, rhs._pipeline_layout); - std::swap(lhs._pipeline_sbt, rhs._pipeline_sbt); - std::swap(lhs._pipeline_bda, rhs._pipeline_bda); - std::swap(lhs._pipeline_context_sbt, rhs._pipeline_context_sbt); - std::swap(lhs._pipeline_context_bda, rhs._pipeline_context_bda); - std::swap(lhs._hashmap_sbt, rhs._hashmap_sbt); - std::swap(lhs._hashmap_bda, rhs._hashmap_bda); - std::swap(lhs._shadow_sbt_map, rhs._shadow_sbt_map); - std::swap(lhs._get_device_address_fn_, rhs._get_device_address_fn_); - std::swap(lhs._get_acceleration_build_sizes_fn, rhs._get_acceleration_build_sizes_fn); -} - GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 0d56e5854d..6e856e9754 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -51,12 +51,12 @@ class VulkanAddressReplacer VulkanAddressReplacer(const VulkanAddressReplacer&) = delete; //! allow moving - VulkanAddressReplacer(VulkanAddressReplacer&& other) noexcept; + VulkanAddressReplacer(VulkanAddressReplacer&& other) noexcept = default; ~VulkanAddressReplacer(); /** - * @brief ProcessCmdTraceRays will check and potentially correct input-parameters to 'VkCmdTraceRays', + * @brief ProcessCmdTraceRays will check and potentially correct input-parameters to 'vkCmdTraceRays', * like buffer-device-addresses and shader-group-handles. * * Depending on capture- and replay-device-properties one of the following strategies will be used: @@ -89,7 +89,7 @@ class VulkanAddressReplacer /** * @brief ProcessCmdBuildAccelerationStructuresKHR will check - * and potentially correct input-parameters to 'VkCmdBuildAccelerationStructuresKHR' + * and potentially correct input-parameters to 'vkCmdBuildAccelerationStructuresKHR' * * @param command_buffer_info a provided VulkanCommandBufferInfo * @param info_count number of elements in 'build_geometry_infos' @@ -103,8 +103,6 @@ class VulkanAddressReplacer VkAccelerationStructureBuildRangeInfoKHR** build_range_infos, const decode::VulkanDeviceAddressTracker& address_tracker); - friend void swap(VulkanAddressReplacer& lhs, VulkanAddressReplacer& rhs) noexcept; - private: struct buffer_context_t { @@ -126,10 +124,20 @@ class VulkanAddressReplacer buffer_context_t hashmap_storage = {}; }; + struct acceleration_structure_asset_t + { + VkAccelerationStructureKHR handle = VK_NULL_HANDLE; + VkDeviceAddress address = 0; + buffer_context_t buffer = {}; + }; + [[nodiscard]] bool init_pipeline(); [[nodiscard]] bool create_buffer(size_t num_bytes, buffer_context_t& buffer_context, uint32_t usage_flags = 0); + [[nodiscard]] std::optional + retrieve_acceleration_structure_asset(VkAccelerationStructureKHR handle); + void barrier(VkCommandBuffer command_buffer, VkBuffer buffer, VkPipelineStageFlags src_stage, @@ -154,6 +162,7 @@ class VulkanAddressReplacer // pipeline dealing with buffer-device-addresses (BDA), replacing addresses VkPipeline _pipeline_bda = VK_NULL_HANDLE; + // TODO: need to provide per VkCommandBuffer pipeline_context_t _pipeline_context_sbt; pipeline_context_t _pipeline_context_bda; @@ -161,7 +170,7 @@ class VulkanAddressReplacer util::linear_hashmap _hashmap_bda; std::unordered_map _shadow_sbt_map; - // acquired function pointers + // required function pointers PFN_vkGetBufferDeviceAddress _get_device_address_fn_ = nullptr; PFN_vkGetAccelerationStructureBuildSizesKHR _get_acceleration_build_sizes_fn = nullptr; }; diff --git a/framework/graphics/vulkan_device_util.cpp b/framework/graphics/vulkan_device_util.cpp index be77b31f3f..27cb2d1b42 100644 --- a/framework/graphics/vulkan_device_util.cpp +++ b/framework/graphics/vulkan_device_util.cpp @@ -28,6 +28,25 @@ GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(graphics) +uint32_t GetMemoryTypeIndex(const VkPhysicalDeviceMemoryProperties& memory_properties, + uint32_t type_bits, + VkMemoryPropertyFlags property_flags) +{ + uint32_t memory_type_index = std::numeric_limits::max(); + + for (uint32_t i = 0; i < memory_properties.memoryTypeCount; ++i) + { + if ((type_bits & (1 << i)) && + ((memory_properties.memoryTypes[i].propertyFlags & property_flags) == property_flags)) + { + memory_type_index = i; + break; + } + } + + return memory_type_index; +} + // Query specific physical device features struct // Requires Vulkan version >= 1.1 or VK_KHR_get_physical_device_properties2 // feature_struct sType must be set, pNext must be nullptr diff --git a/framework/graphics/vulkan_device_util.h b/framework/graphics/vulkan_device_util.h index 3472d253c9..eaa91dace1 100644 --- a/framework/graphics/vulkan_device_util.h +++ b/framework/graphics/vulkan_device_util.h @@ -37,6 +37,10 @@ struct VulkanReplayDeviceInfo; GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(graphics) +uint32_t GetMemoryTypeIndex(const VkPhysicalDeviceMemoryProperties& memory_properties, + uint32_t type_bits, + VkMemoryPropertyFlags property_flags); + struct VulkanDevicePropertyFeatureInfo { uint32_t property_shaderGroupHandleSize{ 0 }; From 9684691bd328c0b306ff41aa44a441592a23b406 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Wed, 11 Dec 2024 13:04:46 +0100 Subject: [PATCH 03/23] shadow acceleration-structure creation --- framework/decode/vulkan_address_replacer.cpp | 118 +++++++++++++++---- framework/decode/vulkan_address_replacer.h | 20 ++-- 2 files changed, 110 insertions(+), 28 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index a74a08db20..2442fab460 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -100,7 +100,9 @@ VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* ? device_table->GetBufferDeviceAddress : device_table->GetBufferDeviceAddressKHR; - _get_acceleration_build_sizes_fn = device_table->GetAccelerationStructureBuildSizesKHR; + _get_acceleration_build_sizes_fn = device_table->GetAccelerationStructureBuildSizesKHR; + _create_acceleration_structure_fn = device_table->CreateAccelerationStructureKHR; + _get_acceleration_structure_device_address_fn = device_table->GetAccelerationStructureDeviceAddressKHR; if (physical_device_info != nullptr && physical_device_info->capture_raytracing_properties && physical_device_info->replay_device_info->raytracing_properties) @@ -444,6 +446,11 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto* buffer_info = address_tracker.GetBufferByHandle(acceleration_structure_info->buffer); GFXRECON_ASSERT(buffer_info != nullptr) + // fails on some!? captures + auto* scratch_buffer_info = + address_tracker.GetBufferByCaptureDeviceAddress(build_geometry_info.scratchData.deviceAddress); + // GFXRECON_ASSERT(scratch_buffer_info != nullptr) + if (buffer_info != nullptr && buffer_info->size < build_size_info.accelerationStructureSize) { GFXRECON_LOG_WARNING("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: buffer-size is " @@ -451,7 +458,67 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( buffer_info->size, build_size_info.accelerationStructureSize); - // TODO: create replacement AS + auto& replacment_as = _shadow_as_map[build_geometry_info.dstAccelerationStructure]; + + if (replacment_as.handle == VK_NULL_HANDLE) + { + // create a replacement acceleration-structure with proper sized buffer + bool success = create_buffer(build_size_info.accelerationStructureSize, + replacment_as.storage, + VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR, + false); + + if (!success) + { + GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: shadow-buffer creation failed"); + return; + } + + VkAccelerationStructureCreateInfoKHR as_create_info = {}; + as_create_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR; + as_create_info.buffer = replacment_as.storage.buffer; + as_create_info.size = build_size_info.accelerationStructureSize; + as_create_info.type = build_geometry_info.type; + + VkResult res = + _create_acceleration_structure_fn(_device, &as_create_info, nullptr, &replacment_as.handle); + + if (res != VK_SUCCESS || replacment_as.handle == VK_NULL_HANDLE) + { + GFXRECON_LOG_ERROR( + "ProcessCmdBuildAccelerationStructuresKHR: shadow acceleration-structure creation failed"); + return; + } + VkAccelerationStructureDeviceAddressInfoKHR acceleration_address_info = {}; + acceleration_address_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR; + acceleration_address_info.accelerationStructure = replacment_as.handle; + replacment_as.address = + _get_acceleration_structure_device_address_fn(_device, &acceleration_address_info); + GFXRECON_ASSERT(replacment_as.address != 0) + + // hot swap acceleration-structure handle + build_geometry_info.dstAccelerationStructure = replacment_as.handle; + } + + // determine required size of scratch-buffer + uint32_t scratch_size = build_geometry_info.mode == VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR + ? build_size_info.buildScratchSize + : build_size_info.updateScratchSize; + + // if (scratch_buffer_info != nullptr && scratch_buffer_info->size < scratch_size) + { + // create a replacement scratch-buffer + bool success = create_buffer(scratch_size, replacment_as.scratch, 0, false); + + if (!success) + { + GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: scratch-buffer creation failed"); + return; + } + + // hot swap scratch-buffer + build_geometry_info.scratchData.deviceAddress = replacment_as.scratch.device_address; + } } } @@ -681,7 +748,8 @@ bool VulkanAddressReplacer::init_pipeline() bool VulkanAddressReplacer::create_buffer(size_t num_bytes, VulkanAddressReplacer::buffer_context_t& buffer_context, - uint32_t usage_flags) + uint32_t usage_flags, + bool use_host_mem) { // nothing to do if (num_bytes <= buffer_context.num_bytes) @@ -712,18 +780,18 @@ bool VulkanAddressReplacer::create_buffer(size_t VkMemoryRequirements memory_requirements; _device_table->GetBufferMemoryRequirements(_device, buffer_context.buffer, &memory_requirements); + VkMemoryPropertyFlags memory_property_flags = + use_host_mem ? VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT + : VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + uint32_t memory_type_index = - graphics::GetMemoryTypeIndex(_memory_properties, - memory_requirements.memoryTypeBits, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + graphics::GetMemoryTypeIndex(_memory_properties, memory_requirements.memoryTypeBits, memory_property_flags); if (memory_type_index == std::numeric_limits::max()) { /* fallback to coherent */ memory_type_index = - graphics::GetMemoryTypeIndex(_memory_properties, - memory_requirements.memoryTypeBits, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + graphics::GetMemoryTypeIndex(_memory_properties, memory_requirements.memoryTypeBits, memory_property_flags); } GFXRECON_ASSERT(memory_type_index != std::numeric_limits::max()); @@ -763,19 +831,16 @@ bool VulkanAddressReplacer::create_buffer(size_t address_info.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO; address_info.buffer = buffer_context.buffer; buffer_context.device_address = _get_device_address_fn_(_device, &address_info); + GFXRECON_ASSERT(buffer_context.device_address != 0); - // map buffer - result = _resource_allocator->MapResourceMemoryDirect( - VK_WHOLE_SIZE, 0, &buffer_context.mapped_data, buffer_context.allocator_data); - return result == VK_SUCCESS; -} - -std::optional -VulkanAddressReplacer::retrieve_acceleration_structure_asset(VkAccelerationStructureKHR) -{ - - // TODO: implement - return {}; + if (use_host_mem) + { + // map buffer + result = _resource_allocator->MapResourceMemoryDirect( + VK_WHOLE_SIZE, 0, &buffer_context.mapped_data, buffer_context.allocator_data); + return result == VK_SUCCESS; + } + return true; } void VulkanAddressReplacer::barrier(VkCommandBuffer command_buffer, @@ -798,5 +863,16 @@ void VulkanAddressReplacer::barrier(VkCommandBuffer command_buffer, command_buffer, src_stage, dst_stage, VkDependencyFlags(0), 0, nullptr, 1, &barrier, 0, nullptr); } +void VulkanAddressReplacer::DestroyShadowResources(VkAccelerationStructureKHR handle) +{ + auto remove_it = _shadow_as_map.find(handle); + + if (remove_it != _shadow_as_map.end()) + { + mark_injected_commands_helper_t mark_injected_commands_helper; + _shadow_as_map.erase(remove_it); + } +} + GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 6e856e9754..41e2575d05 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -103,6 +103,8 @@ class VulkanAddressReplacer VkAccelerationStructureBuildRangeInfoKHR** build_range_infos, const decode::VulkanDeviceAddressTracker& address_tracker); + void DestroyShadowResources(VkAccelerationStructureKHR handle); + private: struct buffer_context_t { @@ -128,15 +130,16 @@ class VulkanAddressReplacer { VkAccelerationStructureKHR handle = VK_NULL_HANDLE; VkDeviceAddress address = 0; - buffer_context_t buffer = {}; + buffer_context_t storage = {}; + buffer_context_t scratch = {}; }; [[nodiscard]] bool init_pipeline(); - [[nodiscard]] bool create_buffer(size_t num_bytes, buffer_context_t& buffer_context, uint32_t usage_flags = 0); - - [[nodiscard]] std::optional - retrieve_acceleration_structure_asset(VkAccelerationStructureKHR handle); + [[nodiscard]] bool create_buffer(size_t num_bytes, + buffer_context_t& buffer_context, + uint32_t usage_flags = 0, + bool use_host_mem = true); void barrier(VkCommandBuffer command_buffer, VkBuffer buffer, @@ -169,10 +172,13 @@ class VulkanAddressReplacer util::linear_hashmap _hashmap_sbt; util::linear_hashmap _hashmap_bda; std::unordered_map _shadow_sbt_map; + std::unordered_map _shadow_as_map; // required function pointers - PFN_vkGetBufferDeviceAddress _get_device_address_fn_ = nullptr; - PFN_vkGetAccelerationStructureBuildSizesKHR _get_acceleration_build_sizes_fn = nullptr; + PFN_vkGetBufferDeviceAddress _get_device_address_fn_ = nullptr; + PFN_vkGetAccelerationStructureBuildSizesKHR _get_acceleration_build_sizes_fn = nullptr; + PFN_vkCreateAccelerationStructureKHR _create_acceleration_structure_fn = nullptr; + PFN_vkGetAccelerationStructureDeviceAddressKHR _get_acceleration_structure_device_address_fn = nullptr; }; GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) From e23d6ae43f4fed55051834e8eb9c254bbb61ab69 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Wed, 11 Dec 2024 15:55:45 +0100 Subject: [PATCH 04/23] work on shadow-AS cleanup and buffer-alignment --- framework/decode/vulkan_address_replacer.cpp | 167 ++++++++--- framework/decode/vulkan_address_replacer.h | 51 +++- framework/decode/vulkan_object_info.h | 3 +- framework/decode/vulkan_rebind_allocator.cpp | 15 +- framework/decode/vulkan_rebind_allocator.h | 3 + .../decode/vulkan_replay_consumer_base.cpp | 260 ++++++++++-------- 6 files changed, 318 insertions(+), 181 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 2442fab460..32b3a13238 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -44,9 +44,14 @@ struct mark_injected_commands_helper_t } }; +inline VkDeviceAddress aligned_address(VkDeviceAddress address, uint64_t alignment) +{ + return alignment ? (address + alignment - 1) & ~(alignment - 1) : address; +} + inline uint32_t aligned_size(uint32_t size, uint32_t alignment) { - return (size + alignment - 1) & ~(alignment - 1); + return alignment ? (size + alignment - 1) & ~(alignment - 1) : size; } inline uint32_t div_up(uint32_t nom, uint32_t denom) @@ -77,12 +82,27 @@ decode::VulkanAddressReplacer::buffer_context_t::~buffer_context_t() { if (buffer != VK_NULL_HANDLE) { + // unmap/destroy buffer + resource_allocator->UnmapResourceMemoryDirect(allocator_data); resource_allocator->DestroyBufferDirect(buffer, nullptr, allocator_data); } if (device_memory != VK_NULL_HANDLE) { resource_allocator->FreeMemoryDirect(device_memory, nullptr, memory_data); } + + resource_allocator = nullptr; + buffer = VK_NULL_HANDLE; + device_memory = VK_NULL_HANDLE; + } +} + +decode::VulkanAddressReplacer::acceleration_structure_asset_t::~acceleration_structure_asset_t() +{ + if (handle != VK_NULL_HANDLE) + { + GFXRECON_ASSERT(destroy_fn != nullptr && device != VK_NULL_HANDLE) + destroy_fn(device, handle, nullptr); } } @@ -100,16 +120,18 @@ VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* ? device_table->GetBufferDeviceAddress : device_table->GetBufferDeviceAddressKHR; - _get_acceleration_build_sizes_fn = device_table->GetAccelerationStructureBuildSizesKHR; - _create_acceleration_structure_fn = device_table->CreateAccelerationStructureKHR; - _get_acceleration_structure_device_address_fn = device_table->GetAccelerationStructureDeviceAddressKHR; - if (physical_device_info != nullptr && physical_device_info->capture_raytracing_properties && physical_device_info->replay_device_info->raytracing_properties) { _capture_ray_properties = *physical_device_info->capture_raytracing_properties; _replay_ray_properties = *physical_device_info->replay_device_info->raytracing_properties; + if (physical_device_info->replay_device_info->acceleration_structure_properties) + { + _replay_acceleration_structure_properties = + *physical_device_info->replay_device_info->acceleration_structure_properties; + } + if (_capture_ray_properties.shaderGroupHandleSize != _replay_ray_properties.shaderGroupHandleSize || _capture_ray_properties.shaderGroupHandleAlignment != _replay_ray_properties.shaderGroupHandleAlignment || _capture_ray_properties.shaderGroupBaseAlignment != _replay_ray_properties.shaderGroupBaseAlignment) @@ -128,9 +150,11 @@ VulkanAddressReplacer::~VulkanAddressReplacer() mark_injected_commands_helper_t mark_injected_commands_helper; // explicitly free resources here, in order to mark destruction API-calls as injected - _pipeline_context_sbt = {}; - _pipeline_context_bda = {}; - _shadow_sbt_map = {}; + _pipeline_sbt_context_map = {}; + _build_as_context_map = {}; + + _shadow_sbt_map = {}; + _shadow_as_map = {}; if (_pipeline_bda != VK_NULL_HANDLE) { @@ -177,7 +201,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } // TODO: testing only -> remove when closing issue #1526 - // valid_sbt_alignment_ = false; + // _valid_sbt_alignment = false; // valid_group_handles = false; if (!_valid_sbt_alignment || !valid_group_handles) @@ -233,20 +257,22 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( _hashmap_sbt.put(lhs, rhs); } - if (!create_buffer(_hashmap_sbt.get_storage(nullptr), _pipeline_context_sbt.hashmap_storage)) + // get a context for this command-buffer + auto& pipeline_context_sbt = _pipeline_sbt_context_map[command_buffer_info->handle]; + + if (!create_buffer(_hashmap_sbt.get_storage(nullptr), pipeline_context_sbt.hashmap_storage)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); } - _hashmap_sbt.get_storage(_pipeline_context_sbt.hashmap_storage.mapped_data); + _hashmap_sbt.get_storage(pipeline_context_sbt.hashmap_storage.mapped_data); // input-handles constexpr uint32_t max_num_handles = 4; - if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), _pipeline_context_sbt.input_handle_buffer)) + if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), pipeline_context_sbt.input_handle_buffer)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); } - auto input_addresses = - reinterpret_cast(_pipeline_context_sbt.input_handle_buffer.mapped_data); + auto input_addresses = reinterpret_cast(pipeline_context_sbt.input_handle_buffer.mapped_data); uint32_t num_addresses = 0; for (const auto& region : { raygen_sbt, miss_sbt, hit_sbt, callable_sbt }) @@ -258,11 +284,11 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } replacer_params_t replacer_params = {}; - replacer_params.hashmap.storage = _pipeline_context_sbt.hashmap_storage.device_address; + replacer_params.hashmap.storage = pipeline_context_sbt.hashmap_storage.device_address; replacer_params.hashmap.size = _hashmap_sbt.size(); replacer_params.hashmap.capacity = _hashmap_sbt.capacity(); - replacer_params.input_handles = _pipeline_context_sbt.input_handle_buffer.device_address; + replacer_params.input_handles = pipeline_context_sbt.input_handle_buffer.device_address; replacer_params.num_handles = num_addresses; if (_valid_sbt_alignment) @@ -289,13 +315,13 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( "Replay adjusted a mismatching raytracing shader-binding-table using a shadow-buffer"); // output-handles - if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), _pipeline_context_sbt.output_handle_buffer)) + if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), pipeline_context_sbt.output_handle_buffer)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); return; } // output to shadow-sbt-buffer - replacer_params.output_handles = _pipeline_context_sbt.output_handle_buffer.device_address; + replacer_params.output_handles = pipeline_context_sbt.output_handle_buffer.device_address; // find/create shadow-SBT-buffer uint32_t sbt_offset = 0; @@ -328,7 +354,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } auto output_addresses = - reinterpret_cast(_pipeline_context_sbt.output_handle_buffer.mapped_data); + reinterpret_cast(pipeline_context_sbt.output_handle_buffer.mapped_data); uint32_t out_index = 0; sbt_offset = 0; for (auto& region : { raygen_sbt, miss_sbt, hit_sbt, callable_sbt }) @@ -390,7 +416,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( GFXRECON_ASSERT(_device_table != nullptr); // TODO: testing only -> remove when closing issue #1526 - constexpr bool force_replace = false; + constexpr bool force_replace = true; std::unordered_set buffer_set; auto address_remap = [&address_tracker, &buffer_set](VkDeviceAddress& capture_address) { @@ -433,11 +459,11 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( { primitive_counts[j] = range_info->primitiveCount; } - _get_acceleration_build_sizes_fn(_device, - VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, - &build_geometry_info, - primitive_counts.data(), - &build_size_info); + _device_table->GetAccelerationStructureBuildSizesKHR(_device, + VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, + &build_geometry_info, + primitive_counts.data(), + &build_size_info); // retrieve VkAccelerationStructureKHR -> VkBuffer -> check/correct size auto* acceleration_structure_info = @@ -446,7 +472,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto* buffer_info = address_tracker.GetBufferByHandle(acceleration_structure_info->buffer); GFXRECON_ASSERT(buffer_info != nullptr) - // fails on some!? captures + // TODO: fails on some!? captures auto* scratch_buffer_info = address_tracker.GetBufferByCaptureDeviceAddress(build_geometry_info.scratchData.deviceAddress); // GFXRECON_ASSERT(scratch_buffer_info != nullptr) @@ -462,10 +488,14 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (replacment_as.handle == VK_NULL_HANDLE) { + replacment_as.device = _device; + replacment_as.destroy_fn = _device_table->DestroyAccelerationStructureKHR; + // create a replacement acceleration-structure with proper sized buffer bool success = create_buffer(build_size_info.accelerationStructureSize, replacment_as.storage, VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR, + 0, false); if (!success) @@ -480,8 +510,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( as_create_info.size = build_size_info.accelerationStructureSize; as_create_info.type = build_geometry_info.type; - VkResult res = - _create_acceleration_structure_fn(_device, &as_create_info, nullptr, &replacment_as.handle); + VkResult res = _device_table->CreateAccelerationStructureKHR( + _device, &as_create_info, nullptr, &replacment_as.handle); if (res != VK_SUCCESS || replacment_as.handle == VK_NULL_HANDLE) { @@ -493,7 +523,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( acceleration_address_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR; acceleration_address_info.accelerationStructure = replacment_as.handle; replacment_as.address = - _get_acceleration_structure_device_address_fn(_device, &acceleration_address_info); + _device_table->GetAccelerationStructureDeviceAddressKHR(_device, &acceleration_address_info); GFXRECON_ASSERT(replacment_as.address != 0) // hot swap acceleration-structure handle @@ -508,7 +538,12 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // if (scratch_buffer_info != nullptr && scratch_buffer_info->size < scratch_size) { // create a replacement scratch-buffer - bool success = create_buffer(scratch_size, replacment_as.scratch, 0, false); + bool success = create_buffer( + scratch_size, + replacment_as.scratch, + 0, + _replay_acceleration_structure_properties.minAccelerationStructureScratchOffsetAlignment, + false); if (!success) { @@ -573,10 +608,16 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto acceleration_structure_map = address_tracker.GetAccelerationStructureDeviceAddressMap(); for (const auto& [capture_address, replay_address] : acceleration_structure_map) { + auto* accel_info = address_tracker.GetAccelerationStructureByCaptureDeviceAddress(capture_address); + GFXRECON_ASSERT(accel_info != nullptr) + if (force_replace || capture_address != replay_address) { + // TODO: extra look-up required for replaced AS + auto new_address = replay_address; + // store addresses we will need to replace - _hashmap_bda.put(capture_address, replay_address); + _hashmap_bda.put(capture_address, new_address); } } @@ -595,28 +636,30 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } } - if (!create_buffer(_hashmap_bda.get_storage(nullptr), _pipeline_context_bda.hashmap_storage)) + auto& pipeline_context_bda = _build_as_context_map[command_buffer_info->handle]; + + if (!create_buffer(_hashmap_bda.get_storage(nullptr), pipeline_context_bda.hashmap_storage)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); } - _hashmap_bda.get_storage(_pipeline_context_bda.hashmap_storage.mapped_data); + _hashmap_bda.get_storage(pipeline_context_bda.hashmap_storage.mapped_data); uint32_t num_bytes = addresses_to_replace.size() * sizeof(VkDeviceAddress); - if (!create_buffer(num_bytes, _pipeline_context_bda.input_handle_buffer)) + if (!create_buffer(num_bytes, pipeline_context_bda.input_handle_buffer)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); } - memcpy(_pipeline_context_bda.input_handle_buffer.mapped_data, addresses_to_replace.data(), num_bytes); + memcpy(pipeline_context_bda.input_handle_buffer.mapped_data, addresses_to_replace.data(), num_bytes); replacer_params_t replacer_params = {}; - replacer_params.hashmap.storage = _pipeline_context_bda.hashmap_storage.device_address; + replacer_params.hashmap.storage = pipeline_context_bda.hashmap_storage.device_address; replacer_params.hashmap.size = _hashmap_bda.size(); replacer_params.hashmap.capacity = _hashmap_bda.capacity(); // in-place - replacer_params.input_handles = _pipeline_context_bda.input_handle_buffer.device_address; - replacer_params.output_handles = _pipeline_context_bda.input_handle_buffer.device_address; + replacer_params.input_handles = pipeline_context_bda.input_handle_buffer.device_address; + replacer_params.output_handles = pipeline_context_bda.input_handle_buffer.device_address; replacer_params.num_handles = addresses_to_replace.size(); @@ -629,7 +672,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( 0, sizeof(replacer_params_t), &replacer_params); - // run a single workgroup + // dispatch workgroups constexpr uint32_t wg_size = 32; _device_table->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); @@ -640,7 +683,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( buf, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT, - VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR, + VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR, VK_ACCESS_SHADER_READ_BIT); } @@ -749,8 +792,13 @@ bool VulkanAddressReplacer::init_pipeline() bool VulkanAddressReplacer::create_buffer(size_t num_bytes, VulkanAddressReplacer::buffer_context_t& buffer_context, uint32_t usage_flags, + uint32_t min_alignment, bool use_host_mem) { + // 4kB min-size + constexpr uint32_t min_buffer_size = 1 << 12; + num_bytes = aligned_size(std::max(num_bytes, min_buffer_size), min_alignment); + // nothing to do if (num_bytes <= buffer_context.num_bytes) { @@ -833,6 +881,9 @@ bool VulkanAddressReplacer::create_buffer(size_t buffer_context.device_address = _get_device_address_fn_(_device, &address_info); GFXRECON_ASSERT(buffer_context.device_address != 0); + // ensure alignment for returned address + buffer_context.device_address = aligned_address(buffer_context.device_address, min_alignment); + if (use_host_mem) { // map buffer @@ -865,12 +916,42 @@ void VulkanAddressReplacer::barrier(VkCommandBuffer command_buffer, void VulkanAddressReplacer::DestroyShadowResources(VkAccelerationStructureKHR handle) { - auto remove_it = _shadow_as_map.find(handle); + auto remove_as_it = _shadow_as_map.find(handle); - if (remove_it != _shadow_as_map.end()) + if (remove_as_it != _shadow_as_map.end()) { mark_injected_commands_helper_t mark_injected_commands_helper; - _shadow_as_map.erase(remove_it); + _shadow_as_map.erase(remove_as_it); + } +} + +void VulkanAddressReplacer::DestroyShadowResources(const VulkanCommandBufferInfo* command_buffer_info) +{ + if (command_buffer_info != nullptr) + { + auto remove_context_it = _build_as_context_map.find(command_buffer_info->handle); + + if (remove_context_it != _build_as_context_map.end()) + { + mark_injected_commands_helper_t mark_injected_commands_helper; + _build_as_context_map.erase(remove_context_it); + } + + auto shadow_sbt_it = _shadow_sbt_map.find(command_buffer_info->handle); + + if (shadow_sbt_it != _shadow_sbt_map.end()) + { + mark_injected_commands_helper_t mark_injected_commands_helper; + _shadow_sbt_map.erase(shadow_sbt_it); + } + + auto pipeline_sbt_it = _pipeline_sbt_context_map.find(command_buffer_info->handle); + + if (pipeline_sbt_it != _pipeline_sbt_context_map.end()) + { + mark_injected_commands_helper_t mark_injected_commands_helper; + _pipeline_sbt_context_map.erase(pipeline_sbt_it); + } } } diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 41e2575d05..a15fa027de 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -103,8 +103,22 @@ class VulkanAddressReplacer VkAccelerationStructureBuildRangeInfoKHR** build_range_infos, const decode::VulkanDeviceAddressTracker& address_tracker); + /** + * @brief DestroyShadowResources should be called upon destruction of provided VkAccelerationStructureKHR handle, + * allowing this class to free potential resources associated with it. + * + * @param handle a provided VkAccelerationStructureKHR handle + */ void DestroyShadowResources(VkAccelerationStructureKHR handle); + /** + * @brief DestroyShadowResources should be called upon destruction of provided command_buffer_info, + * allowing this class to free potential resources associated with it. + * + * @param handle a provided VulkanCommandBufferInfo + */ + void DestroyShadowResources(const VulkanCommandBufferInfo* command_buffer_info); + private: struct buffer_context_t { @@ -132,14 +146,19 @@ class VulkanAddressReplacer VkDeviceAddress address = 0; buffer_context_t storage = {}; buffer_context_t scratch = {}; + + VkDevice device = VK_NULL_HANDLE; + PFN_vkDestroyAccelerationStructureKHR destroy_fn = nullptr; + ~acceleration_structure_asset_t(); }; [[nodiscard]] bool init_pipeline(); [[nodiscard]] bool create_buffer(size_t num_bytes, buffer_context_t& buffer_context, - uint32_t usage_flags = 0, - bool use_host_mem = true); + uint32_t usage_flags = 0, + uint32_t min_alignment = 0, + bool use_host_mem = true); void barrier(VkCommandBuffer command_buffer, VkBuffer buffer, @@ -148,10 +167,11 @@ class VulkanAddressReplacer VkPipelineStageFlags dst_stage, VkAccessFlags dst_access); - const encode::VulkanDeviceTable* _device_table = nullptr; - VkPhysicalDeviceMemoryProperties _memory_properties = {}; - VkPhysicalDeviceRayTracingPipelinePropertiesKHR _capture_ray_properties{}, _replay_ray_properties{}; - bool _valid_sbt_alignment = true; + const encode::VulkanDeviceTable* _device_table = nullptr; + VkPhysicalDeviceMemoryProperties _memory_properties = {}; + VkPhysicalDeviceRayTracingPipelinePropertiesKHR _capture_ray_properties{}, _replay_ray_properties{}; + VkPhysicalDeviceAccelerationStructurePropertiesKHR _replay_acceleration_structure_properties{}; + bool _valid_sbt_alignment = true; VkDevice _device = VK_NULL_HANDLE; decode::VulkanResourceAllocator* _resource_allocator = nullptr; @@ -165,20 +185,21 @@ class VulkanAddressReplacer // pipeline dealing with buffer-device-addresses (BDA), replacing addresses VkPipeline _pipeline_bda = VK_NULL_HANDLE; - // TODO: need to provide per VkCommandBuffer - pipeline_context_t _pipeline_context_sbt; - pipeline_context_t _pipeline_context_bda; - util::linear_hashmap _hashmap_sbt; util::linear_hashmap _hashmap_bda; std::unordered_map _shadow_sbt_map; - std::unordered_map _shadow_as_map; + + // TODO: check if this is sufficient + std::unordered_map _pipeline_sbt_context_map; + + // resources related to acceleration-structures + std::unordered_map _shadow_as_map; + + // TODO: check if this is sufficient + std::unordered_map _build_as_context_map; // required function pointers - PFN_vkGetBufferDeviceAddress _get_device_address_fn_ = nullptr; - PFN_vkGetAccelerationStructureBuildSizesKHR _get_acceleration_build_sizes_fn = nullptr; - PFN_vkCreateAccelerationStructureKHR _create_acceleration_structure_fn = nullptr; - PFN_vkGetAccelerationStructureDeviceAddressKHR _get_acceleration_structure_device_address_fn = nullptr; + PFN_vkGetBufferDeviceAddress _get_device_address_fn_ = nullptr; }; GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_object_info.h b/framework/decode/vulkan_object_info.h index 047d167282..2c5167dc08 100644 --- a/framework/decode/vulkan_object_info.h +++ b/framework/decode/vulkan_object_info.h @@ -166,7 +166,8 @@ struct VulkanReplayDeviceInfo std::optional memory_properties; // extensions - std::optional raytracing_properties; + std::optional raytracing_properties; + std::optional acceleration_structure_properties; }; template diff --git a/framework/decode/vulkan_rebind_allocator.cpp b/framework/decode/vulkan_rebind_allocator.cpp index 4245791c8b..8fb843f2f0 100644 --- a/framework/decode/vulkan_rebind_allocator.cpp +++ b/framework/decode/vulkan_rebind_allocator.cpp @@ -247,9 +247,20 @@ VkResult VulkanRebindAllocator::CreateBuffer(const VkBufferCreateInfo* create VkResult result = VK_ERROR_INITIALIZATION_FAILED; + auto aligned_size = [](uint32_t size, uint32_t alignment) -> uint32_t + { + return (size + alignment - 1) & ~(alignment - 1); + }; + + // define a general minimum aligned-size for buffers, as certain vulkan resources will require this + // e.g. buffers containing shader-binding-tables or scratch-buffers for acceleration-structure build-operations + constexpr uint32_t min_alignment = 64; + if ((create_info != nullptr) && (buffer != nullptr) && (allocator_data != nullptr)) { - result = functions_.create_buffer(device_, create_info, nullptr, buffer); + auto modified_info = *create_info; + modified_info.size = aligned_size(create_info->size, min_alignment); + result = functions_.create_buffer(device_, &modified_info, nullptr, buffer); if (result >= 0) { @@ -570,6 +581,7 @@ VkResult VulkanRebindAllocator::BindBufferMemory(VkBuffer VkMemoryRequirements requirements; functions_.get_buffer_memory_requirements(device_, buffer, &requirements); + requirements.alignment = std::max(requirements.alignment, min_buffer_alignment_); VmaAllocationCreateInfo create_info; create_info.flags = 0; @@ -653,6 +665,7 @@ VkResult VulkanRebindAllocator::BindBufferMemory2(uint32_t VkMemoryRequirements requirements; functions_.get_buffer_memory_requirements(device_, buffer, &requirements); + requirements.alignment = std::max(requirements.alignment, min_buffer_alignment_); VmaAllocationCreateInfo create_info; create_info.flags = 0; diff --git a/framework/decode/vulkan_rebind_allocator.h b/framework/decode/vulkan_rebind_allocator.h index 827c3e9c19..143be38e5d 100644 --- a/framework/decode/vulkan_rebind_allocator.h +++ b/framework/decode/vulkan_rebind_allocator.h @@ -450,6 +450,9 @@ class VulkanRebindAllocator : public VulkanResourceAllocator VkCommandPool cmd_pool_ = VK_NULL_HANDLE; VkQueue staging_queue_ = VK_NULL_HANDLE; uint32_t staging_queue_family_{}; + + //! define a general minimum alignment for buffers + uint32_t min_buffer_alignment_ = 128; }; GFXRECON_END_NAMESPACE(decode) diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index 20be2e9fbe..719519a0c7 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -1401,6 +1401,14 @@ void VulkanReplayConsumerBase::SetPhysicalDeviceProperties(VulkanPhysicalDeviceI physical_device_info->replay_device_info->raytracing_properties = *ray_replay_props; physical_device_info->replay_device_info->raytracing_properties->pNext = nullptr; } + + if (auto acceleration_structure_replay_props = + graphics::vulkan_struct_get_pnext(replay_properties)) + { + physical_device_info->replay_device_info->acceleration_structure_properties = + *acceleration_structure_replay_props; + physical_device_info->replay_device_info->acceleration_structure_properties->pNext = nullptr; + } } void VulkanReplayConsumerBase::SetPhysicalDeviceMemoryProperties( @@ -4297,24 +4305,31 @@ void VulkanReplayConsumerBase::OverrideFreeCommandBuffers(PFN_vkFreeCommandBuffe assert((device_info != nullptr) && (pCommandBuffers != nullptr) && (pCommandBuffers->GetHandlePointer() != nullptr)); - if (options_.dumping_resources && command_pool_info != nullptr) + if (command_pool_info != nullptr) { + const format::HandleId* cmd_buf_handles = pCommandBuffers->GetPointer(); for (uint32_t i = 0; i < command_buffer_count; ++i) { - auto it = command_pool_info->child_ids.find(cmd_buf_handles[i]); - if (it != command_pool_info->child_ids.end()) + if (options_.dumping_resources) { - if (options_.dumping_resources) + auto it = command_pool_info->child_ids.find(cmd_buf_handles[i]); + if (it != command_pool_info->child_ids.end()) { VulkanCommandBufferInfo* cb_info = object_info_table_->GetVkCommandBufferInfo(*it); assert(cb_info != nullptr); resource_dumper_->ResetCommandBuffer(cb_info->handle); } } + + // free potential shadow-resources associated with this command-buffer + VulkanCommandBufferInfo* cb_info = object_info_table_->GetVkCommandBufferInfo(cmd_buf_handles[i]); + if (cb_info != nullptr) + { + GetDeviceAddressReplacer(device_info).DestroyShadowResources(cb_info); + } } } - const VkCommandBuffer* in_pCommandBuffers = pCommandBuffers->GetHandlePointer(); func(device_info->handle, command_pool_info->handle, command_buffer_count, in_pCommandBuffers); } @@ -7755,6 +7770,9 @@ void VulkanReplayConsumerBase::OverrideDestroyAccelerationStructureKHR( // remove from address-tracking GetDeviceAddressTracker(device_info).RemoveAccelerationStructure(acceleration_structure_info); + + // free potential shadow-resources + GetDeviceAddressReplacer(device_info).DestroyShadowResources(acceleration_structure); } func(device_info->handle, acceleration_structure, GetAllocationCallbacks(pAllocator)); } @@ -7869,122 +7887,122 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR( &pPipelines->GetPointer()[createInfoCount]); } - // NOTE: this is almost never true, even on newest desktop-drivers - // TODO: consider removing all of the feature_rayTracingPipelineShaderGroupHandleCaptureReplay logic here - if (device_info->property_feature_info.feature_rayTracingPipelineShaderGroupHandleCaptureReplay) - { - // Modify pipeline create infos with capture replay flag and data. - std::vector modified_create_infos; - std::vector> modified_pgroups; - modified_create_infos.reserve(createInfoCount); - modified_pgroups.resize(createInfoCount); - for (uint32_t create_info_i = 0; create_info_i < createInfoCount; ++create_info_i) - { - format::HandleId pipeline_capture_id = (*pPipelines[create_info_i].GetPointer()); - - // Enable capture replay flag. - modified_create_infos.push_back(in_pCreateInfos[create_info_i]); - modified_create_infos[create_info_i].flags |= - VK_PIPELINE_CREATE_RAY_TRACING_SHADER_GROUP_HANDLE_CAPTURE_REPLAY_BIT_KHR; - - uint32_t group_info_count = in_pCreateInfos[create_info_i].groupCount; - bool has_data = (device_info->shader_group_handles.find(pipeline_capture_id) != - device_info->shader_group_handles.end()); - - if (has_data) - { - assert(device_info->shader_group_handles.at(pipeline_capture_id).size() == - (device_info->property_feature_info.property_shaderGroupHandleCaptureReplaySize * - group_info_count)); - } - else - { - GFXRECON_LOG_WARNING("Missing shader group handle data in for ray tracing pipeline (ID = %" PRIu64 ").", - pipeline_capture_id); - } - - // Set pShaderGroupCaptureReplayHandle in shader group create infos. - std::vector& modified_group_infos = modified_pgroups[create_info_i]; - modified_group_infos.reserve(group_info_count); - - for (uint32_t group_info_i = 0; group_info_i < group_info_count; ++group_info_i) - { - modified_group_infos.push_back(in_pCreateInfos[create_info_i].pGroups[group_info_i]); - - if (has_data) - { - uint32_t byte_offset = - device_info->property_feature_info.property_shaderGroupHandleCaptureReplaySize * group_info_i; - modified_group_infos[group_info_i].pShaderGroupCaptureReplayHandle = - device_info->shader_group_handles.at(pipeline_capture_id).data() + byte_offset; - } - else - { - modified_group_infos[group_info_i].pShaderGroupCaptureReplayHandle = nullptr; - } - } - - // Use modified shader group infos. - modified_create_infos[create_info_i].pGroups = modified_group_infos.data(); - } - - if (omitted_pipeline_cache_data_) - { - AllowCompileDuringPipelineCreation(createInfoCount, modified_create_infos.data()); - } - - VkPipeline* created_pipelines = nullptr; - - if (deferred_operation_info) - { - created_pipelines = deferred_operation_info->replayPipelines.data(); - } - else - { - created_pipelines = out_pPipelines; - } - - result = GetDeviceTable(device)->CreateRayTracingPipelinesKHR(device, - in_deferredOperation, - overridePipelineCache, - createInfoCount, - modified_create_infos.data(), - in_pAllocator, - created_pipelines); - - if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) || - (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)) - { - // The above return values mean the command is not deferred and driver will finish all workload in current - // thread. Therefore the created pipelines can be read and copied to out_pPipelines which will be - // referenced later. - // - // Note: - // Some pipelines might actually fail creation if the return value is VK_PIPELINE_COMPILE_REQUIRED_EXT. - // These failed pipelines will generate VK_NULL_HANDLE. - // - // If the return value is VK_OPERATION_DEFERRED_KHR, it means the command is deferred, and thus pipeline - // creation is not finished. Subsequent handling will be done by - // vkDeferredOperationJoinKHR/vkGetDeferredOperationResultKHR after pipeline creation is finished. - - if (deferred_operation_info) - { - memcpy(out_pPipelines, created_pipelines, createInfoCount * sizeof(VkPipeline)); - - // Eventhough vkCreateRayTracingPipelinesKHR was called with a valid deferred operation object, the - // driver may opt to not defer the command. In this case, set pending_state flag to false to skip - // vkDeferredOperationJoinKHR handling. - deferred_operation_info->pending_state = false; - } - } - - if (deferred_operation_info) - { - deferred_operation_info->record_modified_create_infos = std::move(modified_create_infos); - deferred_operation_info->record_modified_pgroups = std::move(modified_pgroups); - } - } - else +// // NOTE: this is almost never true, even on newest desktop-drivers +// // TODO: consider removing all of the feature_rayTracingPipelineShaderGroupHandleCaptureReplay logic here +// if (device_info->property_feature_info.feature_rayTracingPipelineShaderGroupHandleCaptureReplay) +// { +// // Modify pipeline create infos with capture replay flag and data. +// std::vector modified_create_infos; +// std::vector> modified_pgroups; +// modified_create_infos.reserve(createInfoCount); +// modified_pgroups.resize(createInfoCount); +// for (uint32_t create_info_i = 0; create_info_i < createInfoCount; ++create_info_i) +// { +// format::HandleId pipeline_capture_id = (*pPipelines[create_info_i].GetPointer()); +// +// // Enable capture replay flag. +// modified_create_infos.push_back(in_pCreateInfos[create_info_i]); +// modified_create_infos[create_info_i].flags |= +// VK_PIPELINE_CREATE_RAY_TRACING_SHADER_GROUP_HANDLE_CAPTURE_REPLAY_BIT_KHR; +// +// uint32_t group_info_count = in_pCreateInfos[create_info_i].groupCount; +// bool has_data = (device_info->shader_group_handles.find(pipeline_capture_id) != +// device_info->shader_group_handles.end()); +// +// if (has_data) +// { +// assert(device_info->shader_group_handles.at(pipeline_capture_id).size() == +// (device_info->property_feature_info.property_shaderGroupHandleCaptureReplaySize * +// group_info_count)); +// } +// else +// { +// GFXRECON_LOG_WARNING("Missing shader group handle data in for ray tracing pipeline (ID = %" PRIu64 ").", +// pipeline_capture_id); +// } +// +// // Set pShaderGroupCaptureReplayHandle in shader group create infos. +// std::vector& modified_group_infos = modified_pgroups[create_info_i]; +// modified_group_infos.reserve(group_info_count); +// +// for (uint32_t group_info_i = 0; group_info_i < group_info_count; ++group_info_i) +// { +// modified_group_infos.push_back(in_pCreateInfos[create_info_i].pGroups[group_info_i]); +// +// if (has_data) +// { +// uint32_t byte_offset = +// device_info->property_feature_info.property_shaderGroupHandleCaptureReplaySize * group_info_i; +// modified_group_infos[group_info_i].pShaderGroupCaptureReplayHandle = +// device_info->shader_group_handles.at(pipeline_capture_id).data() + byte_offset; +// } +// else +// { +// modified_group_infos[group_info_i].pShaderGroupCaptureReplayHandle = nullptr; +// } +// } +// +// // Use modified shader group infos. +// modified_create_infos[create_info_i].pGroups = modified_group_infos.data(); +// } +// +// if (omitted_pipeline_cache_data_) +// { +// AllowCompileDuringPipelineCreation(createInfoCount, modified_create_infos.data()); +// } +// +// VkPipeline* created_pipelines = nullptr; +// +// if (deferred_operation_info) +// { +// created_pipelines = deferred_operation_info->replayPipelines.data(); +// } +// else +// { +// created_pipelines = out_pPipelines; +// } +// +// result = GetDeviceTable(device)->CreateRayTracingPipelinesKHR(device, +// in_deferredOperation, +// overridePipelineCache, +// createInfoCount, +// modified_create_infos.data(), +// in_pAllocator, +// created_pipelines); +// +// if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) || +// (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)) +// { +// // The above return values mean the command is not deferred and driver will finish all workload in current +// // thread. Therefore the created pipelines can be read and copied to out_pPipelines which will be +// // referenced later. +// // +// // Note: +// // Some pipelines might actually fail creation if the return value is VK_PIPELINE_COMPILE_REQUIRED_EXT. +// // These failed pipelines will generate VK_NULL_HANDLE. +// // +// // If the return value is VK_OPERATION_DEFERRED_KHR, it means the command is deferred, and thus pipeline +// // creation is not finished. Subsequent handling will be done by +// // vkDeferredOperationJoinKHR/vkGetDeferredOperationResultKHR after pipeline creation is finished. +// +// if (deferred_operation_info) +// { +// memcpy(out_pPipelines, created_pipelines, createInfoCount * sizeof(VkPipeline)); +// +// // Eventhough vkCreateRayTracingPipelinesKHR was called with a valid deferred operation object, the +// // driver may opt to not defer the command. In this case, set pending_state flag to false to skip +// // vkDeferredOperationJoinKHR handling. +// deferred_operation_info->pending_state = false; +// } +// } +// +// if (deferred_operation_info) +// { +// deferred_operation_info->record_modified_create_infos = std::move(modified_create_infos); +// deferred_operation_info->record_modified_pgroups = std::move(modified_pgroups); +// } +// } +// else { if (omitted_pipeline_cache_data_) { From 2c907c157d92aa4c87b60a6485c2edf7b6a142bb Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 17 Dec 2024 14:03:35 +0100 Subject: [PATCH 05/23] work on AccelerationStructure replacement. portable replay starts working here --- framework/decode/vulkan_address_replacer.cpp | 84 ++++++++++++++++--- framework/decode/vulkan_address_replacer.h | 9 ++ framework/decode/vulkan_rebind_allocator.cpp | 16 ++-- .../decode/vulkan_replay_consumer_base.cpp | 41 +++++---- 4 files changed, 111 insertions(+), 39 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 32b3a13238..1da0436813 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -416,7 +416,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( GFXRECON_ASSERT(_device_table != nullptr); // TODO: testing only -> remove when closing issue #1526 - constexpr bool force_replace = true; + bool force_replace = false; std::unordered_set buffer_set; auto address_remap = [&address_tracker, &buffer_set](VkDeviceAddress& capture_address) { @@ -459,11 +459,15 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( { primitive_counts[j] = range_info->primitiveCount; } - _device_table->GetAccelerationStructureBuildSizesKHR(_device, - VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, - &build_geometry_info, - primitive_counts.data(), - &build_size_info); + + { + mark_injected_commands_helper_t mark_injected_commands_helper; + _device_table->GetAccelerationStructureBuildSizesKHR(_device, + VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, + &build_geometry_info, + primitive_counts.data(), + &build_size_info); + } // retrieve VkAccelerationStructureKHR -> VkBuffer -> check/correct size auto* acceleration_structure_info = @@ -479,10 +483,16 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (buffer_info != nullptr && buffer_info->size < build_size_info.accelerationStructureSize) { - GFXRECON_LOG_WARNING("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: buffer-size is " - "too small (%d < %d)", - buffer_info->size, - build_size_info.accelerationStructureSize); + // GFXRECON_LOG_WARNING("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: + // buffer-size is " + // "too small (%d < %d)", + // buffer_info->size, + // build_size_info.accelerationStructureSize); + GFXRECON_LOG_INFO_ONCE( + "Replay adjusted mismatching acceleration-structures using shadow-structures and -buffers") + + // now definitely requiring address-replaced + force_replace = true; auto& replacment_as = _shadow_as_map[build_geometry_info.dstAccelerationStructure]; @@ -613,9 +623,18 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (force_replace || capture_address != replay_address) { - // TODO: extra look-up required for replaced AS auto new_address = replay_address; + // extra look-up required for potentially replaced AS + if (accel_info != nullptr) + { + auto shadow_as_it = _shadow_as_map.find(accel_info->handle); + if (shadow_as_it != _shadow_as_map.end()) + { + new_address = shadow_as_it->second.address; + } + } + // store addresses we will need to replace _hashmap_bda.put(capture_address, new_address); } @@ -701,6 +720,47 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } } +void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR(const VulkanCommandBufferInfo* command_buffer_info, + VkCopyAccelerationStructureInfoKHR* info, + const VulkanDeviceAddressTracker& address_tracker) +{ + GFXRECON_UNREFERENCED_PARAMETER(command_buffer_info); + GFXRECON_UNREFERENCED_PARAMETER(address_tracker); + + if (info != nullptr) + { + auto swap_acceleration_structure = [this](VkAccelerationStructureKHR& as) { + auto shadow_as_it = _shadow_as_map.find(as); + if (shadow_as_it != _shadow_as_map.end()) + { + as = shadow_as_it->second.handle; + } + }; + + // correct in-place + swap_acceleration_structure(info->src); + swap_acceleration_structure(info->dst); + } +} + +void VulkanAddressReplacer::ProcessCmdWriteAccelerationStructuresPropertiesKHR( + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t count, + VkAccelerationStructureKHR* acceleration_structures, + const VulkanDeviceAddressTracker& address_tracker) +{ + GFXRECON_UNREFERENCED_PARAMETER(command_buffer_info); + + for (uint32_t i = 0; i < count; ++i) + { + auto shadow_as_it = _shadow_as_map.find(acceleration_structures[i]); + if (shadow_as_it != _shadow_as_map.end()) + { + acceleration_structures[i] = shadow_as_it->second.handle; + } + } +} + bool VulkanAddressReplacer::init_pipeline() { if (_pipeline_sbt != VK_NULL_HANDLE) @@ -797,7 +857,7 @@ bool VulkanAddressReplacer::create_buffer(size_t { // 4kB min-size constexpr uint32_t min_buffer_size = 1 << 12; - num_bytes = aligned_size(std::max(num_bytes, min_buffer_size), min_alignment); + num_bytes = std::max(num_bytes + min_alignment, min_buffer_size); // nothing to do if (num_bytes <= buffer_context.num_bytes) diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index a15fa027de..192bde264c 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -103,6 +103,15 @@ class VulkanAddressReplacer VkAccelerationStructureBuildRangeInfoKHR** build_range_infos, const decode::VulkanDeviceAddressTracker& address_tracker); + void ProcessCmdCopyAccelerationStructuresKHR(const VulkanCommandBufferInfo* command_buffer_info, + VkCopyAccelerationStructureInfoKHR *info, + const decode::VulkanDeviceAddressTracker& address_tracker); + + void ProcessCmdWriteAccelerationStructuresPropertiesKHR(const VulkanCommandBufferInfo* command_buffer_info, + uint32_t count, + VkAccelerationStructureKHR* acceleration_structures, + const decode::VulkanDeviceAddressTracker& address_tracker); + /** * @brief DestroyShadowResources should be called upon destruction of provided VkAccelerationStructureKHR handle, * allowing this class to free potential resources associated with it. diff --git a/framework/decode/vulkan_rebind_allocator.cpp b/framework/decode/vulkan_rebind_allocator.cpp index 8fb843f2f0..8328ccdd28 100644 --- a/framework/decode/vulkan_rebind_allocator.cpp +++ b/framework/decode/vulkan_rebind_allocator.cpp @@ -247,19 +247,15 @@ VkResult VulkanRebindAllocator::CreateBuffer(const VkBufferCreateInfo* create VkResult result = VK_ERROR_INITIALIZATION_FAILED; - auto aligned_size = [](uint32_t size, uint32_t alignment) -> uint32_t - { - return (size + alignment - 1) & ~(alignment - 1); - }; - - // define a general minimum aligned-size for buffers, as certain vulkan resources will require this - // e.g. buffers containing shader-binding-tables or scratch-buffers for acceleration-structure build-operations - constexpr uint32_t min_alignment = 64; - if ((create_info != nullptr) && (buffer != nullptr) && (allocator_data != nullptr)) { + auto aligned_size = [](uint32_t size, uint32_t alignment) -> uint32_t + { + return (size + alignment - 1) & ~(alignment - 1); + }; + auto modified_info = *create_info; - modified_info.size = aligned_size(create_info->size, min_alignment); + modified_info.size = aligned_size(create_info->size, min_buffer_alignment_); result = functions_.create_buffer(device_, &modified_info, nullptr, buffer); if (result >= 0) diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index 719519a0c7..62d3eb22e2 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -7806,17 +7806,20 @@ void VulkanReplayConsumerBase::OverrideCmdCopyAccelerationStructureKHR( VulkanCommandBufferInfo* command_buffer_info, StructPointerDecoder* pInfo) { + GFXRECON_ASSERT(command_buffer_info != nullptr && pInfo != nullptr) VulkanDeviceInfo* device_info = object_info_table_->GetVkDeviceInfo(command_buffer_info->parent_id); - if (device_info->allocator->SupportsOpaqueDeviceAddresses()) - { - VkCommandBuffer command_buffer = command_buffer_info->handle; - VkCopyAccelerationStructureInfoKHR* info = pInfo->GetPointer(); - func(command_buffer, info); - } - else if (!loading_trim_state_) + GFXRECON_ASSERT(device_info != nullptr) + + VkCommandBuffer command_buffer = command_buffer_info->handle; + VkCopyAccelerationStructureInfoKHR* info = pInfo->GetPointer(); + { - // TODO: raytracing delegate-hook (issue #1526) + const auto& address_tracker = GetDeviceAddressTracker(device_info); + auto& address_replacer = GetDeviceAddressReplacer(device_info); + + address_replacer.ProcessCmdCopyAccelerationStructuresKHR(command_buffer_info, info, address_tracker); } + func(command_buffer, info); } void VulkanReplayConsumerBase::OverrideCmdWriteAccelerationStructuresPropertiesKHR( @@ -7828,18 +7831,22 @@ void VulkanReplayConsumerBase::OverrideCmdWriteAccelerationStructuresPropertiesK gfxrecon::decode::VulkanQueryPoolInfo* query_pool_info, uint32_t firstQuery) { + GFXRECON_ASSERT(command_buffer_info != nullptr) VulkanDeviceInfo* device_info = object_info_table_->GetVkDeviceInfo(command_buffer_info->parent_id); - if (device_info->allocator->SupportsOpaqueDeviceAddresses()) - { - VkCommandBuffer command_buffer = command_buffer_info->handle; - const VkAccelerationStructureKHR* acceleration_structs = pAccelerationStructures->GetHandlePointer(); - VkQueryPool query_pool = query_pool_info->handle; - func(command_buffer, count, acceleration_structs, queryType, query_pool, firstQuery); - } - else if (!loading_trim_state_) + GFXRECON_ASSERT(device_info != nullptr) + + VkCommandBuffer command_buffer = command_buffer_info->handle; + VkAccelerationStructureKHR* acceleration_structs = pAccelerationStructures->GetHandlePointer(); + VkQueryPool query_pool = query_pool_info->handle; + { - // TODO: raytracing delegate-hook (issue #1526) + const auto& address_tracker = GetDeviceAddressTracker(device_info); + auto& address_replacer = GetDeviceAddressReplacer(device_info); + + address_replacer.ProcessCmdWriteAccelerationStructuresPropertiesKHR( + command_buffer_info, count, acceleration_structs, address_tracker); } + func(command_buffer, count, acceleration_structs, queryType, query_pool, firstQuery); } VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR( From 805405a2694bd76f3ecbce8ceb4dcb48363913bc Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Wed, 18 Dec 2024 12:59:22 +0100 Subject: [PATCH 06/23] Add and integrate ProcessUpdateDescriptorSets --- framework/decode/vulkan_address_replacer.cpp | 99 +++++---- framework/decode/vulkan_address_replacer.h | 37 +++- framework/decode/vulkan_rebind_allocator.h | 2 +- .../decode/vulkan_replay_consumer_base.cpp | 206 ++++-------------- 4 files changed, 133 insertions(+), 211 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 1da0436813..d6a291541e 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -20,6 +20,7 @@ ** DEALINGS IN THE SOFTWARE. */ +#include "graphics/vulkan_struct_get_pnext.h" #include "decode/vulkan_address_replacer.h" #include "decode/vulkan_address_replacer_shaders.h" #include "decode/mark_injected_commands.h" @@ -293,7 +294,8 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( if (_valid_sbt_alignment) { - GFXRECON_LOG_INFO_ONCE("Replay adjusted mismatching raytracing shader-group-handles"); + GFXRECON_LOG_INFO_ONCE("VulkanAddressReplacer::ProcessCmdTraceRays: Replay adjusted mismatching raytracing " + "shader-group-handles"); // rewrite group-handles in-place replacer_params.output_handles = replacer_params.input_handles; @@ -311,8 +313,8 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } else { - GFXRECON_LOG_INFO_ONCE( - "Replay adjusted a mismatching raytracing shader-binding-table using a shadow-buffer"); + GFXRECON_LOG_INFO_ONCE("VulkanAddressReplacer::ProcessCmdTraceRays: Replay adjusted a mismatching " + "raytracing shader-binding-table using a shadow-buffer"); // output-handles if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), pipeline_context_sbt.output_handle_buffer)) @@ -476,20 +478,11 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto* buffer_info = address_tracker.GetBufferByHandle(acceleration_structure_info->buffer); GFXRECON_ASSERT(buffer_info != nullptr) - // TODO: fails on some!? captures - auto* scratch_buffer_info = - address_tracker.GetBufferByCaptureDeviceAddress(build_geometry_info.scratchData.deviceAddress); - // GFXRECON_ASSERT(scratch_buffer_info != nullptr) - if (buffer_info != nullptr && buffer_info->size < build_size_info.accelerationStructureSize) { - // GFXRECON_LOG_WARNING("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: - // buffer-size is " - // "too small (%d < %d)", - // buffer_info->size, - // build_size_info.accelerationStructureSize); GFXRECON_LOG_INFO_ONCE( - "Replay adjusted mismatching acceleration-structures using shadow-structures and -buffers") + "VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: Replay adjusted mismatching " + "acceleration-structures using shadow-structures and -buffers") // now definitely requiring address-replaced force_replace = true; @@ -510,7 +503,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (!success) { - GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: shadow-buffer creation failed"); + GFXRECON_LOG_ERROR("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: " + "shadow-buffer creation failed"); return; } @@ -545,25 +539,20 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( ? build_size_info.buildScratchSize : build_size_info.updateScratchSize; - // if (scratch_buffer_info != nullptr && scratch_buffer_info->size < scratch_size) - { - // create a replacement scratch-buffer - bool success = create_buffer( + // create a replacement scratch-buffer + if (!create_buffer( scratch_size, replacment_as.scratch, 0, _replay_acceleration_structure_properties.minAccelerationStructureScratchOffsetAlignment, - false); - - if (!success) - { - GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: scratch-buffer creation failed"); - return; - } - - // hot swap scratch-buffer - build_geometry_info.scratchData.deviceAddress = replacment_as.scratch.device_address; + false)) + { + GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: scratch-buffer creation failed"); + return; } + + // hot swap scratch-buffer + build_geometry_info.scratchData.deviceAddress = replacment_as.scratch.device_address; } } @@ -720,13 +709,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } } -void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR(const VulkanCommandBufferInfo* command_buffer_info, - VkCopyAccelerationStructureInfoKHR* info, - const VulkanDeviceAddressTracker& address_tracker) +void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR(VkCopyAccelerationStructureInfoKHR* info) { - GFXRECON_UNREFERENCED_PARAMETER(command_buffer_info); - GFXRECON_UNREFERENCED_PARAMETER(address_tracker); - if (info != nullptr) { auto swap_acceleration_structure = [this](VkAccelerationStructureKHR& as) { @@ -744,13 +728,8 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR(const Vulkan } void VulkanAddressReplacer::ProcessCmdWriteAccelerationStructuresPropertiesKHR( - const VulkanCommandBufferInfo* command_buffer_info, - uint32_t count, - VkAccelerationStructureKHR* acceleration_structures, - const VulkanDeviceAddressTracker& address_tracker) + uint32_t count, VkAccelerationStructureKHR* acceleration_structures) { - GFXRECON_UNREFERENCED_PARAMETER(command_buffer_info); - for (uint32_t i = 0; i < count; ++i) { auto shadow_as_it = _shadow_as_map.find(acceleration_structures[i]); @@ -761,6 +740,42 @@ void VulkanAddressReplacer::ProcessCmdWriteAccelerationStructuresPropertiesKHR( } } +void VulkanAddressReplacer::ProcessUpdateDescriptorSets(uint32_t descriptor_write_count, + VkWriteDescriptorSet* descriptor_writes, + uint32_t descriptor_copy_count, + VkCopyDescriptorSet* descriptor_copies) +{ + GFXRECON_UNREFERENCED_PARAMETER(descriptor_copy_count); + GFXRECON_UNREFERENCED_PARAMETER(descriptor_copies); + + for (uint32_t i = 0; i < descriptor_write_count; ++i) + { + VkWriteDescriptorSet& write = descriptor_writes[i]; + + if (write.descriptorType != VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR) + { + continue; + } + + if (auto write_as = graphics::vulkan_struct_get_pnext(&write)) + { + for (uint32_t j = 0; j < write_as->accelerationStructureCount; ++j) + { + auto acceleration_structure_it = _shadow_as_map.find(write_as->pAccelerationStructures[j]); + if (acceleration_structure_it != _shadow_as_map.end()) + { + // we found an existing replacement-structure -> swap + auto* out_array = const_cast(write_as->pAccelerationStructures); + out_array[j] = acceleration_structure_it->second.handle; + + GFXRECON_LOG_INFO_ONCE("VulkanAddressReplacer::ProcessUpdateDescriptorSets: Replay adjusted " + "AccelerationStructure handles") + } + } + } + } +} + bool VulkanAddressReplacer::init_pipeline() { if (_pipeline_sbt != VK_NULL_HANDLE) @@ -855,6 +870,8 @@ bool VulkanAddressReplacer::create_buffer(size_t uint32_t min_alignment, bool use_host_mem) { + GFXRECON_ASSERT(util::is_pow_2(min_alignment)); + // 4kB min-size constexpr uint32_t min_buffer_size = 1 << 12; num_bytes = std::max(num_bytes + min_alignment, min_buffer_size); diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 192bde264c..382d257b55 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -103,14 +103,37 @@ class VulkanAddressReplacer VkAccelerationStructureBuildRangeInfoKHR** build_range_infos, const decode::VulkanDeviceAddressTracker& address_tracker); - void ProcessCmdCopyAccelerationStructuresKHR(const VulkanCommandBufferInfo* command_buffer_info, - VkCopyAccelerationStructureInfoKHR *info, - const decode::VulkanDeviceAddressTracker& address_tracker); + /** + * @brief ProcessCmdCopyAccelerationStructuresKHR will check + * and potentially correct input-parameters to 'vkCmdCopyAccelerationStructuresKHR' + * + * @param info a provided VkCopyAccelerationStructureInfoKHR* + */ + void ProcessCmdCopyAccelerationStructuresKHR(VkCopyAccelerationStructureInfoKHR* info); - void ProcessCmdWriteAccelerationStructuresPropertiesKHR(const VulkanCommandBufferInfo* command_buffer_info, - uint32_t count, - VkAccelerationStructureKHR* acceleration_structures, - const decode::VulkanDeviceAddressTracker& address_tracker); + /** + * @brief ProcessCmdWriteAccelerationStructuresPropertiesKHR will check + * and potentially correct input-parameters to 'vkCmdWriteAccelerationStructuresPropertiesKHR' + * + * @param count element count in acceleration_structures + * @param acceleration_structures provided array of VkAccelerationStructureKHR-handles + */ + void ProcessCmdWriteAccelerationStructuresPropertiesKHR(uint32_t count, + VkAccelerationStructureKHR* acceleration_structures); + + /** + * @brief ProcessUpdateDescriptorSets will check + * and potentially correct input-parameters to 'vkUpdateDescriptorSets' + * + * @param descriptor_write_count element count in descriptor_writes + * @param descriptor_writes provided array of VkWriteDescriptorSet + * @param descriptor_copy_count element count in descriptor_copies + * @param descriptor_copies provided array of VkCopyDescriptorSet + */ + void ProcessUpdateDescriptorSets(uint32_t descriptor_write_count, + VkWriteDescriptorSet* descriptor_writes, + uint32_t descriptor_copy_count, + VkCopyDescriptorSet* descriptor_copies); /** * @brief DestroyShadowResources should be called upon destruction of provided VkAccelerationStructureKHR handle, diff --git a/framework/decode/vulkan_rebind_allocator.h b/framework/decode/vulkan_rebind_allocator.h index 143be38e5d..912e49600b 100644 --- a/framework/decode/vulkan_rebind_allocator.h +++ b/framework/decode/vulkan_rebind_allocator.h @@ -452,7 +452,7 @@ class VulkanRebindAllocator : public VulkanResourceAllocator uint32_t staging_queue_family_{}; //! define a general minimum alignment for buffers - uint32_t min_buffer_alignment_ = 128; + uint32_t min_buffer_alignment_ = 64; }; GFXRECON_END_NAMESPACE(decode) diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index 62d3eb22e2..73937001d9 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -7814,10 +7814,8 @@ void VulkanReplayConsumerBase::OverrideCmdCopyAccelerationStructureKHR( VkCopyAccelerationStructureInfoKHR* info = pInfo->GetPointer(); { - const auto& address_tracker = GetDeviceAddressTracker(device_info); - auto& address_replacer = GetDeviceAddressReplacer(device_info); - - address_replacer.ProcessCmdCopyAccelerationStructuresKHR(command_buffer_info, info, address_tracker); + auto& address_replacer = GetDeviceAddressReplacer(device_info); + address_replacer.ProcessCmdCopyAccelerationStructuresKHR(info); } func(command_buffer, info); } @@ -7840,11 +7838,8 @@ void VulkanReplayConsumerBase::OverrideCmdWriteAccelerationStructuresPropertiesK VkQueryPool query_pool = query_pool_info->handle; { - const auto& address_tracker = GetDeviceAddressTracker(device_info); - auto& address_replacer = GetDeviceAddressReplacer(device_info); - - address_replacer.ProcessCmdWriteAccelerationStructuresPropertiesKHR( - command_buffer_info, count, acceleration_structs, address_tracker); + auto& address_replacer = GetDeviceAddressReplacer(device_info); + address_replacer.ProcessCmdWriteAccelerationStructuresPropertiesKHR(count, acceleration_structs); } func(command_buffer, count, acceleration_structs, queryType, query_pool, firstQuery); } @@ -7876,7 +7871,6 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR( VkPipelineCache overridePipelineCache = in_pipelineCache; // If there is no pipeline cache and we want to create a new one - if (in_pipelineCache == VK_NULL_HANDLE && options_.add_new_pipeline_caches) { overridePipelineCache = CreateNewPipelineCache(device_info, *pPipelines->GetPointer()); @@ -7894,157 +7888,38 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR( &pPipelines->GetPointer()[createInfoCount]); } -// // NOTE: this is almost never true, even on newest desktop-drivers -// // TODO: consider removing all of the feature_rayTracingPipelineShaderGroupHandleCaptureReplay logic here -// if (device_info->property_feature_info.feature_rayTracingPipelineShaderGroupHandleCaptureReplay) -// { -// // Modify pipeline create infos with capture replay flag and data. -// std::vector modified_create_infos; -// std::vector> modified_pgroups; -// modified_create_infos.reserve(createInfoCount); -// modified_pgroups.resize(createInfoCount); -// for (uint32_t create_info_i = 0; create_info_i < createInfoCount; ++create_info_i) -// { -// format::HandleId pipeline_capture_id = (*pPipelines[create_info_i].GetPointer()); -// -// // Enable capture replay flag. -// modified_create_infos.push_back(in_pCreateInfos[create_info_i]); -// modified_create_infos[create_info_i].flags |= -// VK_PIPELINE_CREATE_RAY_TRACING_SHADER_GROUP_HANDLE_CAPTURE_REPLAY_BIT_KHR; -// -// uint32_t group_info_count = in_pCreateInfos[create_info_i].groupCount; -// bool has_data = (device_info->shader_group_handles.find(pipeline_capture_id) != -// device_info->shader_group_handles.end()); -// -// if (has_data) -// { -// assert(device_info->shader_group_handles.at(pipeline_capture_id).size() == -// (device_info->property_feature_info.property_shaderGroupHandleCaptureReplaySize * -// group_info_count)); -// } -// else -// { -// GFXRECON_LOG_WARNING("Missing shader group handle data in for ray tracing pipeline (ID = %" PRIu64 ").", -// pipeline_capture_id); -// } -// -// // Set pShaderGroupCaptureReplayHandle in shader group create infos. -// std::vector& modified_group_infos = modified_pgroups[create_info_i]; -// modified_group_infos.reserve(group_info_count); -// -// for (uint32_t group_info_i = 0; group_info_i < group_info_count; ++group_info_i) -// { -// modified_group_infos.push_back(in_pCreateInfos[create_info_i].pGroups[group_info_i]); -// -// if (has_data) -// { -// uint32_t byte_offset = -// device_info->property_feature_info.property_shaderGroupHandleCaptureReplaySize * group_info_i; -// modified_group_infos[group_info_i].pShaderGroupCaptureReplayHandle = -// device_info->shader_group_handles.at(pipeline_capture_id).data() + byte_offset; -// } -// else -// { -// modified_group_infos[group_info_i].pShaderGroupCaptureReplayHandle = nullptr; -// } -// } -// -// // Use modified shader group infos. -// modified_create_infos[create_info_i].pGroups = modified_group_infos.data(); -// } -// -// if (omitted_pipeline_cache_data_) -// { -// AllowCompileDuringPipelineCreation(createInfoCount, modified_create_infos.data()); -// } -// -// VkPipeline* created_pipelines = nullptr; -// -// if (deferred_operation_info) -// { -// created_pipelines = deferred_operation_info->replayPipelines.data(); -// } -// else -// { -// created_pipelines = out_pPipelines; -// } -// -// result = GetDeviceTable(device)->CreateRayTracingPipelinesKHR(device, -// in_deferredOperation, -// overridePipelineCache, -// createInfoCount, -// modified_create_infos.data(), -// in_pAllocator, -// created_pipelines); -// -// if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) || -// (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)) -// { -// // The above return values mean the command is not deferred and driver will finish all workload in current -// // thread. Therefore the created pipelines can be read and copied to out_pPipelines which will be -// // referenced later. -// // -// // Note: -// // Some pipelines might actually fail creation if the return value is VK_PIPELINE_COMPILE_REQUIRED_EXT. -// // These failed pipelines will generate VK_NULL_HANDLE. -// // -// // If the return value is VK_OPERATION_DEFERRED_KHR, it means the command is deferred, and thus pipeline -// // creation is not finished. Subsequent handling will be done by -// // vkDeferredOperationJoinKHR/vkGetDeferredOperationResultKHR after pipeline creation is finished. -// -// if (deferred_operation_info) -// { -// memcpy(out_pPipelines, created_pipelines, createInfoCount * sizeof(VkPipeline)); -// -// // Eventhough vkCreateRayTracingPipelinesKHR was called with a valid deferred operation object, the -// // driver may opt to not defer the command. In this case, set pending_state flag to false to skip -// // vkDeferredOperationJoinKHR handling. -// deferred_operation_info->pending_state = false; -// } -// } -// -// if (deferred_operation_info) -// { -// deferred_operation_info->record_modified_create_infos = std::move(modified_create_infos); -// deferred_operation_info->record_modified_pgroups = std::move(modified_pgroups); -// } -// } -// else - { - if (omitted_pipeline_cache_data_) - { - AllowCompileDuringPipelineCreation(createInfoCount, - const_cast(in_pCreateInfos)); - } - - VkPipeline* created_pipelines = nullptr; + if (omitted_pipeline_cache_data_) + { + AllowCompileDuringPipelineCreation(createInfoCount, + const_cast(in_pCreateInfos)); + } - if (deferred_operation_info) - { - created_pipelines = deferred_operation_info->replayPipelines.data(); - } - else - { - created_pipelines = out_pPipelines; - } + VkPipeline* created_pipelines = nullptr; - result = GetDeviceTable(device)->CreateRayTracingPipelinesKHR(device, - in_deferredOperation, - overridePipelineCache, - createInfoCount, - in_pCreateInfos, - in_pAllocator, - created_pipelines); + if (deferred_operation_info) + { + created_pipelines = deferred_operation_info->replayPipelines.data(); + } + else + { + created_pipelines = out_pPipelines; + } - if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) || - (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)) - { + result = GetDeviceTable(device)->CreateRayTracingPipelinesKHR(device, + in_deferredOperation, + overridePipelineCache, + createInfoCount, + in_pCreateInfos, + in_pAllocator, + created_pipelines); - if (deferred_operation_info) - { - memcpy(out_pPipelines, created_pipelines, createInfoCount * sizeof(VkPipeline)); - deferred_operation_info->pending_state = false; - } + if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) || + (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)) + { + if (deferred_operation_info) + { + memcpy(out_pPipelines, created_pipelines, createInfoCount * sizeof(VkPipeline)); + deferred_operation_info->pending_state = false; } } @@ -8054,7 +7929,7 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR( for (uint32_t i = 0; i < createInfoCount; ++i) { - VulkanPipelineInfo* pipeline_info = reinterpret_cast(pPipelines->GetConsumerData(i)); + auto* pipeline_info = reinterpret_cast(pPipelines->GetConsumerData(i)); const Decoded_VkPipelineShaderStageCreateInfo* stages_info_meta = create_info_meta[i].pStages->GetMetaStructPointer(); @@ -8073,7 +7948,6 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR( } // If a pipeline cache was created, track it to know when to destroy it/save it to file - if (in_pipelineCache != overridePipelineCache && result == VK_SUCCESS) { TrackNewPipelineCache(device_info, @@ -8082,7 +7956,6 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR( pPipelines->GetHandlePointer(), createInfoCount); } - return result; } @@ -9728,8 +9601,17 @@ void VulkanReplayConsumerBase::OverrideUpdateDescriptorSets( uint32_t descriptor_copy_count, StructPointerDecoder* p_pescriptor_copies) { - const VkWriteDescriptorSet* in_pDescriptorWrites = p_descriptor_writes->GetPointer(); - const VkCopyDescriptorSet* in_pDescriptorCopies = p_pescriptor_copies->GetPointer(); + GFXRECON_ASSERT(device_info != nullptr); + + VkWriteDescriptorSet* in_pDescriptorWrites = p_descriptor_writes->GetPointer(); + VkCopyDescriptorSet* in_pDescriptorCopies = p_pescriptor_copies->GetPointer(); + + { + // check/correct specific resource handles (i.e. VkAccelerationStructure) + auto& address_replacer = GetDeviceAddressReplacer(device_info); + address_replacer.ProcessUpdateDescriptorSets( + descriptor_write_count, in_pDescriptorWrites, descriptor_copy_count, in_pDescriptorCopies); + } func( device_info->handle, descriptor_write_count, in_pDescriptorWrites, descriptor_copy_count, in_pDescriptorCopies); From 1e74976c9cc930f3ae973a1ecb070087869fe51a Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Wed, 18 Dec 2024 14:58:46 +0100 Subject: [PATCH 07/23] Process meta-commands (rebuild AS when trimming) --- framework/decode/vulkan_address_replacer.cpp | 49 ++++++++++--- framework/decode/vulkan_address_replacer.h | 42 ++++++++++- .../decode/vulkan_replay_consumer_base.cpp | 73 ++++++++----------- 3 files changed, 105 insertions(+), 59 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index d6a291541e..e7791d1052 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -417,7 +417,6 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( { GFXRECON_ASSERT(_device_table != nullptr); - // TODO: testing only -> remove when closing issue #1526 bool force_replace = false; std::unordered_set buffer_set; @@ -649,6 +648,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (!create_buffer(_hashmap_bda.get_storage(nullptr), pipeline_context_bda.hashmap_storage)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); + return; } _hashmap_bda.get_storage(pipeline_context_bda.hashmap_storage.mapped_data); @@ -657,6 +657,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (!create_buffer(num_bytes, pipeline_context_bda.input_handle_buffer)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); + return; } memcpy(pipeline_context_bda.input_handle_buffer.mapped_data, addresses_to_replace.data(), num_bytes); @@ -757,7 +758,7 @@ void VulkanAddressReplacer::ProcessUpdateDescriptorSets(uint32_t de continue; } - if (auto write_as = graphics::vulkan_struct_get_pnext(&write)) + if (auto* write_as = graphics::vulkan_struct_get_pnext(&write)) { for (uint32_t j = 0; j < write_as->accelerationStructureCount; ++j) { @@ -776,6 +777,27 @@ void VulkanAddressReplacer::ProcessUpdateDescriptorSets(uint32_t de } } +void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( + uint32_t info_count, + VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, + VkAccelerationStructureBuildRangeInfoKHR** range_infos, + std::vector>& instance_buffers_data) +{ + // TODO: command-pool/buffer +} + +void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( + uint32_t info_count, VkCopyAccelerationStructureInfoKHR* copy_infos) +{ + // TODO: command-pool/buffer +} + +void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand( + VkQueryType query_type, VkAccelerationStructureKHR acceleration_structure) +{ + // TODO: command-pool/buffer +} + bool VulkanAddressReplacer::init_pipeline() { if (_pipeline_sbt != VK_NULL_HANDLE) @@ -993,20 +1015,23 @@ void VulkanAddressReplacer::barrier(VkCommandBuffer command_buffer, void VulkanAddressReplacer::DestroyShadowResources(VkAccelerationStructureKHR handle) { - auto remove_as_it = _shadow_as_map.find(handle); - - if (remove_as_it != _shadow_as_map.end()) + if (handle != VK_NULL_HANDLE) { - mark_injected_commands_helper_t mark_injected_commands_helper; - _shadow_as_map.erase(remove_as_it); + auto remove_as_it = _shadow_as_map.find(handle); + + if (remove_as_it != _shadow_as_map.end()) + { + mark_injected_commands_helper_t mark_injected_commands_helper; + _shadow_as_map.erase(remove_as_it); + } } } -void VulkanAddressReplacer::DestroyShadowResources(const VulkanCommandBufferInfo* command_buffer_info) +void VulkanAddressReplacer::DestroyShadowResources(VkCommandBuffer handle) { - if (command_buffer_info != nullptr) + if (handle != VK_NULL_HANDLE) { - auto remove_context_it = _build_as_context_map.find(command_buffer_info->handle); + auto remove_context_it = _build_as_context_map.find(handle); if (remove_context_it != _build_as_context_map.end()) { @@ -1014,7 +1039,7 @@ void VulkanAddressReplacer::DestroyShadowResources(const VulkanCommandBufferInfo _build_as_context_map.erase(remove_context_it); } - auto shadow_sbt_it = _shadow_sbt_map.find(command_buffer_info->handle); + auto shadow_sbt_it = _shadow_sbt_map.find(handle); if (shadow_sbt_it != _shadow_sbt_map.end()) { @@ -1022,7 +1047,7 @@ void VulkanAddressReplacer::DestroyShadowResources(const VulkanCommandBufferInfo _shadow_sbt_map.erase(shadow_sbt_it); } - auto pipeline_sbt_it = _pipeline_sbt_context_map.find(command_buffer_info->handle); + auto pipeline_sbt_it = _pipeline_sbt_context_map.find(handle); if (pipeline_sbt_it != _pipeline_sbt_context_map.end()) { diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 382d257b55..9a758a258d 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -135,6 +135,42 @@ class VulkanAddressReplacer uint32_t descriptor_copy_count, VkCopyDescriptorSet* descriptor_copies); + /** + * @brief Process information contained in a metadata-block in order to build acceleration-structures. + * + * Will use an internal command-pool, submit work to a VkQueue and perform a host-synchronization. + * + * @param info_count element count in 'geometry_infos' + * @param geometry_infos provided array of VkAccelerationStructureBuildGeometryInfoKHR + * @param range_infos provided array of pointers to VkAccelerationStructureBuildRangeInfoKHR + * @param instance_buffers_data an array of arrays of VkAccelerationStructureInstanceKHR + */ + void ProcessBuildVulkanAccelerationStructuresMetaCommand( + uint32_t info_count, + VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, + VkAccelerationStructureBuildRangeInfoKHR** range_infos, + std::vector>& instance_buffers_data); + + /** + * @brief Process information contained in a metadata-block in order to copy acceleration-structures. + * + * Will use an internal command-pool, submit work to a VkQueue and perform a host-synchronization. + * + * @param info_count element count in 'copy_infos' + * @param copy_infos provided array of VkCopyAccelerationStructureInfoKHR + */ + void ProcessCopyVulkanAccelerationStructuresMetaCommand(uint32_t info_count, + VkCopyAccelerationStructureInfoKHR* copy_infos); + /** + * @brief Process information contained in a metadata-block in order to write information in a query-pool. + * + * @param query_type element count in 'copy_infos' + * @param acceleration_structure provided acceleration-structure handle + */ + void + ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand(VkQueryType query_type, + VkAccelerationStructureKHR acceleration_structure); + /** * @brief DestroyShadowResources should be called upon destruction of provided VkAccelerationStructureKHR handle, * allowing this class to free potential resources associated with it. @@ -144,12 +180,12 @@ class VulkanAddressReplacer void DestroyShadowResources(VkAccelerationStructureKHR handle); /** - * @brief DestroyShadowResources should be called upon destruction of provided command_buffer_info, + * @brief DestroyShadowResources should be called upon destruction of provided VkCommandBuffer handle, * allowing this class to free potential resources associated with it. * - * @param handle a provided VulkanCommandBufferInfo + * @param handle a provided VkCommandBuffer handle */ - void DestroyShadowResources(const VulkanCommandBufferInfo* command_buffer_info); + void DestroyShadowResources(VkCommandBuffer handle); private: struct buffer_context_t diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index 73937001d9..b2787ed344 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -4302,12 +4302,11 @@ void VulkanReplayConsumerBase::OverrideFreeCommandBuffers(PFN_vkFreeCommandBuffe uint32_t command_buffer_count, HandlePointerDecoder* pCommandBuffers) { - assert((device_info != nullptr) && (pCommandBuffers != nullptr) && - (pCommandBuffers->GetHandlePointer() != nullptr)); + GFXRECON_ASSERT((device_info != nullptr) && (pCommandBuffers != nullptr) && + (pCommandBuffers->GetHandlePointer() != nullptr)); if (command_pool_info != nullptr) { - const format::HandleId* cmd_buf_handles = pCommandBuffers->GetPointer(); for (uint32_t i = 0; i < command_buffer_count; ++i) { @@ -4326,7 +4325,7 @@ void VulkanReplayConsumerBase::OverrideFreeCommandBuffers(PFN_vkFreeCommandBuffe VulkanCommandBufferInfo* cb_info = object_info_table_->GetVkCommandBufferInfo(cmd_buf_handles[i]); if (cb_info != nullptr) { - GetDeviceAddressReplacer(device_info).DestroyShadowResources(cb_info); + GetDeviceAddressReplacer(device_info).DestroyShadowResources(cb_info->handle); } } } @@ -9528,22 +9527,16 @@ void VulkanReplayConsumerBase::Process_vkCreateRayTracingPipelinesKHR( void VulkanReplayConsumerBase::ProcessCopyVulkanAccelerationStructuresMetaCommand( format::HandleId device, StructPointerDecoder* copy_infos) { - VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device); - GFXRECON_ASSERT(device_info != nullptr); - - auto allocator = device_info->allocator.get(); - GFXRECON_ASSERT(allocator != nullptr); - - if (allocator->SupportsOpaqueDeviceAddresses() || !loading_trim_state_) + if (loading_trim_state_) { - return; - } + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device); + GFXRECON_ASSERT(device_info != nullptr); - MapStructArrayHandles(copy_infos->GetMetaStructPointer(), copy_infos->GetLength(), GetObjectInfoTable()); + MapStructArrayHandles(copy_infos->GetMetaStructPointer(), copy_infos->GetLength(), GetObjectInfoTable()); - // TODO: implement - // acceleration_structure_builders_[device]->ProcessCopyVulkanAccelerationStructuresMetaCommand( - // copy_infos->GetLength(), copy_infos->GetPointer()); + auto& ar = GetDeviceAddressReplacer(device_info); + ar.ProcessCopyVulkanAccelerationStructuresMetaCommand(copy_infos->GetLength(), copy_infos->GetPointer()); + } } void VulkanReplayConsumerBase::ProcessBuildVulkanAccelerationStructuresMetaCommand( @@ -9553,44 +9546,36 @@ void VulkanReplayConsumerBase::ProcessBuildVulkanAccelerationStructuresMetaComma StructPointerDecoder* ppRangeInfos, std::vector>& instance_buffers_data) { - VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device); - GFXRECON_ASSERT(device_info != nullptr); - - auto allocator = device_info->allocator.get(); - GFXRECON_ASSERT(allocator != nullptr); - - if (allocator->SupportsOpaqueDeviceAddresses() || !loading_trim_state_) + if (loading_trim_state_) { - return; - } + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device); + GFXRECON_ASSERT(device_info != nullptr); + + MapStructArrayHandles(pInfos->GetMetaStructPointer(), pInfos->GetLength(), GetObjectInfoTable()); - MapStructArrayHandles(pInfos->GetMetaStructPointer(), pInfos->GetLength(), GetObjectInfoTable()); + VkAccelerationStructureBuildGeometryInfoKHR* build_geometry_infos = pInfos->GetPointer(); + VkAccelerationStructureBuildRangeInfoKHR** range_infos = ppRangeInfos->GetPointer(); - // TODO: implement - // acceleration_structure_builders_[device]->ProcessBuildVulkanAccelerationStructuresMetaCommand( - // info_count, pInfos->GetPointer(), ppRangeInfos->GetPointer(), instance_buffers_data); + GetDeviceAddressReplacer(device_info) + .ProcessBuildVulkanAccelerationStructuresMetaCommand( + info_count, pInfos->GetPointer(), ppRangeInfos->GetPointer(), instance_buffers_data); + } } void VulkanReplayConsumerBase::ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand( format::HandleId device_id, VkQueryType query_type, format::HandleId acceleration_structure_id) { - VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device_id); - GFXRECON_ASSERT(device_info != nullptr); - - auto allocator = device_info->allocator.get(); - GFXRECON_ASSERT(allocator != nullptr); - - if (allocator->SupportsOpaqueDeviceAddresses() || !loading_trim_state_) + if (loading_trim_state_) { - return; - } + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device_id); + GFXRECON_ASSERT(device_info != nullptr); - VkAccelerationStructureKHR acceleration_structure = MapHandle( - acceleration_structure_id, &VulkanObjectInfoTable::GetVkAccelerationStructureKHRInfo); + VkAccelerationStructureKHR acceleration_structure = MapHandle( + acceleration_structure_id, &VulkanObjectInfoTable::GetVkAccelerationStructureKHRInfo); - // TODO: implement - // acceleration_structure_builders_[device_id]->ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand( - // query_type, acceleration_structure); + GetDeviceAddressReplacer(device_info) + .ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand(query_type, acceleration_structure); + } } void VulkanReplayConsumerBase::OverrideUpdateDescriptorSets( From c48e0009e52e3c533cd4588e6a07950c1efdd047 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Mon, 6 Jan 2025 14:09:23 +0100 Subject: [PATCH 08/23] Implement ProcessBuildVulkanAccelerationStructuresMetaCommand. - Add internal command/queue-assets, manage lifetime --- framework/decode/vulkan_address_replacer.cpp | 140 +++++++++++++++++- framework/decode/vulkan_address_replacer.h | 12 +- .../decode/vulkan_replay_consumer_base.cpp | 7 +- 3 files changed, 152 insertions(+), 7 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index e7791d1052..e4df88468c 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -45,6 +45,58 @@ struct mark_injected_commands_helper_t } }; +//! RAII helper submit a command-buffer to a queue and synchronize via fence +struct queue_submit_helper_t +{ + const encode::VulkanDeviceTable* device_table = nullptr; + VkDevice device = VK_NULL_HANDLE; + VkCommandBuffer command_buffer = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + VkQueue queue = VK_NULL_HANDLE; + + queue_submit_helper_t(const encode::VulkanDeviceTable* device_table_, + VkDevice device_, + VkCommandBuffer command_buffer_, + VkQueue queue_, + VkFence fence_) : + device(device_), device_table(device_table_), command_buffer(command_buffer_), fence(fence_), queue(queue_) + { + mark_injected_commands_helper_t mark_injected_commands_helper; + + device_table->ResetFences(device, 1, &fence); + + VkCommandBufferBeginInfo command_buffer_begin_info; + command_buffer_begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + command_buffer_begin_info.pNext = nullptr; + command_buffer_begin_info.flags = 0; + command_buffer_begin_info.pInheritanceInfo = nullptr; + device_table->BeginCommandBuffer(command_buffer, &command_buffer_begin_info); + } + + ~queue_submit_helper_t() + { + mark_injected_commands_helper_t mark_injected_commands_helper; + + device_table->EndCommandBuffer(command_buffer); + + VkSubmitInfo submit_info = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; + submit_info.pNext = nullptr; + submit_info.waitSemaphoreCount = 0; + submit_info.pWaitSemaphores = nullptr; + submit_info.pWaitDstStageMask = nullptr; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &command_buffer; + submit_info.signalSemaphoreCount = 0; + submit_info.pSignalSemaphores = nullptr; + + // submit + device_table->QueueSubmit(queue, 1, &submit_info, fence); + + // sync + device_table->WaitForFences(device, 1, &fence, VK_TRUE, std::numeric_limits::max()); + } +}; + inline VkDeviceAddress aligned_address(VkDeviceAddress address, uint64_t alignment) { return alignment ? (address + alignment - 1) & ~(alignment - 1) : address; @@ -169,6 +221,20 @@ VulkanAddressReplacer::~VulkanAddressReplacer() { _device_table->DestroyPipelineLayout(_device, _pipeline_layout, nullptr); } + + if (_fence != VK_NULL_HANDLE) + { + _device_table->DestroyFence(_device, _fence, nullptr); + } + if (_command_buffer != VK_NULL_HANDLE) + { + GFXRECON_ASSERT(_command_pool != VK_NULL_HANDLE) + _device_table->FreeCommandBuffers(_device, _command_pool, 1, &_command_buffer); + } + if (_command_pool != VK_NULL_HANDLE) + { + _device_table->DestroyCommandPool(_device, _command_pool, nullptr); + } } void VulkanAddressReplacer::ProcessCmdTraceRays( @@ -781,21 +847,38 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( uint32_t info_count, VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, VkAccelerationStructureBuildRangeInfoKHR** range_infos, - std::vector>& instance_buffers_data) + std::vector>& instance_buffers_data, + const decode::VulkanDeviceAddressTracker& address_tracker) { - // TODO: command-pool/buffer + if (info_count > 0 && init_queue_assets()) + { + // reset/submit/sync command-buffer + queue_submit_helper_t queue_submit_helper(_device_table, _device, _command_buffer, _queue, _fence); + + // dummy-wrapper + VulkanCommandBufferInfo command_buffer_info = {}; + command_buffer_info.handle = _command_buffer; + ProcessCmdBuildAccelerationStructuresKHR( + &command_buffer_info, info_count, geometry_infos, range_infos, address_tracker); + } } void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( uint32_t info_count, VkCopyAccelerationStructureInfoKHR* copy_infos) { - // TODO: command-pool/buffer + if (init_queue_assets()) + { + // TODO: reset/submit/sync command-buffer + } } void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand( VkQueryType query_type, VkAccelerationStructureKHR acceleration_structure) { - // TODO: command-pool/buffer + if (init_queue_assets()) + { + // TODO: reset/submit/sync command-buffer + } } bool VulkanAddressReplacer::init_pipeline() @@ -886,6 +969,55 @@ bool VulkanAddressReplacer::init_pipeline() return true; } +bool VulkanAddressReplacer::init_queue_assets() +{ + if (_queue != VK_NULL_HANDLE) + { + return true; + }; + + VkCommandPoolCreateInfo create_info = {}; + create_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + create_info.queueFamilyIndex = 0; + + VkResult result = _device_table->CreateCommandPool(_device, &create_info, nullptr, &_command_pool); + if (result != VK_SUCCESS) + { + GFXRECON_LOG_ERROR("VulkanAddressReplacer: internal command-pool creation failed"); + return false; + } + + VkCommandBufferAllocateInfo alloc_info = {}; + alloc_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + alloc_info.pNext = nullptr; + alloc_info.commandPool = _command_pool; + alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + alloc_info.commandBufferCount = 1; + result = _device_table->AllocateCommandBuffers(_device, &alloc_info, &_command_buffer); + if (result != VK_SUCCESS) + { + GFXRECON_LOG_ERROR("VulkanAddressReplacer: internal command-buffer creation failed"); + return false; + } + + VkFenceCreateInfo fence_create_info; + fence_create_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + fence_create_info.pNext = nullptr; + fence_create_info.flags = VK_FENCE_CREATE_SIGNALED_BIT; + result = _device_table->CreateFence(_device, &fence_create_info, nullptr, &_fence); + if (result != VK_SUCCESS) + { + GFXRECON_LOG_ERROR("VulkanAddressReplacer: internal fence creation failed"); + return false; + } + + _device_table->GetDeviceQueue(_device, 0, 0, &_queue); + GFXRECON_ASSERT(_queue != VK_NULL_HANDLE); + return _queue != VK_NULL_HANDLE; +} + bool VulkanAddressReplacer::create_buffer(size_t num_bytes, VulkanAddressReplacer::buffer_context_t& buffer_context, uint32_t usage_flags, diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 9a758a258d..d561cba368 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -144,12 +144,14 @@ class VulkanAddressReplacer * @param geometry_infos provided array of VkAccelerationStructureBuildGeometryInfoKHR * @param range_infos provided array of pointers to VkAccelerationStructureBuildRangeInfoKHR * @param instance_buffers_data an array of arrays of VkAccelerationStructureInstanceKHR + * @param address_tracker const reference to a VulkanDeviceAddressTracker */ void ProcessBuildVulkanAccelerationStructuresMetaCommand( uint32_t info_count, VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, VkAccelerationStructureBuildRangeInfoKHR** range_infos, - std::vector>& instance_buffers_data); + std::vector>& instance_buffers_data, + const decode::VulkanDeviceAddressTracker& address_tracker); /** * @brief Process information contained in a metadata-block in order to copy acceleration-structures. @@ -222,6 +224,8 @@ class VulkanAddressReplacer [[nodiscard]] bool init_pipeline(); + [[nodiscard]] bool init_queue_assets(); + [[nodiscard]] bool create_buffer(size_t num_bytes, buffer_context_t& buffer_context, uint32_t usage_flags = 0, @@ -253,6 +257,12 @@ class VulkanAddressReplacer // pipeline dealing with buffer-device-addresses (BDA), replacing addresses VkPipeline _pipeline_bda = VK_NULL_HANDLE; + // required assets for submitting meta-commands + VkCommandPool _command_pool = VK_NULL_HANDLE; + VkCommandBuffer _command_buffer = VK_NULL_HANDLE; + VkFence _fence = VK_NULL_HANDLE; + VkQueue _queue = VK_NULL_HANDLE; + util::linear_hashmap _hashmap_sbt; util::linear_hashmap _hashmap_bda; std::unordered_map _shadow_sbt_map; diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index b2787ed344..e7fbe67bd9 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -9557,8 +9557,11 @@ void VulkanReplayConsumerBase::ProcessBuildVulkanAccelerationStructuresMetaComma VkAccelerationStructureBuildRangeInfoKHR** range_infos = ppRangeInfos->GetPointer(); GetDeviceAddressReplacer(device_info) - .ProcessBuildVulkanAccelerationStructuresMetaCommand( - info_count, pInfos->GetPointer(), ppRangeInfos->GetPointer(), instance_buffers_data); + .ProcessBuildVulkanAccelerationStructuresMetaCommand(info_count, + pInfos->GetPointer(), + ppRangeInfos->GetPointer(), + instance_buffers_data, + GetDeviceAddressTracker(device_info)); } } From 4490d47b10e3fab387bf2ae67843885b3469331e Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Mon, 6 Jan 2025 17:40:11 +0100 Subject: [PATCH 09/23] Add missing tracking of AS scratch-buffers, needed for trimming --- framework/decode/vulkan_address_replacer.cpp | 17 +++++-- framework/decode/vulkan_address_replacer.h | 2 - framework/decode/vulkan_rebind_allocator.h | 2 +- framework/encode/vulkan_handle_wrappers.h | 2 +- framework/encode/vulkan_state_tracker.cpp | 52 +++++++++++--------- framework/encode/vulkan_state_writer.cpp | 12 ++--- 6 files changed, 48 insertions(+), 39 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index e4df88468c..233ef2a6a9 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -486,7 +486,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( bool force_replace = false; std::unordered_set buffer_set; - auto address_remap = [&address_tracker, &buffer_set](VkDeviceAddress& capture_address) { + auto address_remap = [&address_tracker, &buffer_set](VkDeviceAddress& capture_address) -> bool { auto buffer_info = address_tracker.GetBufferByCaptureDeviceAddress(capture_address); if (buffer_info != nullptr && buffer_info->replay_address != 0) @@ -498,11 +498,13 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // in-place address-remap via const-cast capture_address = buffer_info->replay_address + offset; + return true; } else { GFXRECON_LOG_WARNING( "ProcessCmdBuildAccelerationStructuresKHR: missing buffer_info->replay_address, remap failed"); + return false; } }; @@ -514,7 +516,11 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto range_info = build_range_infos[i]; // check/correct scratch-address - address_remap(build_geometry_info.scratchData.deviceAddress); + if(!address_remap(build_geometry_info.scratchData.deviceAddress)) + { + GFXRECON_LOG_WARNING( + "ProcessCmdBuildAccelerationStructuresKHR: missing scratch-buffer"); + } // check capture/replay acceleration-structure buffer-sizes { @@ -850,6 +856,9 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( std::vector>& instance_buffers_data, const decode::VulkanDeviceAddressTracker& address_tracker) { + // TODO: figure out what to do + GFXRECON_UNREFERENCED_PARAMETER(instance_buffers_data); + if (info_count > 0 && init_queue_assets()) { // reset/submit/sync command-buffer @@ -866,9 +875,9 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( uint32_t info_count, VkCopyAccelerationStructureInfoKHR* copy_infos) { - if (init_queue_assets()) + for (uint32_t i = 0; i < info_count; ++i) { - // TODO: reset/submit/sync command-buffer + ProcessCmdCopyAccelerationStructuresKHR(copy_infos + i); } } diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index d561cba368..c15cb16213 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -156,8 +156,6 @@ class VulkanAddressReplacer /** * @brief Process information contained in a metadata-block in order to copy acceleration-structures. * - * Will use an internal command-pool, submit work to a VkQueue and perform a host-synchronization. - * * @param info_count element count in 'copy_infos' * @param copy_infos provided array of VkCopyAccelerationStructureInfoKHR */ diff --git a/framework/decode/vulkan_rebind_allocator.h b/framework/decode/vulkan_rebind_allocator.h index 912e49600b..143be38e5d 100644 --- a/framework/decode/vulkan_rebind_allocator.h +++ b/framework/decode/vulkan_rebind_allocator.h @@ -452,7 +452,7 @@ class VulkanRebindAllocator : public VulkanResourceAllocator uint32_t staging_queue_family_{}; //! define a general minimum alignment for buffers - uint32_t min_buffer_alignment_ = 64; + uint32_t min_buffer_alignment_ = 128; }; GFXRECON_END_NAMESPACE(decode) diff --git a/framework/encode/vulkan_handle_wrappers.h b/framework/encode/vulkan_handle_wrappers.h index 6c6143aa52..940f394341 100644 --- a/framework/encode/vulkan_handle_wrappers.h +++ b/framework/encode/vulkan_handle_wrappers.h @@ -586,7 +586,7 @@ struct AccelerationStructureKHRWrapper : public HandleWrapper geometry_info_memory; std::vector build_range_infos; - std::vector input_buffers; + std::unordered_map input_buffers; }; std::optional latest_update_command_{ std::nullopt }; std::optional latest_build_command_{ std::nullopt }; diff --git a/framework/encode/vulkan_state_tracker.cpp b/framework/encode/vulkan_state_tracker.cpp index 3d9a2d846a..3354e7326e 100644 --- a/framework/encode/vulkan_state_tracker.cpp +++ b/framework/encode/vulkan_state_tracker.cpp @@ -433,6 +433,8 @@ void VulkanStateTracker::TrackAccelerationStructureBuildCommand( continue; } + std::vector to_extract = { build_info.scratchData.deviceAddress }; + auto wrapper = vulkan_wrappers::GetWrapper( build_info.dstAccelerationStructure); @@ -442,24 +444,26 @@ void VulkanStateTracker::TrackAccelerationStructureBuildCommand( { auto geometry = build_info.pGeometries != nullptr ? build_info.pGeometries + g : build_info.ppGeometries[g]; - std::vector to_extract; switch (geometry->geometryType) { case VkGeometryTypeKHR::VK_GEOMETRY_TYPE_TRIANGLES_KHR: { - to_extract = { geometry->geometry.triangles.vertexData.deviceAddress, - geometry->geometry.triangles.indexData.deviceAddress, - geometry->geometry.triangles.transformData.deviceAddress }; + for (const auto& address : { geometry->geometry.triangles.vertexData.deviceAddress, + geometry->geometry.triangles.indexData.deviceAddress, + geometry->geometry.triangles.transformData.deviceAddress }) + { + to_extract.push_back(address); + } break; } case VkGeometryTypeKHR::VK_GEOMETRY_TYPE_AABBS_KHR: { - to_extract = { geometry->geometry.aabbs.data.deviceAddress }; + to_extract.push_back(geometry->geometry.aabbs.data.deviceAddress); break; } case VkGeometryTypeKHR::VK_GEOMETRY_TYPE_INSTANCES_KHR: { - to_extract = { geometry->geometry.instances.data.deviceAddress }; + to_extract.push_back(geometry->geometry.instances.data.deviceAddress); break; } case VK_GEOMETRY_TYPE_MAX_ENUM_KHR: @@ -479,7 +483,7 @@ void VulkanStateTracker::TrackAccelerationStructureBuildCommand( GFXRECON_ASSERT(target_buffer_wrapper != nullptr); vulkan_wrappers::AccelerationStructureKHRWrapper::ASInputBuffer& buffer = - dst_command.input_buffers.emplace_back(); + dst_command.input_buffers[target_buffer_wrapper->handle_id]; buffer.capture_address = address; buffer.handle = target_buffer_wrapper->handle; @@ -1979,23 +1983,23 @@ void gfxrecon::encode::VulkanStateTracker::DestroyState(vulkan_wrappers::BufferW { continue; } - for (vulkan_wrappers::AccelerationStructureKHRWrapper::ASInputBuffer& buffer : (*command)->input_buffers) - { - if (wrapper->handle_id == buffer.handle_id) - { - buffer.destroyed = true; - auto [resource_util, created] = resource_utils_.try_emplace( - buffer.bind_device->handle, - graphics::VulkanResourcesUtil(buffer.bind_device->handle, - buffer.bind_device->physical_device->handle, - buffer.bind_device->layer_table, - *buffer.bind_device->physical_device->layer_table_ref, - buffer.bind_device->physical_device->memory_properties)); - buffer.bind_device->layer_table.GetBufferMemoryRequirements( - buffer.bind_device->handle, buffer.handle, &buffer.memory_requirements); - resource_util->second.ReadFromBufferResource( - buffer.handle, buffer.created_size, 0, buffer.queue_family_index, buffer.bytes); - } + + auto it = (*command)->input_buffers.find(wrapper->handle_id); + if (it != (*command)->input_buffers.end()) + { + vulkan_wrappers::AccelerationStructureKHRWrapper::ASInputBuffer& buffer = it->second; + buffer.destroyed = true; + auto [resource_util, created] = resource_utils_.try_emplace( + buffer.bind_device->handle, + graphics::VulkanResourcesUtil(buffer.bind_device->handle, + buffer.bind_device->physical_device->handle, + buffer.bind_device->layer_table, + *buffer.bind_device->physical_device->layer_table_ref, + buffer.bind_device->physical_device->memory_properties)); + buffer.bind_device->layer_table.GetBufferMemoryRequirements( + buffer.bind_device->handle, buffer.handle, &buffer.memory_requirements); + resource_util->second.ReadFromBufferResource( + buffer.handle, buffer.created_size, 0, buffer.queue_family_index, buffer.bytes); } } }); diff --git a/framework/encode/vulkan_state_writer.cpp b/framework/encode/vulkan_state_writer.cpp index afe054fb97..271497719e 100644 --- a/framework/encode/vulkan_state_writer.cpp +++ b/framework/encode/vulkan_state_writer.cpp @@ -1697,8 +1697,7 @@ void VulkanStateWriter::WriteAccelerationStructureStateMetaCommands(const Vulkan if (wrapper->latest_build_command_) { build_container->push_back(&wrapper->latest_build_command_.value()); - for (const vulkan_wrappers::AccelerationStructureKHRWrapper::ASInputBuffer& buffer : - wrapper->latest_build_command_->input_buffers) + for (const auto& [handle_id, buffer] : wrapper->latest_build_command_->input_buffers) { max_resource_size = std::max(max_resource_size, buffer.bytes.size()); } @@ -1707,8 +1706,7 @@ void VulkanStateWriter::WriteAccelerationStructureStateMetaCommands(const Vulkan if (wrapper->latest_update_command_) { update_container->push_back(&wrapper->latest_update_command_.value()); - for (const vulkan_wrappers::AccelerationStructureKHRWrapper::ASInputBuffer& buffer : - wrapper->latest_update_command_->input_buffers) + for (const auto& [handle_id, buffer] : wrapper->latest_update_command_->input_buffers) { max_resource_size = std::max(max_resource_size, buffer.bytes.size()); } @@ -1763,7 +1761,7 @@ void VulkanStateWriter::WriteAccelerationStructureStateMetaCommands(const Vulkan void VulkanStateWriter::WriteAccelerationStructureBuildState(const gfxrecon::format::HandleId& device, AccelerationStructureBuildCommandData& command) { - for (ASInputBuffer& buffer : command.input_buffers) + for (auto& [handle_id, buffer] : command.input_buffers) { if (buffer.destroyed) { @@ -1775,7 +1773,7 @@ void VulkanStateWriter::WriteAccelerationStructureBuildState(const gfxrecon::for UpdateAddresses(command); EncodeAccelerationStructureBuildMetaCommand(device, command); - for (ASInputBuffer& buffer : command.input_buffers) + for (auto& [handle_id, buffer] : command.input_buffers) { if (buffer.destroyed) { @@ -1825,7 +1823,7 @@ void VulkanStateWriter::UpdateAddresses(AccelerationStructureBuildCommandData& c } } - for (const ASInputBuffer& buffer : command.input_buffers) + for (const auto& [handle_id, buffer] : command.input_buffers) { if (buffer.destroyed) { From e25bbf05ebd04953d414512b1c61379aad69f049 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 7 Jan 2025 12:19:41 +0100 Subject: [PATCH 10/23] Drop instance_buffers_data, cleanup, doc --- framework/decode/metadata_consumer_base.h | 3 +- framework/decode/vulkan_address_replacer.cpp | 30 +++++++------------ framework/decode/vulkan_address_replacer.h | 19 ++++++++---- framework/decode/vulkan_decoder_base.cpp | 29 ++---------------- .../decode/vulkan_replay_consumer_base.cpp | 4 +-- .../decode/vulkan_replay_consumer_base.h | 3 +- 6 files changed, 31 insertions(+), 57 deletions(-) diff --git a/framework/decode/metadata_consumer_base.h b/framework/decode/metadata_consumer_base.h index 546ae77227..80f7c28d95 100644 --- a/framework/decode/metadata_consumer_base.h +++ b/framework/decode/metadata_consumer_base.h @@ -116,8 +116,7 @@ class MetadataConsumerBase format::HandleId device_id, uint32_t info_count, StructPointerDecoder* geometry_infos, - StructPointerDecoder* range_infos, - std::vector>& instance_buffers_data) + StructPointerDecoder* range_infos) {} virtual void ProcessCopyVulkanAccelerationStructuresMetaCommand( diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 233ef2a6a9..f78bb0677b 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -516,13 +516,13 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto range_info = build_range_infos[i]; // check/correct scratch-address - if(!address_remap(build_geometry_info.scratchData.deviceAddress)) + if (!address_remap(build_geometry_info.scratchData.deviceAddress)) { - GFXRECON_LOG_WARNING( - "ProcessCmdBuildAccelerationStructuresKHR: missing scratch-buffer"); + GFXRECON_LOG_WARNING("ProcessCmdBuildAccelerationStructuresKHR: missing scratch-buffer"); } // check capture/replay acceleration-structure buffer-sizes + if (!_resource_allocator->SupportsOpaqueDeviceAddresses()) { VkAccelerationStructureBuildSizesInfoKHR build_size_info = {}; build_size_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR; @@ -555,7 +555,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( "VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: Replay adjusted mismatching " "acceleration-structures using shadow-structures and -buffers") - // now definitely requiring address-replaced + // now definitely requiring address-replacement force_replace = true; auto& replacment_as = _shadow_as_map[build_geometry_info.dstAccelerationStructure]; @@ -705,14 +705,10 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // mark injected commands mark_injected_commands_helper_t mark_injected_commands_helper; - if (_pipeline_bda == VK_NULL_HANDLE) + if (_pipeline_bda == VK_NULL_HANDLE && !init_pipeline()) { - if (!init_pipeline()) - { - GFXRECON_LOG_WARNING_ONCE( - "ProcessCmdBuildAccelerationStructuresKHR: internal pipeline-creation failed") - return; - } + GFXRECON_LOG_WARNING_ONCE("ProcessCmdBuildAccelerationStructuresKHR: internal pipeline-creation failed") + return; } auto& pipeline_context_bda = _build_as_context_map[command_buffer_info->handle]; @@ -850,15 +846,11 @@ void VulkanAddressReplacer::ProcessUpdateDescriptorSets(uint32_t de } void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( - uint32_t info_count, - VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, - VkAccelerationStructureBuildRangeInfoKHR** range_infos, - std::vector>& instance_buffers_data, - const decode::VulkanDeviceAddressTracker& address_tracker) + uint32_t info_count, + VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, + VkAccelerationStructureBuildRangeInfoKHR** range_infos, + const decode::VulkanDeviceAddressTracker& address_tracker) { - // TODO: figure out what to do - GFXRECON_UNREFERENCED_PARAMETER(instance_buffers_data); - if (info_count > 0 && init_queue_assets()) { // reset/submit/sync command-buffer diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index c15cb16213..12350240b9 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -91,6 +91,17 @@ class VulkanAddressReplacer * @brief ProcessCmdBuildAccelerationStructuresKHR will check * and potentially correct input-parameters to 'vkCmdBuildAccelerationStructuresKHR' * + * Depending on capture- and replay-device-properties this includes the following: + * + * if replaying on same device/driver using the default-allocator (no --rebind): + * - happy day, nothing to do! + * + * if replaying on a different device/driver OR using the rebind-allocator (via --rebind): + * - remap buffer-device-addresses for triangle-, aabb- and instance-geometries referenced in `build_geometry_infos` + * - check buffer-sizes for acceleration-structures and scratch-buffers + * - if necessary, create shadow acceleration-structures and -buffers, adjust references + * - apply in-place correction of acceleration-structure device-addresses referenced by top-level builds + * * @param command_buffer_info a provided VulkanCommandBufferInfo * @param info_count number of elements in 'build_geometry_infos' * @param build_geometry_infos provided array of VkAccelerationStructureBuildGeometryInfoKHR @@ -138,19 +149,17 @@ class VulkanAddressReplacer /** * @brief Process information contained in a metadata-block in order to build acceleration-structures. * - * Will use an internal command-pool, submit work to a VkQueue and perform a host-synchronization. + * Will use an internal command-buffer, submit work to a VkQueue and perform host-synchronization. * * @param info_count element count in 'geometry_infos' * @param geometry_infos provided array of VkAccelerationStructureBuildGeometryInfoKHR * @param range_infos provided array of pointers to VkAccelerationStructureBuildRangeInfoKHR - * @param instance_buffers_data an array of arrays of VkAccelerationStructureInstanceKHR * @param address_tracker const reference to a VulkanDeviceAddressTracker */ void ProcessBuildVulkanAccelerationStructuresMetaCommand( uint32_t info_count, VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, VkAccelerationStructureBuildRangeInfoKHR** range_infos, - std::vector>& instance_buffers_data, const decode::VulkanDeviceAddressTracker& address_tracker); /** @@ -265,13 +274,13 @@ class VulkanAddressReplacer util::linear_hashmap _hashmap_bda; std::unordered_map _shadow_sbt_map; - // TODO: check if this is sufficient + // pipeline-contexts dealing with shader-binding-tables, per command-buffer std::unordered_map _pipeline_sbt_context_map; // resources related to acceleration-structures std::unordered_map _shadow_as_map; - // TODO: check if this is sufficient + // pipeline-contexts dealing with acceleration-structure builds, per command-buffer std::unordered_map _build_as_context_map; // required function pointers diff --git a/framework/decode/vulkan_decoder_base.cpp b/framework/decode/vulkan_decoder_base.cpp index fed5c31f5a..4c34afd288 100644 --- a/framework/decode/vulkan_decoder_base.cpp +++ b/framework/decode/vulkan_decoder_base.cpp @@ -578,35 +578,12 @@ void VulkanDecoderBase::DispatchVulkanAccelerationStructuresBuildMetaCommand(con std::size_t bytes_read = ValueDecoder::DecodeHandleIdValue(parameter_buffer, buffer_size, &device_id); bytes_read += pInfos.Decode(parameter_buffer + bytes_read, buffer_size - bytes_read); - bytes_read += ppRangeInfos.Decode(parameter_buffer + bytes_read, buffer_size - bytes_read); - - std::vector> instance_buffers; - if (bytes_read < buffer_size) - { - for (uint32_t i = 0; i < pInfos.GetLength(); ++i) - { - if (pInfos.GetPointer()[i].type != VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR) - { - continue; - } - - uint32_t geometry_count = pInfos.GetPointer()[i].geometryCount; - for (uint32_t g = 0; g < geometry_count; ++g) - { - instance_buffers.emplace_back(ppRangeInfos.GetPointer()[g]->primitiveCount); - util::platform::MemoryCopy(instance_buffers.back().data(), - instance_buffers.back().size() * sizeof(VkAccelerationStructureInstanceKHR), - parameter_buffer + bytes_read, - instance_buffers.back().size() * sizeof(VkAccelerationStructureInstanceKHR)); - bytes_read += instance_buffers.back().size() * sizeof(VkAccelerationStructureInstanceKHR); - } - } - } + ppRangeInfos.Decode(parameter_buffer + bytes_read, buffer_size - bytes_read); for (auto consumer : consumers_) { consumer->ProcessBuildVulkanAccelerationStructuresMetaCommand( - device_id, pInfos.GetLength(), &pInfos, &ppRangeInfos, instance_buffers); + device_id, pInfos.GetLength(), &pInfos, &ppRangeInfos); } } @@ -634,7 +611,7 @@ void VulkanDecoderBase::DispatchVulkanAccelerationStructuresWritePropertiesMetaC std::size_t bytes_read = ValueDecoder::DecodeHandleIdValue(parameter_buffer, sizeof(format::HandleId), &device_id); bytes_read += ValueDecoder::DecodeEnumValue(parameter_buffer + bytes_read, sizeof(VkQueryType), &query_type); - bytes_read += ValueDecoder::DecodeHandleIdValue( + ValueDecoder::DecodeHandleIdValue( parameter_buffer + bytes_read, sizeof(format::HandleId), &acceleration_structure_id); for (auto consumer : consumers_) diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index e7fbe67bd9..8c171e1d81 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -9543,8 +9543,7 @@ void VulkanReplayConsumerBase::ProcessBuildVulkanAccelerationStructuresMetaComma format::HandleId device, uint32_t info_count, StructPointerDecoder* pInfos, - StructPointerDecoder* ppRangeInfos, - std::vector>& instance_buffers_data) + StructPointerDecoder* ppRangeInfos) { if (loading_trim_state_) { @@ -9560,7 +9559,6 @@ void VulkanReplayConsumerBase::ProcessBuildVulkanAccelerationStructuresMetaComma .ProcessBuildVulkanAccelerationStructuresMetaCommand(info_count, pInfos->GetPointer(), ppRangeInfos->GetPointer(), - instance_buffers_data, GetDeviceAddressTracker(device_info)); } } diff --git a/framework/decode/vulkan_replay_consumer_base.h b/framework/decode/vulkan_replay_consumer_base.h index cd45c3bdfa..b109bca9bc 100644 --- a/framework/decode/vulkan_replay_consumer_base.h +++ b/framework/decode/vulkan_replay_consumer_base.h @@ -205,8 +205,7 @@ class VulkanReplayConsumerBase : public VulkanConsumer format::HandleId device, uint32_t info_count, StructPointerDecoder* pInfos, - StructPointerDecoder* ppRangeInfos, - std::vector>& instance_buffers_data) override; + StructPointerDecoder* ppRangeInfos) override; void ProcessCopyVulkanAccelerationStructuresMetaCommand( format::HandleId device, StructPointerDecoder* copy_infos) override; From 448f41e3b3b37242895923f8216cc11334503ce4 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Thu, 9 Jan 2025 10:52:19 +0100 Subject: [PATCH 11/23] Seperate AS- and scratch shadow-objects, minor improvements --- framework/decode/vulkan_address_replacer.cpp | 179 +++++++++++-------- 1 file changed, 104 insertions(+), 75 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index f78bb0677b..da77f11410 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -152,9 +152,8 @@ decode::VulkanAddressReplacer::buffer_context_t::~buffer_context_t() decode::VulkanAddressReplacer::acceleration_structure_asset_t::~acceleration_structure_asset_t() { - if (handle != VK_NULL_HANDLE) + if (handle != VK_NULL_HANDLE && destroy_fn != nullptr && device != VK_NULL_HANDLE) { - GFXRECON_ASSERT(destroy_fn != nullptr && device != VK_NULL_HANDLE) destroy_fn(device, handle, nullptr); } } @@ -271,6 +270,38 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // _valid_sbt_alignment = false; // valid_group_handles = false; + std::unordered_set buffer_set; + + auto address_remap = [&address_tracker, &buffer_set](VkStridedDeviceAddressRegionKHR* address_region) { + if (address_region->size > 0) + { + auto buffer_info = address_tracker.GetBufferByCaptureDeviceAddress(address_region->deviceAddress); + GFXRECON_ASSERT(buffer_info != nullptr); + + if (buffer_info->replay_address != 0) + { + // keep track of used handles + buffer_set.insert(buffer_info->handle); + + uint64_t offset = address_region->deviceAddress - buffer_info->capture_address; + + // in-place address-remap + address_region->deviceAddress = buffer_info->replay_address + offset; + } + else + { + GFXRECON_LOG_INFO_ONCE( + "VulkanAddressReplacer::ProcessCmdTraceRays: missing buffer_info->replay_address, remap failed") + } + } + }; + + // in-place remap: capture-addresses -> replay-addresses + address_remap(raygen_sbt); + address_remap(miss_sbt); + address_remap(hit_sbt); + address_remap(callable_sbt); + if (!_valid_sbt_alignment || !valid_group_handles) { // mark injected commands @@ -284,37 +315,6 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( return; } } - std::unordered_set buffer_set; - - auto address_remap = [&address_tracker, &buffer_set](VkStridedDeviceAddressRegionKHR* address_region) { - if (address_region->size > 0) - { - auto buffer_info = address_tracker.GetBufferByCaptureDeviceAddress(address_region->deviceAddress); - GFXRECON_ASSERT(buffer_info != nullptr); - - if (buffer_info->replay_address != 0) - { - // keep track of used handles - buffer_set.insert(buffer_info->handle); - - uint64_t offset = address_region->deviceAddress - buffer_info->capture_address; - - // in-place address-remap - address_region->deviceAddress = buffer_info->replay_address + offset; - } - else - { - GFXRECON_LOG_INFO_ONCE( - "VulkanAddressReplacer::ProcessCmdTraceRays: missing buffer_info->replay_address, remap failed") - } - } - }; - - // in-place remap: capture-addresses -> replay-addresses - address_remap(raygen_sbt); - address_remap(miss_sbt); - address_remap(hit_sbt); - address_remap(callable_sbt); // prepare linear hashmap _hashmap_sbt.clear(); @@ -489,6 +489,12 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto address_remap = [&address_tracker, &buffer_set](VkDeviceAddress& capture_address) -> bool { auto buffer_info = address_tracker.GetBufferByCaptureDeviceAddress(capture_address); + // skip over null-addresses + if (capture_address == 0) + { + return false; + } + if (buffer_info != nullptr && buffer_info->replay_address != 0) { // keep track of used handles @@ -515,6 +521,9 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto& build_geometry_info = build_geometry_infos[i]; auto range_info = build_range_infos[i]; + const VulkanBufferInfo* scratch_buffer_info = + address_tracker.GetBufferByCaptureDeviceAddress(build_geometry_info.scratchData.deviceAddress); + // check/correct scratch-address if (!address_remap(build_geometry_info.scratchData.deviceAddress)) { @@ -549,7 +558,16 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto* buffer_info = address_tracker.GetBufferByHandle(acceleration_structure_info->buffer); GFXRECON_ASSERT(buffer_info != nullptr) - if (buffer_info != nullptr && buffer_info->size < build_size_info.accelerationStructureSize) + bool as_buffer_usable = + buffer_info != nullptr && buffer_info->size >= build_size_info.accelerationStructureSize; + + // determine required size of scratch-buffer + uint32_t scratch_size = build_geometry_info.mode == VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR + ? build_size_info.buildScratchSize + : build_size_info.updateScratchSize; + bool scratch_buffer_usable = scratch_buffer_info != nullptr && scratch_buffer_info->size >= scratch_size; + + if (!as_buffer_usable || !scratch_buffer_usable) { GFXRECON_LOG_INFO_ONCE( "VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: Replay adjusted mismatching " @@ -562,53 +580,63 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (replacment_as.handle == VK_NULL_HANDLE) { - replacment_as.device = _device; - replacment_as.destroy_fn = _device_table->DestroyAccelerationStructureKHR; - - // create a replacement acceleration-structure with proper sized buffer - bool success = create_buffer(build_size_info.accelerationStructureSize, - replacment_as.storage, - VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR, - 0, - false); - - if (!success) + if (as_buffer_usable) { - GFXRECON_LOG_ERROR("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: " - "shadow-buffer creation failed"); - return; + replacment_as.handle = build_geometry_info.dstAccelerationStructure; + auto accel_info = address_tracker.GetAccelerationStructureByHandle( + build_geometry_info.dstAccelerationStructure); + GFXRECON_ASSERT(accel_info != nullptr && accel_info->replay_address != 0); + replacment_as.address = accel_info->replay_address; } - - VkAccelerationStructureCreateInfoKHR as_create_info = {}; - as_create_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR; - as_create_info.buffer = replacment_as.storage.buffer; - as_create_info.size = build_size_info.accelerationStructureSize; - as_create_info.type = build_geometry_info.type; - - VkResult res = _device_table->CreateAccelerationStructureKHR( - _device, &as_create_info, nullptr, &replacment_as.handle); - - if (res != VK_SUCCESS || replacment_as.handle == VK_NULL_HANDLE) + else { - GFXRECON_LOG_ERROR( - "ProcessCmdBuildAccelerationStructuresKHR: shadow acceleration-structure creation failed"); - return; + replacment_as.device = _device; + replacment_as.destroy_fn = _device_table->DestroyAccelerationStructureKHR; + + // create a replacement acceleration-structure with proper sized buffer + bool success = create_buffer(build_size_info.accelerationStructureSize, + replacment_as.storage, + VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR, + 0, + false); + + if (!success) + { + GFXRECON_LOG_ERROR("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: " + "shadow-buffer creation failed"); + return; + } + + VkAccelerationStructureCreateInfoKHR as_create_info = {}; + as_create_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR; + as_create_info.buffer = replacment_as.storage.buffer; + as_create_info.size = build_size_info.accelerationStructureSize; + as_create_info.type = build_geometry_info.type; + + VkResult res = _device_table->CreateAccelerationStructureKHR( + _device, &as_create_info, nullptr, &replacment_as.handle); + + if (res != VK_SUCCESS || replacment_as.handle == VK_NULL_HANDLE) + { + GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: shadow " + "acceleration-structure creation failed"); + return; + } + VkAccelerationStructureDeviceAddressInfoKHR acceleration_address_info = {}; + acceleration_address_info.sType = + VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR; + acceleration_address_info.accelerationStructure = replacment_as.handle; + replacment_as.address = _device_table->GetAccelerationStructureDeviceAddressKHR( + _device, &acceleration_address_info); + GFXRECON_ASSERT(replacment_as.address != 0) } - VkAccelerationStructureDeviceAddressInfoKHR acceleration_address_info = {}; - acceleration_address_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR; - acceleration_address_info.accelerationStructure = replacment_as.handle; - replacment_as.address = - _device_table->GetAccelerationStructureDeviceAddressKHR(_device, &acceleration_address_info); - GFXRECON_ASSERT(replacment_as.address != 0) - - // hot swap acceleration-structure handle - build_geometry_info.dstAccelerationStructure = replacment_as.handle; } - // determine required size of scratch-buffer - uint32_t scratch_size = build_geometry_info.mode == VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR - ? build_size_info.buildScratchSize - : build_size_info.updateScratchSize; + // tmp + GFXRECON_ASSERT(build_geometry_info.srcAccelerationStructure == VK_NULL_HANDLE); + + // hot swap acceleration-structure handle + build_geometry_info.dstAccelerationStructure = replacment_as.handle; // create a replacement scratch-buffer if (!create_buffer( @@ -639,6 +667,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto& triangles = geometry->geometry.triangles; address_remap(triangles.vertexData.deviceAddress); address_remap(triangles.indexData.deviceAddress); + address_remap(triangles.transformData.deviceAddress); break; } case VK_GEOMETRY_TYPE_AABBS_KHR: From 01d19b45000b1a097b3c8b2c85e0fadd31787a85 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Mon, 13 Jan 2025 12:16:11 +0100 Subject: [PATCH 12/23] Add VulkanCaptureManager::OverrideCmdCopyAccelerationStructureKHR --- framework/decode/vulkan_address_replacer.cpp | 175 +++++++++++------- framework/decode/vulkan_address_replacer.h | 35 ++-- framework/decode/vulkan_object_info.h | 2 + .../decode/vulkan_replay_consumer_base.cpp | 13 +- framework/encode/vulkan_capture_manager.cpp | 12 +- framework/encode/vulkan_capture_manager.h | 3 + framework/encode/vulkan_state_tracker.cpp | 20 +- framework/encode/vulkan_state_tracker.h | 3 + framework/encode/vulkan_state_writer.cpp | 15 +- framework/encode/vulkan_state_writer.h | 8 +- .../generated_vulkan_api_call_encoders.cpp | 5 +- .../vulkan_generators/capture_overrides.json | 1 + 12 files changed, 191 insertions(+), 101 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index da77f11410..55c78d69a9 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -327,7 +327,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // get a context for this command-buffer auto& pipeline_context_sbt = _pipeline_sbt_context_map[command_buffer_info->handle]; - if (!create_buffer(_hashmap_sbt.get_storage(nullptr), pipeline_context_sbt.hashmap_storage)) + if (!create_buffer(pipeline_context_sbt.hashmap_storage, _hashmap_sbt.get_storage(nullptr))) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); } @@ -335,7 +335,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // input-handles constexpr uint32_t max_num_handles = 4; - if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), pipeline_context_sbt.input_handle_buffer)) + if (!create_buffer(pipeline_context_sbt.input_handle_buffer, max_num_handles * sizeof(VkDeviceAddress))) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); } @@ -383,7 +383,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( "raytracing shader-binding-table using a shadow-buffer"); // output-handles - if (!create_buffer(max_num_handles * sizeof(VkDeviceAddress), pipeline_context_sbt.output_handle_buffer)) + if (!create_buffer(pipeline_context_sbt.output_handle_buffer, max_num_handles * sizeof(VkDeviceAddress))) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); return; @@ -415,7 +415,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // raygen: stride == size raygen_sbt->size = raygen_sbt->stride = _replay_ray_properties.shaderGroupBaseAlignment; - if (!create_buffer(sbt_offset, shadow_buf_context, VK_BUFFER_USAGE_SHADER_BINDING_TABLE_BIT_KHR)) + if (!create_buffer(shadow_buf_context, sbt_offset, VK_BUFFER_USAGE_SHADER_BINDING_TABLE_BIT_KHR)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: shadow shader-binding-table creation failed"); return; @@ -536,13 +536,13 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( VkAccelerationStructureBuildSizesInfoKHR build_size_info = {}; build_size_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR; - std::vector primitive_counts(build_geometry_info.geometryCount); - for (uint32_t j = 0; j < build_geometry_info.geometryCount; ++j) { - primitive_counts[j] = range_info->primitiveCount; - } + std::vector primitive_counts(build_geometry_info.geometryCount); + for (uint32_t j = 0; j < build_geometry_info.geometryCount; ++j) + { + primitive_counts[j] = range_info->primitiveCount; + } - { mark_injected_commands_helper_t mark_injected_commands_helper; _device_table->GetAccelerationStructureBuildSizesKHR(_device, VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, @@ -576,59 +576,24 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // now definitely requiring address-replacement force_replace = true; - auto& replacment_as = _shadow_as_map[build_geometry_info.dstAccelerationStructure]; + auto& replacement_as = _shadow_as_map[build_geometry_info.dstAccelerationStructure]; - if (replacment_as.handle == VK_NULL_HANDLE) + if (replacement_as.handle == VK_NULL_HANDLE) { if (as_buffer_usable) { - replacment_as.handle = build_geometry_info.dstAccelerationStructure; - auto accel_info = address_tracker.GetAccelerationStructureByHandle( + replacement_as.handle = build_geometry_info.dstAccelerationStructure; + auto accel_info = address_tracker.GetAccelerationStructureByHandle( build_geometry_info.dstAccelerationStructure); GFXRECON_ASSERT(accel_info != nullptr && accel_info->replay_address != 0); - replacment_as.address = accel_info->replay_address; + replacement_as.address = accel_info->replay_address; } - else + else if (!create_acceleration_asset(replacement_as, + build_geometry_info.type, + build_size_info.accelerationStructureSize, + scratch_size)) { - replacment_as.device = _device; - replacment_as.destroy_fn = _device_table->DestroyAccelerationStructureKHR; - - // create a replacement acceleration-structure with proper sized buffer - bool success = create_buffer(build_size_info.accelerationStructureSize, - replacment_as.storage, - VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR, - 0, - false); - - if (!success) - { - GFXRECON_LOG_ERROR("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: " - "shadow-buffer creation failed"); - return; - } - - VkAccelerationStructureCreateInfoKHR as_create_info = {}; - as_create_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR; - as_create_info.buffer = replacment_as.storage.buffer; - as_create_info.size = build_size_info.accelerationStructureSize; - as_create_info.type = build_geometry_info.type; - - VkResult res = _device_table->CreateAccelerationStructureKHR( - _device, &as_create_info, nullptr, &replacment_as.handle); - - if (res != VK_SUCCESS || replacment_as.handle == VK_NULL_HANDLE) - { - GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: shadow " - "acceleration-structure creation failed"); - return; - } - VkAccelerationStructureDeviceAddressInfoKHR acceleration_address_info = {}; - acceleration_address_info.sType = - VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR; - acceleration_address_info.accelerationStructure = replacment_as.handle; - replacment_as.address = _device_table->GetAccelerationStructureDeviceAddressKHR( - _device, &acceleration_address_info); - GFXRECON_ASSERT(replacment_as.address != 0) + // problem creating shadow-AS } } @@ -636,12 +601,12 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( GFXRECON_ASSERT(build_geometry_info.srcAccelerationStructure == VK_NULL_HANDLE); // hot swap acceleration-structure handle - build_geometry_info.dstAccelerationStructure = replacment_as.handle; + build_geometry_info.dstAccelerationStructure = replacement_as.handle; // create a replacement scratch-buffer if (!create_buffer( + replacement_as.scratch, scratch_size, - replacment_as.scratch, 0, _replay_acceleration_structure_properties.minAccelerationStructureScratchOffsetAlignment, false)) @@ -651,7 +616,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } // hot swap scratch-buffer - build_geometry_info.scratchData.deviceAddress = replacment_as.scratch.device_address; + build_geometry_info.scratchData.deviceAddress = replacement_as.scratch.device_address; } } @@ -725,6 +690,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } // store addresses we will need to replace + GFXRECON_ASSERT(new_address != 0); _hashmap_bda.put(capture_address, new_address); } } @@ -742,7 +708,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( auto& pipeline_context_bda = _build_as_context_map[command_buffer_info->handle]; - if (!create_buffer(_hashmap_bda.get_storage(nullptr), pipeline_context_bda.hashmap_storage)) + if (!create_buffer(pipeline_context_bda.hashmap_storage, _hashmap_bda.get_storage(nullptr))) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); return; @@ -751,7 +717,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( uint32_t num_bytes = addresses_to_replace.size() * sizeof(VkDeviceAddress); - if (!create_buffer(num_bytes, pipeline_context_bda.input_handle_buffer)) + if (!create_buffer(pipeline_context_bda.input_handle_buffer, num_bytes)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: input-handle-buffer creation failed"); return; @@ -807,7 +773,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } } -void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR(VkCopyAccelerationStructureInfoKHR* info) +void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR( + VkCopyAccelerationStructureInfoKHR* info, const decode::VulkanDeviceAddressTracker& address_tracker) { if (info != nullptr) { @@ -822,6 +789,31 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR(VkCopyAccele // correct in-place swap_acceleration_structure(info->src); swap_acceleration_structure(info->dst); + + GFXRECON_ASSERT(info->dst != VK_NULL_HANDLE); + + // // tmp -> we don't arrive here during trim!? + // auto replace_it = _shadow_as_map.find(info->dst); + // if (replace_it == _shadow_as_map.end()) + // { + // if (info->dst != VK_NULL_HANDLE) + // { + // auto as_info = address_tracker.GetAccelerationStructureByHandle(info->dst); + // GFXRECON_ASSERT(as_info != nullptr); + // + // acceleration_structure_asset_t& new_dst = _shadow_as_map[info->dst]; + // uint32_t fake_as_buffer_size = 5 * (1 << 20); + // + // if (create_acceleration_asset(new_dst, as_info->type, fake_as_buffer_size, 0)) + // { + // swap_acceleration_structure(info->dst); + // } + // } + // else + // { + // GFXRECON_ASSERT(info->dst != VK_NULL_HANDLE); + // } + // } } } @@ -890,15 +882,27 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( command_buffer_info.handle = _command_buffer; ProcessCmdBuildAccelerationStructuresKHR( &command_buffer_info, info_count, geometry_infos, range_infos, address_tracker); + + // issue build-command + _device_table->CmdBuildAccelerationStructuresKHR(_command_buffer, info_count, geometry_infos, range_infos); } } void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( - uint32_t info_count, VkCopyAccelerationStructureInfoKHR* copy_infos) + uint32_t info_count, + VkCopyAccelerationStructureInfoKHR* copy_infos, + const decode::VulkanDeviceAddressTracker& address_tracker) { - for (uint32_t i = 0; i < info_count; ++i) + if (info_count > 0 && init_queue_assets()) { - ProcessCmdCopyAccelerationStructuresKHR(copy_infos + i); + // reset/submit/sync command-buffer + queue_submit_helper_t queue_submit_helper(_device_table, _device, _command_buffer, _queue, _fence); + + for (uint32_t i = 0; i < info_count; ++i) + { + ProcessCmdCopyAccelerationStructuresKHR(copy_infos + i, address_tracker); + } + _device_table->CmdCopyAccelerationStructureKHR(_command_buffer, copy_infos); } } @@ -1048,8 +1052,8 @@ bool VulkanAddressReplacer::init_queue_assets() return _queue != VK_NULL_HANDLE; } -bool VulkanAddressReplacer::create_buffer(size_t num_bytes, - VulkanAddressReplacer::buffer_context_t& buffer_context, +bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_t& buffer_context, + size_t num_bytes, uint32_t usage_flags, uint32_t min_alignment, bool use_host_mem) @@ -1219,5 +1223,46 @@ void VulkanAddressReplacer::DestroyShadowResources(VkCommandBuffer handle) } } +bool VulkanAddressReplacer::create_acceleration_asset(VulkanAddressReplacer::acceleration_structure_asset_t& as_asset, + VkAccelerationStructureTypeKHR type, + size_t num_buffer_bytes, + size_t num_scratch_bytes) +{ + as_asset.device = _device; + as_asset.destroy_fn = _device_table->DestroyAccelerationStructureKHR; + + // create a replacement acceleration-structure with proper sized buffer + bool success = create_buffer( + as_asset.storage, num_buffer_bytes, VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR, 0, false); + + if (!success) + { + GFXRECON_LOG_ERROR("VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: " + "shadow-buffer creation failed"); + return false; + } + + VkAccelerationStructureCreateInfoKHR as_create_info = {}; + as_create_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR; + as_create_info.buffer = as_asset.storage.buffer; + as_create_info.size = num_buffer_bytes; + as_create_info.type = type; + + VkResult res = _device_table->CreateAccelerationStructureKHR(_device, &as_create_info, nullptr, &as_asset.handle); + + if (res != VK_SUCCESS || as_asset.handle == VK_NULL_HANDLE) + { + GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: shadow " + "acceleration-structure creation failed"); + return false; + } + VkAccelerationStructureDeviceAddressInfoKHR acceleration_address_info = {}; + acceleration_address_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR; + acceleration_address_info.accelerationStructure = as_asset.handle; + as_asset.address = _device_table->GetAccelerationStructureDeviceAddressKHR(_device, &acceleration_address_info); + GFXRECON_ASSERT(as_asset.address != 0) + return true; +} + GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 12350240b9..fb373a80f0 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -118,9 +118,11 @@ class VulkanAddressReplacer * @brief ProcessCmdCopyAccelerationStructuresKHR will check * and potentially correct input-parameters to 'vkCmdCopyAccelerationStructuresKHR' * - * @param info a provided VkCopyAccelerationStructureInfoKHR* + * @param info a provided VkCopyAccelerationStructureInfoKHR* + * @param address_tracker const reference to a VulkanDeviceAddressTracker, used for mapping device-addresses */ - void ProcessCmdCopyAccelerationStructuresKHR(VkCopyAccelerationStructureInfoKHR* info); + void ProcessCmdCopyAccelerationStructuresKHR(VkCopyAccelerationStructureInfoKHR* info, + const decode::VulkanDeviceAddressTracker& address_tracker); /** * @brief ProcessCmdWriteAccelerationStructuresPropertiesKHR will check @@ -156,20 +158,22 @@ class VulkanAddressReplacer * @param range_infos provided array of pointers to VkAccelerationStructureBuildRangeInfoKHR * @param address_tracker const reference to a VulkanDeviceAddressTracker */ - void ProcessBuildVulkanAccelerationStructuresMetaCommand( - uint32_t info_count, - VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, - VkAccelerationStructureBuildRangeInfoKHR** range_infos, - const decode::VulkanDeviceAddressTracker& address_tracker); + void + ProcessBuildVulkanAccelerationStructuresMetaCommand(uint32_t info_count, + VkAccelerationStructureBuildGeometryInfoKHR* geometry_infos, + VkAccelerationStructureBuildRangeInfoKHR** range_infos, + const decode::VulkanDeviceAddressTracker& address_tracker); /** * @brief Process information contained in a metadata-block in order to copy acceleration-structures. * - * @param info_count element count in 'copy_infos' - * @param copy_infos provided array of VkCopyAccelerationStructureInfoKHR + * @param info_count element count in 'copy_infos' + * @param copy_infos provided array of VkCopyAccelerationStructureInfoKHR + * @param address_tracker const reference to a VulkanDeviceAddressTracker */ - void ProcessCopyVulkanAccelerationStructuresMetaCommand(uint32_t info_count, - VkCopyAccelerationStructureInfoKHR* copy_infos); + void ProcessCopyVulkanAccelerationStructuresMetaCommand(uint32_t info_count, + VkCopyAccelerationStructureInfoKHR* copy_infos, + const decode::VulkanDeviceAddressTracker& address_tracker); /** * @brief Process information contained in a metadata-block in order to write information in a query-pool. * @@ -233,12 +237,17 @@ class VulkanAddressReplacer [[nodiscard]] bool init_queue_assets(); - [[nodiscard]] bool create_buffer(size_t num_bytes, - buffer_context_t& buffer_context, + [[nodiscard]] bool create_buffer(buffer_context_t& buffer_context, + size_t num_bytes, uint32_t usage_flags = 0, uint32_t min_alignment = 0, bool use_host_mem = true); + [[nodiscard]] bool create_acceleration_asset(acceleration_structure_asset_t& as_asset, + VkAccelerationStructureTypeKHR type, + size_t num_buffer_bytes, + size_t num_scratch_bytes); + void barrier(VkCommandBuffer command_buffer, VkBuffer buffer, VkPipelineStageFlags src_stage, diff --git a/framework/decode/vulkan_object_info.h b/framework/decode/vulkan_object_info.h index 2c5167dc08..76f43e09a0 100644 --- a/framework/decode/vulkan_object_info.h +++ b/framework/decode/vulkan_object_info.h @@ -709,6 +709,8 @@ struct VulkanAccelerationStructureKHRInfo : public VulkanObjectInfo(pAccelerationStructureKHR->GetConsumerData(0)); GFXRECON_ASSERT(acceleration_structure_info); + acceleration_structure_info->type = replay_create_info->type; acceleration_structure_info->buffer = replay_create_info->buffer; if (device_info->property_feature_info.feature_accelerationStructureCaptureReplay) @@ -7745,7 +7746,6 @@ VkResult VulkanReplayConsumerBase::OverrideCreateAccelerationStructureKHR( { result = func(device, replay_create_info, GetAllocationCallbacks(pAllocator), replay_accel_struct); } - return result; } @@ -7813,8 +7813,9 @@ void VulkanReplayConsumerBase::OverrideCmdCopyAccelerationStructureKHR( VkCopyAccelerationStructureInfoKHR* info = pInfo->GetPointer(); { - auto& address_replacer = GetDeviceAddressReplacer(device_info); - address_replacer.ProcessCmdCopyAccelerationStructuresKHR(info); + const auto& address_tracker = GetDeviceAddressTracker(device_info); + auto& address_replacer = GetDeviceAddressReplacer(device_info); + address_replacer.ProcessCmdCopyAccelerationStructuresKHR(info, address_tracker); } func(command_buffer, info); } @@ -9534,8 +9535,10 @@ void VulkanReplayConsumerBase::ProcessCopyVulkanAccelerationStructuresMetaComman MapStructArrayHandles(copy_infos->GetMetaStructPointer(), copy_infos->GetLength(), GetObjectInfoTable()); - auto& ar = GetDeviceAddressReplacer(device_info); - ar.ProcessCopyVulkanAccelerationStructuresMetaCommand(copy_infos->GetLength(), copy_infos->GetPointer()); + const auto& address_tracker = GetDeviceAddressTracker(device_info); + auto& address_replacer = GetDeviceAddressReplacer(device_info); + address_replacer.ProcessCopyVulkanAccelerationStructuresMetaCommand( + copy_infos->GetLength(), copy_infos->GetPointer(), address_tracker); } } diff --git a/framework/encode/vulkan_capture_manager.cpp b/framework/encode/vulkan_capture_manager.cpp index e9fbd30c0f..b0be82eb09 100644 --- a/framework/encode/vulkan_capture_manager.cpp +++ b/framework/encode/vulkan_capture_manager.cpp @@ -951,11 +951,21 @@ void VulkanCaptureManager::OverrideCmdBuildAccelerationStructuresKHR( { state_tracker_->TrackAccelerationStructureBuildCommand(commandBuffer, infoCount, pInfos, ppBuildRangeInfos); } - const VulkanDeviceTable* device_table = vulkan_wrappers::GetDeviceTable(commandBuffer); device_table->CmdBuildAccelerationStructuresKHR(commandBuffer, infoCount, pInfos, ppBuildRangeInfos); } +void VulkanCaptureManager::OverrideCmdCopyAccelerationStructureKHR(VkCommandBuffer command_buffer, + const VkCopyAccelerationStructureInfoKHR* pInfo) +{ + if (IsCaptureModeTrack()) + { + state_tracker_->TrackAccelerationStructureCopyCommand(command_buffer, pInfo); + } + const VulkanDeviceTable* device_table = vulkan_wrappers::GetDeviceTable(command_buffer); + device_table->CmdCopyAccelerationStructureKHR(command_buffer, pInfo); +} + VkResult VulkanCaptureManager::OverrideAllocateMemory(VkDevice device, const VkMemoryAllocateInfo* pAllocateInfo, const VkAllocationCallbacks* pAllocator, diff --git a/framework/encode/vulkan_capture_manager.h b/framework/encode/vulkan_capture_manager.h index f4916da2f8..e345013f7f 100644 --- a/framework/encode/vulkan_capture_manager.h +++ b/framework/encode/vulkan_capture_manager.h @@ -298,6 +298,9 @@ class VulkanCaptureManager : public ApiCaptureManager const VkAccelerationStructureBuildGeometryInfoKHR* pInfos, const VkAccelerationStructureBuildRangeInfoKHR* const* ppBuildRangeInfos); + void OverrideCmdCopyAccelerationStructureKHR(VkCommandBuffer command_buffer, + const VkCopyAccelerationStructureInfoKHR* pInfo); + VkResult OverrideAllocateMemory(VkDevice device, const VkMemoryAllocateInfo* pAllocateInfo, const VkAllocationCallbacks* pAllocator, diff --git a/framework/encode/vulkan_state_tracker.cpp b/framework/encode/vulkan_state_tracker.cpp index 3354e7326e..d8b373ade3 100644 --- a/framework/encode/vulkan_state_tracker.cpp +++ b/framework/encode/vulkan_state_tracker.cpp @@ -537,6 +537,24 @@ void VulkanStateTracker::TrackAccelerationStructureBuildCommand( } } +void VulkanStateTracker::TrackAccelerationStructureCopyCommand(VkCommandBuffer command_buffer, + const VkCopyAccelerationStructureInfoKHR* info) +{ + // TODO: Support other types of copies (clone, serialize, deserialize) + if (info == nullptr || info->mode != VK_COPY_ACCELERATION_STRUCTURE_MODE_COMPACT_KHR) + { + return; + } + auto wrapper = vulkan_wrappers::GetWrapper(info->src); + if (!wrapper->latest_copy_command_) + { + wrapper->latest_copy_command_ = + vulkan_wrappers::AccelerationStructureKHRWrapper::AccelerationStructureCopyCommandData{}; + } + wrapper->latest_copy_command_->device = wrapper->device->handle_id; + wrapper->latest_copy_command_->info = *info; +} + void VulkanStateTracker::TrackImageMemoryBinding( VkDevice device, VkImage image, VkDeviceMemory memory, VkDeviceSize memoryOffset, const void* bind_info_pnext) { @@ -2812,7 +2830,7 @@ void VulkanStateTracker::TrackMappedAssetsWrites(format::HandleId memory_id) for (const auto& entry : memories_page_status) { - assert(entry.second.status_tracker.HasActiveWriteBlock()); +// assert(entry.second.status_tracker.HasActiveWriteBlock()); const util::PageStatusTracker& page_status = entry.second.status_tracker; diff --git a/framework/encode/vulkan_state_tracker.h b/framework/encode/vulkan_state_tracker.h index 05f7056d5a..4979ee124f 100644 --- a/framework/encode/vulkan_state_tracker.h +++ b/framework/encode/vulkan_state_tracker.h @@ -454,6 +454,9 @@ class VulkanStateTracker const VkAccelerationStructureBuildGeometryInfoKHR* infos, const VkAccelerationStructureBuildRangeInfoKHR* const* pp_buildRange_infos); + void TrackAccelerationStructureCopyCommand(VkCommandBuffer command_buffer, + const VkCopyAccelerationStructureInfoKHR* info); + void TrackDeviceMemoryDeviceAddress(VkDevice device, VkDeviceMemory memory, VkDeviceAddress address); void TrackRayTracingPipelineProperties(VkPhysicalDevice physicalDevice, diff --git a/framework/encode/vulkan_state_writer.cpp b/framework/encode/vulkan_state_writer.cpp index 271497719e..0a0c3b2302 100644 --- a/framework/encode/vulkan_state_writer.cpp +++ b/framework/encode/vulkan_state_writer.cpp @@ -1668,7 +1668,7 @@ void VulkanStateWriter::WriteAccelerationStructureStateMetaCommands(const Vulkan std::vector blas_build; std::vector tlas_build; std::vector write_properties; - AccelerationStructureCopyCommandData copies; + std::vector copy_infos; std::vector blas_update; std::vector tlas_update; }; @@ -1714,7 +1714,7 @@ void VulkanStateWriter::WriteAccelerationStructureStateMetaCommands(const Vulkan if (wrapper->latest_copy_command_) { - per_device_container.copies.infos.push_back(wrapper->latest_copy_command_.value().info); + per_device_container.copy_infos.push_back(wrapper->latest_copy_command_.value().info); } if (wrapper->latest_write_properties_command_) @@ -1738,7 +1738,10 @@ void VulkanStateWriter::WriteAccelerationStructureStateMetaCommands(const Vulkan EncodeAccelerationStructureWritePropertiesCommand(device, cmd_properties); } - EncodeAccelerationStructureCopyMetaCommand(device, command.copies); + for (const auto& copy_info : command.copy_infos) + { + EncodeAccelerationStructureCopyMetaCommand(device, copy_info); + } for (auto& tlas_build : command.tlas_build) { @@ -1868,8 +1871,8 @@ void VulkanStateWriter::EncodeAccelerationStructureBuildMetaCommand( ++blocks_written_; } -void VulkanStateWriter::EncodeAccelerationStructureCopyMetaCommand(format::HandleId device_id, - const AccelerationStructureCopyCommandData& command) +void VulkanStateWriter::EncodeAccelerationStructureCopyMetaCommand(format::HandleId device_id, + const VkCopyAccelerationStructureInfoKHR& info) { parameter_stream_.Clear(); @@ -1880,7 +1883,7 @@ void VulkanStateWriter::EncodeAccelerationStructureCopyMetaCommand(format::Handl format::ApiFamilyId::ApiFamily_Vulkan, format::MetaDataType::kVulkanCopyAccelerationStructuresCommand); encoder_.EncodeHandleIdValue(device_id); - EncodeStructArray(&encoder_, command.infos.data(), command.infos.size()); + EncodeStruct(&encoder_, info); header.meta_header.block_header.size += parameter_stream_.GetDataSize(); diff --git a/framework/encode/vulkan_state_writer.h b/framework/encode/vulkan_state_writer.h index 2e53a2f351..573c9115c0 100644 --- a/framework/encode/vulkan_state_writer.h +++ b/framework/encode/vulkan_state_writer.h @@ -383,12 +383,8 @@ class VulkanStateWriter void EncodeAccelerationStructureBuildMetaCommand(format::HandleId device_id, const AccelerationStructureBuildCommandData& command); - struct AccelerationStructureCopyCommandData - { - std::vector infos; - }; - void EncodeAccelerationStructureCopyMetaCommand(format::HandleId device_id, - const AccelerationStructureCopyCommandData& command); + void EncodeAccelerationStructureCopyMetaCommand(format::HandleId device_id, + const VkCopyAccelerationStructureInfoKHR& info); struct AccelerationStructureWritePropertiesCommandData { diff --git a/framework/generated/generated_vulkan_api_call_encoders.cpp b/framework/generated/generated_vulkan_api_call_encoders.cpp index faa5fe4b27..a08a4d49c0 100644 --- a/framework/generated/generated_vulkan_api_call_encoders.cpp +++ b/framework/generated/generated_vulkan_api_call_encoders.cpp @@ -25774,10 +25774,7 @@ VKAPI_ATTR void VKAPI_CALL CmdCopyAccelerationStructureKHR( manager->EndCommandApiCallCapture(commandBuffer, TrackCmdCopyAccelerationStructureKHRHandles, pInfo); } - auto handle_unwrap_memory = manager->GetHandleUnwrapMemory(); - const VkCopyAccelerationStructureInfoKHR* pInfo_unwrapped = vulkan_wrappers::UnwrapStructPtrHandles(pInfo, handle_unwrap_memory); - - vulkan_wrappers::GetDeviceTable(commandBuffer)->CmdCopyAccelerationStructureKHR(commandBuffer, pInfo_unwrapped); + manager->OverrideCmdCopyAccelerationStructureKHR(commandBuffer, pInfo); CustomEncoderPostCall::Dispatch(manager, commandBuffer, pInfo); } diff --git a/framework/generated/khronos_generators/vulkan_generators/capture_overrides.json b/framework/generated/khronos_generators/vulkan_generators/capture_overrides.json index dfe43ff1a7..e16a0084f1 100644 --- a/framework/generated/khronos_generators/vulkan_generators/capture_overrides.json +++ b/framework/generated/khronos_generators/vulkan_generators/capture_overrides.json @@ -13,6 +13,7 @@ "vkGetPhysicalDeviceQueueFamilyProperties2": "manager->OverrideGetPhysicalDeviceQueueFamilyProperties2", "vkGetPhysicalDeviceQueueFamilyProperties2KHR": "manager->OverrideGetPhysicalDeviceQueueFamilyProperties2KHR", "vkCmdBuildAccelerationStructuresKHR": "manager->OverrideCmdBuildAccelerationStructuresKHR", + "vkCmdCopyAccelerationStructureKHR": "manager->OverrideCmdCopyAccelerationStructureKHR", "vkGetPhysicalDeviceProperties2": "manager->OverrideGetPhysicalDeviceProperties2", "vkGetPhysicalDeviceProperties2KHR": "manager->OverrideGetPhysicalDeviceProperties2" } From 1f24751563da8d457ca3dbc3409327bc5025897e Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Mon, 13 Jan 2025 16:08:16 +0100 Subject: [PATCH 13/23] work on ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand --- framework/decode/vulkan_address_replacer.cpp | 37 +++++++++++++++++-- framework/decode/vulkan_address_replacer.h | 9 ++++- .../decode/vulkan_replay_consumer_base.cpp | 3 +- framework/encode/vulkan_capture_manager.cpp | 19 ++++++++++ framework/encode/vulkan_capture_manager.h | 7 ++++ framework/encode/vulkan_state_tracker.cpp | 22 +++++++++++ framework/encode/vulkan_state_tracker.h | 7 ++++ .../generated_vulkan_api_call_encoders.cpp | 2 +- .../vulkan_generators/capture_overrides.json | 1 + 9 files changed, 101 insertions(+), 6 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 55c78d69a9..4c8d1a2366 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -221,6 +221,10 @@ VulkanAddressReplacer::~VulkanAddressReplacer() _device_table->DestroyPipelineLayout(_device, _pipeline_layout, nullptr); } + if (_query_pool != VK_NULL_HANDLE) + { + _device_table->DestroyQueryPool(_device, _query_pool, nullptr); + } if (_fence != VK_NULL_HANDLE) { _device_table->DestroyFence(_device, _fence, nullptr); @@ -792,7 +796,6 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR( GFXRECON_ASSERT(info->dst != VK_NULL_HANDLE); - // // tmp -> we don't arrive here during trim!? // auto replace_it = _shadow_as_map.find(info->dst); // if (replace_it == _shadow_as_map.end()) // { @@ -818,7 +821,11 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR( } void VulkanAddressReplacer::ProcessCmdWriteAccelerationStructuresPropertiesKHR( - uint32_t count, VkAccelerationStructureKHR* acceleration_structures) + uint32_t count, + VkAccelerationStructureKHR* acceleration_structures, + VkQueryType query_type, + VkQueryPool pool, + uint32_t first_query) { for (uint32_t i = 0; i < count; ++i) { @@ -884,6 +891,7 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( &command_buffer_info, info_count, geometry_infos, range_infos, address_tracker); // issue build-command + mark_injected_commands_helper_t mark_injected_commands_helper; _device_table->CmdBuildAccelerationStructuresKHR(_command_buffer, info_count, geometry_infos, range_infos); } } @@ -902,6 +910,7 @@ void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( { ProcessCmdCopyAccelerationStructuresKHR(copy_infos + i, address_tracker); } + mark_injected_commands_helper_t mark_injected_commands_helper; _device_table->CmdCopyAccelerationStructureKHR(_command_buffer, copy_infos); } } @@ -911,7 +920,15 @@ void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMe { if (init_queue_assets()) { - // TODO: reset/submit/sync command-buffer + // reset/submit/sync command-buffer + queue_submit_helper_t queue_submit_helper(_device_table, _device, _command_buffer, _queue, _fence); + + ProcessCmdWriteAccelerationStructuresPropertiesKHR(1, &acceleration_structure, query_type, _query_pool, 0); + + // issue vkCmdWriteAccelerationStructuresPropertiesKHR + mark_injected_commands_helper_t mark_injected_commands_helper; + _device_table->CmdWriteAccelerationStructuresPropertiesKHR( + _command_buffer, 1, &acceleration_structure, query_type, _query_pool, 0); } } @@ -1047,6 +1064,20 @@ bool VulkanAddressReplacer::init_queue_assets() return false; } + VkQueryPoolCreateInfo pool_info; + pool_info.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO; + pool_info.pNext = nullptr; + pool_info.flags = 0; + pool_info.queryType = VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR; + pool_info.queryCount = 1; + pool_info.pipelineStatistics = 0; + result = _device_table->CreateQueryPool(_device, &pool_info, nullptr, &_query_pool); + if (result != VK_SUCCESS) + { + GFXRECON_LOG_ERROR("VulkanAddressReplacer: internal query-pool creation failed"); + return false; + } + _device_table->GetDeviceQueue(_device, 0, 0, &_queue); GFXRECON_ASSERT(_queue != VK_NULL_HANDLE); return _queue != VK_NULL_HANDLE; diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index fb373a80f0..46c20e20db 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -130,9 +130,15 @@ class VulkanAddressReplacer * * @param count element count in acceleration_structures * @param acceleration_structures provided array of VkAccelerationStructureKHR-handles + * @param query_type the query's type + * @param pool provided VkQuerypool handle + * @param first_query index of first query */ void ProcessCmdWriteAccelerationStructuresPropertiesKHR(uint32_t count, - VkAccelerationStructureKHR* acceleration_structures); + VkAccelerationStructureKHR* acceleration_structures, + VkQueryType query_type, + VkQueryPool pool, + uint32_t first_query); /** * @brief ProcessUpdateDescriptorSets will check @@ -278,6 +284,7 @@ class VulkanAddressReplacer VkCommandBuffer _command_buffer = VK_NULL_HANDLE; VkFence _fence = VK_NULL_HANDLE; VkQueue _queue = VK_NULL_HANDLE; + VkQueryPool _query_pool = VK_NULL_HANDLE; util::linear_hashmap _hashmap_sbt; util::linear_hashmap _hashmap_bda; diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index b895226379..e27a08ed4c 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -7839,7 +7839,8 @@ void VulkanReplayConsumerBase::OverrideCmdWriteAccelerationStructuresPropertiesK { auto& address_replacer = GetDeviceAddressReplacer(device_info); - address_replacer.ProcessCmdWriteAccelerationStructuresPropertiesKHR(count, acceleration_structs); + address_replacer.ProcessCmdWriteAccelerationStructuresPropertiesKHR( + count, acceleration_structs, queryType, query_pool, firstQuery); } func(command_buffer, count, acceleration_structs, queryType, query_pool, firstQuery); } diff --git a/framework/encode/vulkan_capture_manager.cpp b/framework/encode/vulkan_capture_manager.cpp index b0be82eb09..b211fe249b 100644 --- a/framework/encode/vulkan_capture_manager.cpp +++ b/framework/encode/vulkan_capture_manager.cpp @@ -966,6 +966,25 @@ void VulkanCaptureManager::OverrideCmdCopyAccelerationStructureKHR(VkCommandBuff device_table->CmdCopyAccelerationStructureKHR(command_buffer, pInfo); } +void VulkanCaptureManager::OverrideCmdWriteAccelerationStructuresPropertiesKHR( + VkCommandBuffer commandBuffer, + uint32_t accelerationStructureCount, + const VkAccelerationStructureKHR* pAccelerationStructures, + VkQueryType queryType, + VkQueryPool queryPool, + uint32_t firstQuery) +{ + if (IsCaptureModeTrack()) + { + state_tracker_->TrackWriteAccelerationStructuresPropertiesCommand( + commandBuffer, accelerationStructureCount, pAccelerationStructures, queryType, queryPool, firstQuery); + } + + const VulkanDeviceTable* device_table = vulkan_wrappers::GetDeviceTable(commandBuffer); + device_table->CmdWriteAccelerationStructuresPropertiesKHR( + commandBuffer, accelerationStructureCount, pAccelerationStructures, queryType, queryPool, firstQuery); +} + VkResult VulkanCaptureManager::OverrideAllocateMemory(VkDevice device, const VkMemoryAllocateInfo* pAllocateInfo, const VkAllocationCallbacks* pAllocator, diff --git a/framework/encode/vulkan_capture_manager.h b/framework/encode/vulkan_capture_manager.h index e345013f7f..50e69c4af9 100644 --- a/framework/encode/vulkan_capture_manager.h +++ b/framework/encode/vulkan_capture_manager.h @@ -301,6 +301,13 @@ class VulkanCaptureManager : public ApiCaptureManager void OverrideCmdCopyAccelerationStructureKHR(VkCommandBuffer command_buffer, const VkCopyAccelerationStructureInfoKHR* pInfo); + void OverrideCmdWriteAccelerationStructuresPropertiesKHR(VkCommandBuffer commandBuffer, + uint32_t accelerationStructureCount, + const VkAccelerationStructureKHR* pAccelerationStructures, + VkQueryType queryType, + VkQueryPool queryPool, + uint32_t firstQuery); + VkResult OverrideAllocateMemory(VkDevice device, const VkMemoryAllocateInfo* pAllocateInfo, const VkAllocationCallbacks* pAllocator, diff --git a/framework/encode/vulkan_state_tracker.cpp b/framework/encode/vulkan_state_tracker.cpp index d8b373ade3..3fc2e1cd3a 100644 --- a/framework/encode/vulkan_state_tracker.cpp +++ b/framework/encode/vulkan_state_tracker.cpp @@ -555,6 +555,28 @@ void VulkanStateTracker::TrackAccelerationStructureCopyCommand(VkCommandBuffer wrapper->latest_copy_command_->info = *info; } +void VulkanStateTracker::TrackWriteAccelerationStructuresPropertiesCommand( + VkCommandBuffer commandBuffer, + uint32_t accelerationStructureCount, + const VkAccelerationStructureKHR* pAccelerationStructures, + VkQueryType queryType, + VkQueryPool queryPool, + uint32_t firstQuery) +{ + auto* cmd_buf_wrapper = vulkan_wrappers::GetWrapper(commandBuffer); + auto* query_pool_wrapper = vulkan_wrappers::GetWrapper(queryPool); + + for (uint32_t i = 0; i < accelerationStructureCount; ++i) + { + auto* wrapper = + vulkan_wrappers::GetWrapper(pAccelerationStructures[i]); + wrapper->latest_write_properties_command_ = + vulkan_wrappers::AccelerationStructureKHRWrapper::AccelerationStructureWritePropertiesCommandData{}; + wrapper->latest_write_properties_command_->device = wrapper->device->handle_id; + wrapper->latest_write_properties_command_->query_type = queryType; + } +} + void VulkanStateTracker::TrackImageMemoryBinding( VkDevice device, VkImage image, VkDeviceMemory memory, VkDeviceSize memoryOffset, const void* bind_info_pnext) { diff --git a/framework/encode/vulkan_state_tracker.h b/framework/encode/vulkan_state_tracker.h index 4979ee124f..652fe5f7e1 100644 --- a/framework/encode/vulkan_state_tracker.h +++ b/framework/encode/vulkan_state_tracker.h @@ -457,6 +457,13 @@ class VulkanStateTracker void TrackAccelerationStructureCopyCommand(VkCommandBuffer command_buffer, const VkCopyAccelerationStructureInfoKHR* info); + void TrackWriteAccelerationStructuresPropertiesCommand(VkCommandBuffer commandBuffer, + uint32_t accelerationStructureCount, + const VkAccelerationStructureKHR* pAccelerationStructures, + VkQueryType queryType, + VkQueryPool queryPool, + uint32_t firstQuery); + void TrackDeviceMemoryDeviceAddress(VkDevice device, VkDeviceMemory memory, VkDeviceAddress address); void TrackRayTracingPipelineProperties(VkPhysicalDevice physicalDevice, diff --git a/framework/generated/generated_vulkan_api_call_encoders.cpp b/framework/generated/generated_vulkan_api_call_encoders.cpp index a08a4d49c0..8c67208945 100644 --- a/framework/generated/generated_vulkan_api_call_encoders.cpp +++ b/framework/generated/generated_vulkan_api_call_encoders.cpp @@ -25926,7 +25926,7 @@ VKAPI_ATTR void VKAPI_CALL CmdWriteAccelerationStructuresPropertiesKHR( manager->EndCommandApiCallCapture(commandBuffer, TrackCmdWriteAccelerationStructuresPropertiesKHRHandles, accelerationStructureCount, pAccelerationStructures, queryPool); } - vulkan_wrappers::GetDeviceTable(commandBuffer)->CmdWriteAccelerationStructuresPropertiesKHR(commandBuffer, accelerationStructureCount, pAccelerationStructures, queryType, queryPool, firstQuery); + manager->OverrideCmdWriteAccelerationStructuresPropertiesKHR(commandBuffer, accelerationStructureCount, pAccelerationStructures, queryType, queryPool, firstQuery); CustomEncoderPostCall::Dispatch(manager, commandBuffer, accelerationStructureCount, pAccelerationStructures, queryType, queryPool, firstQuery); } diff --git a/framework/generated/khronos_generators/vulkan_generators/capture_overrides.json b/framework/generated/khronos_generators/vulkan_generators/capture_overrides.json index e16a0084f1..946e1266d3 100644 --- a/framework/generated/khronos_generators/vulkan_generators/capture_overrides.json +++ b/framework/generated/khronos_generators/vulkan_generators/capture_overrides.json @@ -14,6 +14,7 @@ "vkGetPhysicalDeviceQueueFamilyProperties2KHR": "manager->OverrideGetPhysicalDeviceQueueFamilyProperties2KHR", "vkCmdBuildAccelerationStructuresKHR": "manager->OverrideCmdBuildAccelerationStructuresKHR", "vkCmdCopyAccelerationStructureKHR": "manager->OverrideCmdCopyAccelerationStructureKHR", + "vkCmdWriteAccelerationStructuresPropertiesKHR": "manager->OverrideCmdWriteAccelerationStructuresPropertiesKHR", "vkGetPhysicalDeviceProperties2": "manager->OverrideGetPhysicalDeviceProperties2", "vkGetPhysicalDeviceProperties2KHR": "manager->OverrideGetPhysicalDeviceProperties2" } From 71260df68c5b1d1744ba97fb0772148312345b7c Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 14 Jan 2025 09:49:27 +0100 Subject: [PATCH 14/23] Overall progress and cleanup. some open things, but slowly finalizing --- framework/decode/vulkan_address_replacer.cpp | 78 ++++++++++++++++--- framework/decode/vulkan_address_replacer.h | 19 ++++- .../decode/vulkan_replay_consumer_base.cpp | 3 + .../decode/vulkan_replay_consumer_base.h | 2 +- framework/encode/vulkan_state_writer.cpp | 15 +--- framework/encode/vulkan_state_writer.h | 4 +- 6 files changed, 95 insertions(+), 26 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 4c8d1a2366..977699a577 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -512,8 +512,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } else { - GFXRECON_LOG_WARNING( - "ProcessCmdBuildAccelerationStructuresKHR: missing buffer_info->replay_address, remap failed"); +// GFXRECON_LOG_WARNING( +// "ProcessCmdBuildAccelerationStructuresKHR: missing buffer_info->replay_address, remap failed"); return false; } }; @@ -529,10 +529,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( address_tracker.GetBufferByCaptureDeviceAddress(build_geometry_info.scratchData.deviceAddress); // check/correct scratch-address - if (!address_remap(build_geometry_info.scratchData.deviceAddress)) - { - GFXRECON_LOG_WARNING("ProcessCmdBuildAccelerationStructuresKHR: missing scratch-buffer"); - } + address_remap(build_geometry_info.scratchData.deviceAddress); // check capture/replay acceleration-structure buffer-sizes if (!_resource_allocator->SupportsOpaqueDeviceAddresses()) @@ -794,8 +791,22 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR( swap_acceleration_structure(info->src); swap_acceleration_structure(info->dst); - GFXRECON_ASSERT(info->dst != VK_NULL_HANDLE); + VkDeviceSize compact_size = 0; + // GFXRECON_ASSERT(info->dst != VK_NULL_HANDLE); + auto compact_size_it = _as_compact_sizes.find(info->dst); + if (compact_size_it != _as_compact_sizes.end()) + { + compact_size = compact_size_it->second; + GFXRECON_LOG_INFO( + "VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR: found compacted AS-size: %ul", + compact_size); + } + // else + // { + // GFXRECON_LOG_ERROR( + // "VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR: compacted AS-size unknown"); + // } // auto replace_it = _shadow_as_map.find(info->dst); // if (replace_it == _shadow_as_map.end()) // { @@ -834,6 +845,12 @@ void VulkanAddressReplacer::ProcessCmdWriteAccelerationStructuresPropertiesKHR( { acceleration_structures[i] = shadow_as_it->second.handle; } + + if (query_type == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR) + { + // read back compacted size later + _as_compact_queries[pool][acceleration_structures[i]] = first_query + i; + } } } @@ -908,10 +925,21 @@ void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( for (uint32_t i = 0; i < info_count; ++i) { - ProcessCmdCopyAccelerationStructuresKHR(copy_infos + i, address_tracker); + auto* copy_info = copy_infos + i; + + if (copy_info->src != VK_NULL_HANDLE && copy_info->dst != VK_NULL_HANDLE) + { + ProcessCmdCopyAccelerationStructuresKHR(copy_info, address_tracker); + + // issue copy command + mark_injected_commands_helper_t mark_injected_commands_helper; + _device_table->CmdCopyAccelerationStructureKHR(_command_buffer, copy_info); + } + else + { + GFXRECON_LOG_ERROR("ProcessCopyVulkanAccelerationStructuresMetaCommand: missing handles"); + } } - mark_injected_commands_helper_t mark_injected_commands_helper; - _device_table->CmdCopyAccelerationStructureKHR(_command_buffer, copy_infos); } } @@ -932,6 +960,36 @@ void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMe } } +void VulkanAddressReplacer::ProcessGetQueryPoolResults(VkDevice device, + VkQueryPool query_pool, + uint32_t firstQuery, + uint32_t queryCount, + size_t dataSize, + void* pData, + VkDeviceSize stride, + VkQueryResultFlags flags) +{ + // intercept queries containing acceleration-structure compaction-sizes + // if (!_as_compact_queries.empty()) + { + bool is_synced = flags & VK_QUERY_RESULT_WAIT_BIT; + + auto it = _as_compact_queries.find(query_pool); + if (is_synced && it != _as_compact_queries.end()) + { + // assuming post-processing here, pData was already written + auto* result_array = reinterpret_cast(pData); + + for (const auto& [as, query_index] : it->second) + { + GFXRECON_LOG_INFO("query-index %d: %d", query_index, result_array[query_index]); + _as_compact_sizes[as] = result_array[query_index]; + } + } + _as_compact_queries.erase(query_pool); + } +} + bool VulkanAddressReplacer::init_pipeline() { if (_pipeline_sbt != VK_NULL_HANDLE) diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 46c20e20db..325f8f8e5a 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -181,15 +181,26 @@ class VulkanAddressReplacer VkCopyAccelerationStructureInfoKHR* copy_infos, const decode::VulkanDeviceAddressTracker& address_tracker); /** - * @brief Process information contained in a metadata-block in order to write information in a query-pool. + * @brief Process information contained in a metadata-block in order to issue a query on internal an query-pool. * - * @param query_type element count in 'copy_infos' + * Will use an internal command-buffer, submit work to a VkQueue and perform host-synchronization. + * + * @param query_type type of query * @param acceleration_structure provided acceleration-structure handle */ void ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand(VkQueryType query_type, VkAccelerationStructureKHR acceleration_structure); + void ProcessGetQueryPoolResults(VkDevice device, + VkQueryPool query_pool, + uint32_t firstQuery, + uint32_t queryCount, + size_t dataSize, + void* pData, + VkDeviceSize stride, + VkQueryResultFlags flags); + /** * @brief DestroyShadowResources should be called upon destruction of provided VkAccelerationStructureKHR handle, * allowing this class to free potential resources associated with it. @@ -299,6 +310,10 @@ class VulkanAddressReplacer // pipeline-contexts dealing with acceleration-structure builds, per command-buffer std::unordered_map _build_as_context_map; + // currently running compaction queries. pool -> AS -> query-pool-index + std::unordered_map> _as_compact_queries; + std::unordered_map _as_compact_sizes; + // required function pointers PFN_vkGetBufferDeviceAddress _get_device_address_fn_ = nullptr; }; diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index e27a08ed4c..3d72341804 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -3516,6 +3516,9 @@ VkResult VulkanReplayConsumerBase::OverrideGetQueryPoolResults(PFN_vkGetQueryPoo } while (((original_result == VK_SUCCESS) && (result == VK_NOT_READY)) && (++retries <= kMaxQueryPoolResultsRetries)); + auto& address_replacer = GetDeviceAddressReplacer(device_info); + address_replacer.ProcessGetQueryPoolResults( + device, query_pool, firstQuery, queryCount, dataSize, pData->GetPointer(), stride, flags); return result; } diff --git a/framework/decode/vulkan_replay_consumer_base.h b/framework/decode/vulkan_replay_consumer_base.h index b109bca9bc..15d38a3258 100644 --- a/framework/decode/vulkan_replay_consumer_base.h +++ b/framework/decode/vulkan_replay_consumer_base.h @@ -208,7 +208,7 @@ class VulkanReplayConsumerBase : public VulkanConsumer StructPointerDecoder* ppRangeInfos) override; void ProcessCopyVulkanAccelerationStructuresMetaCommand( - format::HandleId device, StructPointerDecoder* copy_infos) override; + format::HandleId device, StructPointerDecoder* copy_info) override; void ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand( format::HandleId device_id, VkQueryType query_type, format::HandleId acceleration_structure_id) override; diff --git a/framework/encode/vulkan_state_writer.cpp b/framework/encode/vulkan_state_writer.cpp index 0a0c3b2302..4a52fa1738 100644 --- a/framework/encode/vulkan_state_writer.cpp +++ b/framework/encode/vulkan_state_writer.cpp @@ -1660,7 +1660,6 @@ void VulkanStateWriter::WriteTlasToBlasDependenciesMetadata(const VulkanStateTab }); } -// Rename this to represent the whole acc structure prepare process void VulkanStateWriter::WriteAccelerationStructureStateMetaCommands(const VulkanStateTable& state_table) { struct AccelerationStructureCommands @@ -1738,10 +1737,7 @@ void VulkanStateWriter::WriteAccelerationStructureStateMetaCommands(const Vulkan EncodeAccelerationStructureWritePropertiesCommand(device, cmd_properties); } - for (const auto& copy_info : command.copy_infos) - { - EncodeAccelerationStructureCopyMetaCommand(device, copy_info); - } + EncodeAccelerationStructuresCopyMetaCommand(device, command.copy_infos); for (auto& tlas_build : command.tlas_build) { @@ -1871,8 +1867,8 @@ void VulkanStateWriter::EncodeAccelerationStructureBuildMetaCommand( ++blocks_written_; } -void VulkanStateWriter::EncodeAccelerationStructureCopyMetaCommand(format::HandleId device_id, - const VkCopyAccelerationStructureInfoKHR& info) +void VulkanStateWriter::EncodeAccelerationStructuresCopyMetaCommand( + format::HandleId device_id, const std::vector& infos) { parameter_stream_.Clear(); @@ -1883,15 +1879,12 @@ void VulkanStateWriter::EncodeAccelerationStructureCopyMetaCommand(format::Handl format::ApiFamilyId::ApiFamily_Vulkan, format::MetaDataType::kVulkanCopyAccelerationStructuresCommand); encoder_.EncodeHandleIdValue(device_id); - EncodeStruct(&encoder_, info); - + EncodeStructArray(&encoder_, infos.data(), infos.size()); header.meta_header.block_header.size += parameter_stream_.GetDataSize(); output_stream_->Write(&header, sizeof(header)); output_stream_->Write(parameter_stream_.GetData(), parameter_stream_.GetDataSize()); - parameter_stream_.Clear(); - ++blocks_written_; } diff --git a/framework/encode/vulkan_state_writer.h b/framework/encode/vulkan_state_writer.h index 573c9115c0..1070d12621 100644 --- a/framework/encode/vulkan_state_writer.h +++ b/framework/encode/vulkan_state_writer.h @@ -383,8 +383,8 @@ class VulkanStateWriter void EncodeAccelerationStructureBuildMetaCommand(format::HandleId device_id, const AccelerationStructureBuildCommandData& command); - void EncodeAccelerationStructureCopyMetaCommand(format::HandleId device_id, - const VkCopyAccelerationStructureInfoKHR& info); + void EncodeAccelerationStructuresCopyMetaCommand(format::HandleId device_id, + const std::vector& infos); struct AccelerationStructureWritePropertiesCommandData { From 2b1bafa5c2a23395b90462be6ec8dd1c7717a04b Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 14 Jan 2025 10:19:40 +0100 Subject: [PATCH 15/23] bring back opaque shadergroup-handles, you never know --- framework/decode/vulkan_address_replacer.cpp | 42 +--- framework/decode/vulkan_rebind_allocator.cpp | 5 +- .../decode/vulkan_replay_consumer_base.cpp | 206 ++++++++++++++---- framework/encode/vulkan_state_tracker.cpp | 6 +- framework/graphics/test/main.cpp | 1 + framework/graphics/vulkan_device_util.cpp | 9 +- 6 files changed, 183 insertions(+), 86 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 977699a577..9cdb0022c2 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -512,8 +512,9 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } else { -// GFXRECON_LOG_WARNING( -// "ProcessCmdBuildAccelerationStructuresKHR: missing buffer_info->replay_address, remap failed"); + // GFXRECON_LOG_WARNING( + // "ProcessCmdBuildAccelerationStructuresKHR: missing buffer_info->replay_address, remap + // failed"); return false; } }; @@ -532,7 +533,6 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( address_remap(build_geometry_info.scratchData.deviceAddress); // check capture/replay acceleration-structure buffer-sizes - if (!_resource_allocator->SupportsOpaqueDeviceAddresses()) { VkAccelerationStructureBuildSizesInfoKHR build_size_info = {}; build_size_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR; @@ -594,7 +594,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( build_size_info.accelerationStructureSize, scratch_size)) { - // problem creating shadow-AS + GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: creation of shadow " + "acceleration-structure failed"); } } @@ -791,43 +792,16 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR( swap_acceleration_structure(info->src); swap_acceleration_structure(info->dst); - VkDeviceSize compact_size = 0; - // GFXRECON_ASSERT(info->dst != VK_NULL_HANDLE); - auto compact_size_it = _as_compact_sizes.find(info->dst); + VkDeviceSize compact_size = 0; + auto compact_size_it = _as_compact_sizes.find(info->dst); if (compact_size_it != _as_compact_sizes.end()) { compact_size = compact_size_it->second; - + _as_compact_sizes.erase(info->dst); GFXRECON_LOG_INFO( "VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR: found compacted AS-size: %ul", compact_size); } - // else - // { - // GFXRECON_LOG_ERROR( - // "VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR: compacted AS-size unknown"); - // } - // auto replace_it = _shadow_as_map.find(info->dst); - // if (replace_it == _shadow_as_map.end()) - // { - // if (info->dst != VK_NULL_HANDLE) - // { - // auto as_info = address_tracker.GetAccelerationStructureByHandle(info->dst); - // GFXRECON_ASSERT(as_info != nullptr); - // - // acceleration_structure_asset_t& new_dst = _shadow_as_map[info->dst]; - // uint32_t fake_as_buffer_size = 5 * (1 << 20); - // - // if (create_acceleration_asset(new_dst, as_info->type, fake_as_buffer_size, 0)) - // { - // swap_acceleration_structure(info->dst); - // } - // } - // else - // { - // GFXRECON_ASSERT(info->dst != VK_NULL_HANDLE); - // } - // } } } diff --git a/framework/decode/vulkan_rebind_allocator.cpp b/framework/decode/vulkan_rebind_allocator.cpp index 8328ccdd28..a3084829d1 100644 --- a/framework/decode/vulkan_rebind_allocator.cpp +++ b/framework/decode/vulkan_rebind_allocator.cpp @@ -249,14 +249,13 @@ VkResult VulkanRebindAllocator::CreateBuffer(const VkBufferCreateInfo* create if ((create_info != nullptr) && (buffer != nullptr) && (allocator_data != nullptr)) { - auto aligned_size = [](uint32_t size, uint32_t alignment) -> uint32_t - { + auto aligned_size = [](uint32_t size, uint32_t alignment) -> uint32_t { return (size + alignment - 1) & ~(alignment - 1); }; auto modified_info = *create_info; modified_info.size = aligned_size(create_info->size, min_buffer_alignment_); - result = functions_.create_buffer(device_, &modified_info, nullptr, buffer); + result = functions_.create_buffer(device_, &modified_info, nullptr, buffer); if (result >= 0) { diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index 3d72341804..e23a7f4103 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -7791,7 +7791,7 @@ void VulkanReplayConsumerBase::OverrideCmdBuildAccelerationStructuresKHR( VkAccelerationStructureBuildGeometryInfoKHR* build_geometry_infos = pInfos->GetPointer(); VkAccelerationStructureBuildRangeInfoKHR** build_range_infos = ppBuildRangeInfos->GetPointer(); -// if (!device_info->allocator->SupportsOpaqueDeviceAddresses()) + if (!device_info->allocator->SupportsOpaqueDeviceAddresses()) { auto& address_tracker = GetDeviceAddressTracker(device_info); auto& address_replacer = GetDeviceAddressReplacer(device_info); @@ -7892,38 +7892,157 @@ VkResult VulkanReplayConsumerBase::OverrideCreateRayTracingPipelinesKHR( &pPipelines->GetPointer()[createInfoCount]); } - if (omitted_pipeline_cache_data_) + // NOTE: as of early 2025, rayTracingPipelineShaderGroupHandleCaptureReplay is not widely supported. + // e.g. newest nvidia desktop-drivers do not support this feature + if (device_info->property_feature_info.feature_rayTracingPipelineShaderGroupHandleCaptureReplay) { - AllowCompileDuringPipelineCreation(createInfoCount, - const_cast(in_pCreateInfos)); - } + // Modify pipeline create infos with capture replay flag and data. + std::vector modified_create_infos; + std::vector> modified_pgroups; + modified_create_infos.reserve(createInfoCount); + modified_pgroups.resize(createInfoCount); + for (uint32_t create_info_i = 0; create_info_i < createInfoCount; ++create_info_i) + { + format::HandleId pipeline_capture_id = (*pPipelines[create_info_i].GetPointer()); - VkPipeline* created_pipelines = nullptr; + // Enable capture replay flag. + modified_create_infos.push_back(in_pCreateInfos[create_info_i]); + modified_create_infos[create_info_i].flags |= + VK_PIPELINE_CREATE_RAY_TRACING_SHADER_GROUP_HANDLE_CAPTURE_REPLAY_BIT_KHR; - if (deferred_operation_info) - { - created_pipelines = deferred_operation_info->replayPipelines.data(); + uint32_t group_info_count = in_pCreateInfos[create_info_i].groupCount; + bool has_data = (device_info->shader_group_handles.find(pipeline_capture_id) != + device_info->shader_group_handles.end()); + + if (has_data) + { + assert(device_info->shader_group_handles.at(pipeline_capture_id).size() == + (device_info->property_feature_info.property_shaderGroupHandleCaptureReplaySize * + group_info_count)); + } + else + { + GFXRECON_LOG_WARNING("Missing shader group handle data in for ray tracing pipeline (ID = %" PRIu64 ").", + pipeline_capture_id); + } + + // Set pShaderGroupCaptureReplayHandle in shader group create infos. + std::vector& modified_group_infos = modified_pgroups[create_info_i]; + modified_group_infos.reserve(group_info_count); + + for (uint32_t group_info_i = 0; group_info_i < group_info_count; ++group_info_i) + { + modified_group_infos.push_back(in_pCreateInfos[create_info_i].pGroups[group_info_i]); + + if (has_data) + { + uint32_t byte_offset = + device_info->property_feature_info.property_shaderGroupHandleCaptureReplaySize * group_info_i; + modified_group_infos[group_info_i].pShaderGroupCaptureReplayHandle = + device_info->shader_group_handles.at(pipeline_capture_id).data() + byte_offset; + } + else + { + modified_group_infos[group_info_i].pShaderGroupCaptureReplayHandle = nullptr; + } + } + + // Use modified shader group infos. + modified_create_infos[create_info_i].pGroups = modified_group_infos.data(); + } + + if (omitted_pipeline_cache_data_) + { + AllowCompileDuringPipelineCreation(createInfoCount, modified_create_infos.data()); + } + + VkPipeline* created_pipelines = nullptr; + + if (deferred_operation_info) + { + created_pipelines = deferred_operation_info->replayPipelines.data(); + } + else + { + created_pipelines = out_pPipelines; + } + + result = GetDeviceTable(device)->CreateRayTracingPipelinesKHR(device, + in_deferredOperation, + overridePipelineCache, + createInfoCount, + modified_create_infos.data(), + in_pAllocator, + created_pipelines); + + if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) || + (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)) + { + // The above return values mean the command is not deferred and driver will finish all workload in current + // thread. Therefore the created pipelines can be read and copied to out_pPipelines which will be + // referenced later. + // + // Note: + // Some pipelines might actually fail creation if the return value is VK_PIPELINE_COMPILE_REQUIRED_EXT. + // These failed pipelines will generate VK_NULL_HANDLE. + // + // If the return value is VK_OPERATION_DEFERRED_KHR, it means the command is deferred, and thus pipeline + // creation is not finished. Subsequent handling will be done by + // vkDeferredOperationJoinKHR/vkGetDeferredOperationResultKHR after pipeline creation is finished. + + if (deferred_operation_info) + { + memcpy(out_pPipelines, created_pipelines, createInfoCount * sizeof(VkPipeline)); + + // Eventhough vkCreateRayTracingPipelinesKHR was called with a valid deferred operation object, the + // driver may opt to not defer the command. In this case, set pending_state flag to false to skip + // vkDeferredOperationJoinKHR handling. + deferred_operation_info->pending_state = false; + } + } + + if (deferred_operation_info) + { + deferred_operation_info->record_modified_create_infos = std::move(modified_create_infos); + deferred_operation_info->record_modified_pgroups = std::move(modified_pgroups); + } } else { - created_pipelines = out_pPipelines; - } + if (omitted_pipeline_cache_data_) + { + AllowCompileDuringPipelineCreation(createInfoCount, + const_cast(in_pCreateInfos)); + } - result = GetDeviceTable(device)->CreateRayTracingPipelinesKHR(device, - in_deferredOperation, - overridePipelineCache, - createInfoCount, - in_pCreateInfos, - in_pAllocator, - created_pipelines); + VkPipeline* created_pipelines = nullptr; - if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) || - (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)) - { if (deferred_operation_info) { - memcpy(out_pPipelines, created_pipelines, createInfoCount * sizeof(VkPipeline)); - deferred_operation_info->pending_state = false; + created_pipelines = deferred_operation_info->replayPipelines.data(); + } + else + { + created_pipelines = out_pPipelines; + } + + result = GetDeviceTable(device)->CreateRayTracingPipelinesKHR(device, + in_deferredOperation, + overridePipelineCache, + createInfoCount, + in_pCreateInfos, + in_pAllocator, + created_pipelines); + + if ((result == VK_SUCCESS) || (result == VK_OPERATION_NOT_DEFERRED_KHR) || + (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)) + { + + if (deferred_operation_info) + { + memcpy(out_pPipelines, created_pipelines, createInfoCount * sizeof(VkPipeline)); + deferred_operation_info->pending_state = false; + } } } @@ -9537,12 +9656,15 @@ void VulkanReplayConsumerBase::ProcessCopyVulkanAccelerationStructuresMetaComman VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device); GFXRECON_ASSERT(device_info != nullptr); - MapStructArrayHandles(copy_infos->GetMetaStructPointer(), copy_infos->GetLength(), GetObjectInfoTable()); + if (!device_info->allocator->SupportsOpaqueDeviceAddresses()) + { + MapStructArrayHandles(copy_infos->GetMetaStructPointer(), copy_infos->GetLength(), GetObjectInfoTable()); - const auto& address_tracker = GetDeviceAddressTracker(device_info); - auto& address_replacer = GetDeviceAddressReplacer(device_info); - address_replacer.ProcessCopyVulkanAccelerationStructuresMetaCommand( - copy_infos->GetLength(), copy_infos->GetPointer(), address_tracker); + const auto& address_tracker = GetDeviceAddressTracker(device_info); + auto& address_replacer = GetDeviceAddressReplacer(device_info); + address_replacer.ProcessCopyVulkanAccelerationStructuresMetaCommand( + copy_infos->GetLength(), copy_infos->GetPointer(), address_tracker); + } } } @@ -9557,16 +9679,17 @@ void VulkanReplayConsumerBase::ProcessBuildVulkanAccelerationStructuresMetaComma VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device); GFXRECON_ASSERT(device_info != nullptr); - MapStructArrayHandles(pInfos->GetMetaStructPointer(), pInfos->GetLength(), GetObjectInfoTable()); + if (!device_info->allocator->SupportsOpaqueDeviceAddresses()) + { + MapStructArrayHandles(pInfos->GetMetaStructPointer(), pInfos->GetLength(), GetObjectInfoTable()); - VkAccelerationStructureBuildGeometryInfoKHR* build_geometry_infos = pInfos->GetPointer(); - VkAccelerationStructureBuildRangeInfoKHR** range_infos = ppRangeInfos->GetPointer(); + VkAccelerationStructureBuildGeometryInfoKHR* build_geometry_infos = pInfos->GetPointer(); + VkAccelerationStructureBuildRangeInfoKHR** range_infos = ppRangeInfos->GetPointer(); - GetDeviceAddressReplacer(device_info) - .ProcessBuildVulkanAccelerationStructuresMetaCommand(info_count, - pInfos->GetPointer(), - ppRangeInfos->GetPointer(), - GetDeviceAddressTracker(device_info)); + GetDeviceAddressReplacer(device_info) + .ProcessBuildVulkanAccelerationStructuresMetaCommand( + info_count, pInfos->GetPointer(), ppRangeInfos->GetPointer(), GetDeviceAddressTracker(device_info)); + } } } @@ -9578,11 +9701,14 @@ void VulkanReplayConsumerBase::ProcessVulkanAccelerationStructuresWritePropertie VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(device_id); GFXRECON_ASSERT(device_info != nullptr); - VkAccelerationStructureKHR acceleration_structure = MapHandle( - acceleration_structure_id, &VulkanObjectInfoTable::GetVkAccelerationStructureKHRInfo); + if (!device_info->allocator->SupportsOpaqueDeviceAddresses()) + { + VkAccelerationStructureKHR acceleration_structure = MapHandle( + acceleration_structure_id, &VulkanObjectInfoTable::GetVkAccelerationStructureKHRInfo); - GetDeviceAddressReplacer(device_info) - .ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand(query_type, acceleration_structure); + GetDeviceAddressReplacer(device_info) + .ProcessVulkanAccelerationStructuresWritePropertiesMetaCommand(query_type, acceleration_structure); + } } } diff --git a/framework/encode/vulkan_state_tracker.cpp b/framework/encode/vulkan_state_tracker.cpp index 3fc2e1cd3a..8406bb4955 100644 --- a/framework/encode/vulkan_state_tracker.cpp +++ b/framework/encode/vulkan_state_tracker.cpp @@ -2028,8 +2028,8 @@ void gfxrecon::encode::VulkanStateTracker::DestroyState(vulkan_wrappers::BufferW if (it != (*command)->input_buffers.end()) { vulkan_wrappers::AccelerationStructureKHRWrapper::ASInputBuffer& buffer = it->second; - buffer.destroyed = true; - auto [resource_util, created] = resource_utils_.try_emplace( + buffer.destroyed = true; + auto [resource_util, created] = resource_utils_.try_emplace( buffer.bind_device->handle, graphics::VulkanResourcesUtil(buffer.bind_device->handle, buffer.bind_device->physical_device->handle, @@ -2852,7 +2852,7 @@ void VulkanStateTracker::TrackMappedAssetsWrites(format::HandleId memory_id) for (const auto& entry : memories_page_status) { -// assert(entry.second.status_tracker.HasActiveWriteBlock()); + assert(entry.second.status_tracker.HasActiveWriteBlock()); const util::PageStatusTracker& page_status = entry.second.status_tracker; diff --git a/framework/graphics/test/main.cpp b/framework/graphics/test/main.cpp index 52b9cbed33..170bc886eb 100644 --- a/framework/graphics/test/main.cpp +++ b/framework/graphics/test/main.cpp @@ -24,6 +24,7 @@ /////////////////////////////////////////////////////////////////////////////// #define CATCH_CONFIG_MAIN +#include #include #include "graphics/vulkan_shader_group_handle.h" diff --git a/framework/graphics/vulkan_device_util.cpp b/framework/graphics/vulkan_device_util.cpp index 27cb2d1b42..eb20ff6111 100644 --- a/framework/graphics/vulkan_device_util.cpp +++ b/framework/graphics/vulkan_device_util.cpp @@ -203,8 +203,7 @@ VulkanDeviceUtil::EnableRequiredPhysicalDeviceFeatures(uint32_t rayTracingPipelineShaderGroupHandleCaptureReplay_original = rt_pipeline_features->rayTracingPipelineShaderGroupHandleCaptureReplay; - if (rt_pipeline_features->rayTracingPipeline && - !rt_pipeline_features->rayTracingPipelineShaderGroupHandleCaptureReplay) + if (rt_pipeline_features->rayTracingPipeline) { VkPhysicalDeviceRayTracingPipelineFeaturesKHR supported_features{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_TRACING_PIPELINE_FEATURES_KHR, nullptr @@ -212,13 +211,11 @@ VulkanDeviceUtil::EnableRequiredPhysicalDeviceFeatures(uint32_t GetPhysicalDeviceFeatures( instance_api_version, instance_table, physical_device, supported_features); - rt_pipeline_features->rayTracingPipelineShaderGroupHandleCaptureReplay = + result.feature_rayTracingPipelineShaderGroupHandleCaptureReplay = + rt_pipeline_features->rayTracingPipelineShaderGroupHandleCaptureReplay && supported_features.rayTracingPipelineShaderGroupHandleCaptureReplay; } - result.feature_rayTracingPipelineShaderGroupHandleCaptureReplay = - rt_pipeline_features->rayTracingPipelineShaderGroupHandleCaptureReplay; - // retrieve raytracing-pipeline-properties VkPhysicalDeviceRayTracingPipelinePropertiesKHR rt_properties{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_TRACING_PIPELINE_PROPERTIES_KHR, nullptr From 16e58772a487fb952effa725f03af500faa00d3d Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 14 Jan 2025 14:04:41 +0100 Subject: [PATCH 16/23] compute-shaders compiled against vulkan1.1 + optimized spirv --- framework/decode/vulkan_address_replacer.cpp | 24 +- .../decode/vulkan_address_replacer_shaders.h | 1250 ++++++----------- 2 files changed, 482 insertions(+), 792 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 9cdb0022c2..a1b24d88bf 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -932,6 +932,27 @@ void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMe _device_table->CmdWriteAccelerationStructuresPropertiesKHR( _command_buffer, 1, &acceleration_structure, query_type, _query_pool, 0); } + + VkDeviceSize compact_size = 0; + + // issue vkCmdWriteAccelerationStructuresPropertiesKHR + mark_injected_commands_helper_t mark_injected_commands_helper; + _device_table->GetQueryPoolResults(_device, + _query_pool, + 0, + 1, + sizeof(VkDeviceSize), + &compact_size, + sizeof(VkDeviceSize), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); + ProcessGetQueryPoolResults(_device, + _query_pool, + 0, + 1, + sizeof(VkDeviceSize), + &compact_size, + sizeof(VkDeviceSize), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); } void VulkanAddressReplacer::ProcessGetQueryPoolResults(VkDevice device, @@ -993,12 +1014,13 @@ bool VulkanAddressReplacer::init_pipeline() } auto create_pipeline = [this](VkPipelineLayout layout, const auto& spirv, VkPipeline& out_pipeline) -> VkResult { + using elem_t = typename std::decay_t::value_type; VkShaderModule compute_module = VK_NULL_HANDLE; VkShaderModuleCreateInfo shader_module_create_info = {}; shader_module_create_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; shader_module_create_info.pNext = VK_NULL_HANDLE; shader_module_create_info.flags = 0; - shader_module_create_info.codeSize = spirv.size(); + shader_module_create_info.codeSize = spirv.size() * sizeof(elem_t); shader_module_create_info.pCode = reinterpret_cast(spirv.data()); VkResult result = diff --git a/framework/decode/vulkan_address_replacer_shaders.h b/framework/decode/vulkan_address_replacer_shaders.h index 5c6771c226..f90b5aec46 100644 --- a/framework/decode/vulkan_address_replacer_shaders.h +++ b/framework/decode/vulkan_address_replacer_shaders.h @@ -287,799 +287,467 @@ void main() } #endif // g_replacer_bda_comp -static const std::array g_replacer_sbt_comp = { - 0x03, 0x02, 0x23, 0x07, 0x00, 0x05, 0x01, 0x00, 0x0b, 0x00, 0x08, 0x00, 0x1a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x11, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00, 0xb6, 0x14, 0x00, 0x00, 0x11, 0x00, - 0x02, 0x00, 0xe3, 0x14, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00, 0x47, 0x4c, 0x53, 0x4c, 0x2e, - 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x03, 0x00, 0xe4, 0x14, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x07, 0x00, 0x05, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6d, 0x61, 0x69, - 0x6e, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, 0x10, 0x00, 0x06, 0x00, 0x04, 0x00, - 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x03, 0x00, 0x02, 0x00, 0x00, 0x00, 0xcc, 0x01, 0x00, 0x00, 0x04, 0x00, 0x07, 0x00, 0x47, 0x4c, 0x5f, 0x45, - 0x58, 0x54, 0x5f, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x00, 0x04, 0x00, 0x08, 0x00, 0x47, 0x4c, 0x5f, 0x45, 0x58, 0x54, 0x5f, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, - 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x32, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x0a, 0x00, 0x47, - 0x4c, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x63, 0x70, 0x70, 0x5f, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x5f, - 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x00, 0x00, 0x04, 0x00, 0x08, - 0x00, 0x47, 0x4c, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x00, 0x05, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6d, - 0x61, 0x69, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x6d, 0x75, 0x72, 0x6d, - 0x75, 0x72, 0x5f, 0x33, 0x32, 0x5f, 0x73, 0x63, 0x72, 0x61, 0x6d, 0x62, 0x6c, 0x65, 0x28, 0x75, 0x31, 0x3b, 0x00, - 0x00, 0x05, 0x00, 0x03, 0x00, 0x09, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0x11, 0x00, - 0x00, 0x00, 0x6d, 0x75, 0x72, 0x6d, 0x75, 0x72, 0x33, 0x5f, 0x33, 0x32, 0x28, 0x75, 0x31, 0x5b, 0x38, 0x5d, 0x3b, - 0x75, 0x31, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x6b, 0x65, 0x79, 0x00, - 0x05, 0x00, 0x04, 0x00, 0x10, 0x00, 0x00, 0x00, 0x73, 0x65, 0x65, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, - 0x00, 0x13, 0x00, 0x00, 0x00, 0x73, 0x68, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, - 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x74, 0x00, 0x00, 0x00, 0x06, 0x00, 0x05, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x16, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x65, 0x71, 0x75, 0x61, 0x6c, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x73, 0x68, 0x61, 0x64, 0x65, 0x72, - 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x74, 0x2d, 0x75, 0x31, 0x5b, - 0x38, 0x5d, 0x31, 0x3b, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x73, 0x68, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x67, - 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x74, 0x2d, 0x75, 0x31, 0x5b, 0x38, 0x5d, - 0x31, 0x3b, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x16, 0x00, 0x00, 0x00, 0x6c, 0x68, 0x73, 0x00, 0x05, 0x00, 0x03, - 0x00, 0x17, 0x00, 0x00, 0x00, 0x72, 0x68, 0x73, 0x00, 0x05, 0x00, 0x0e, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x69, 0x73, - 0x5f, 0x6e, 0x75, 0x6c, 0x6c, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x73, 0x68, 0x61, 0x64, 0x65, 0x72, - 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x74, 0x2d, 0x75, 0x31, 0x5b, - 0x38, 0x5d, 0x31, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, - 0x00, 0x05, 0x00, 0x0d, 0x00, 0x20, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x2d, 0x73, 0x68, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, 0x64, - 0x6c, 0x65, 0x5f, 0x74, 0x2d, 0x75, 0x31, 0x5b, 0x38, 0x5d, 0x31, 0x3b, 0x00, 0x00, 0x00, 0x05, 0x00, 0x06, 0x00, - 0x1f, 0x00, 0x00, 0x00, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x00, 0x00, 0x00, - 0x00, 0x05, 0x00, 0x05, 0x00, 0x23, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, 0x70, 0x5f, 0x74, 0x00, - 0x00, 0x00, 0x06, 0x00, 0x05, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x00, 0x06, 0x00, 0x05, 0x00, 0x23, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x73, 0x69, 0x7a, 0x65, - 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, 0x23, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x63, 0x61, 0x70, - 0x61, 0x63, 0x69, 0x74, 0x79, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0x25, 0x00, 0x00, 0x00, 0x73, 0x68, - 0x61, 0x64, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x74, - 0x00, 0x00, 0x00, 0x06, 0x00, 0x05, 0x00, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x61, 0x74, 0x61, - 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x06, 0x00, 0x26, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, 0x70, - 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x74, 0x00, 0x00, 0x06, 0x00, 0x04, 0x00, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x6b, 0x65, 0x79, 0x00, 0x06, 0x00, 0x05, 0x00, 0x26, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, 0x06, 0x00, 0x28, 0x00, 0x00, 0x00, 0x48, 0x61, 0x73, 0x68, - 0x4d, 0x61, 0x70, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x00, 0x00, 0x06, 0x00, 0x04, 0x00, 0x28, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x05, 0x00, 0x13, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x67, 0x65, - 0x74, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, 0x70, 0x5f, 0x74, 0x2d, - 0x31, 0x2d, 0x75, 0x31, 0x2d, 0x75, 0x31, 0x31, 0x3b, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x73, 0x68, 0x61, - 0x64, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x74, 0x2d, - 0x75, 0x31, 0x5b, 0x38, 0x5d, 0x31, 0x3b, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x68, 0x61, - 0x73, 0x68, 0x6d, 0x61, 0x70, 0x00, 0x05, 0x00, 0x03, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x6b, 0x65, 0x79, 0x00, 0x05, - 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x41, 0x00, 0x00, 0x00, - 0x69, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x00, 0x00, - 0x00, 0x05, 0x00, 0x05, 0x00, 0x4d, 0x00, 0x00, 0x00, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x61, 0x62, 0x6c, 0x65, 0x00, - 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x86, - 0x00, 0x00, 0x00, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x61, 0x62, 0x6c, 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, - 0x8a, 0x00, 0x00, 0x00, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x61, 0x62, 0x6c, 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, - 0x00, 0x9d, 0x00, 0x00, 0x00, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0xac, 0x00, - 0x00, 0x00, 0x69, 0x64, 0x78, 0x00, 0x05, 0x00, 0x06, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x6d, - 0x61, 0x70, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x74, 0x00, 0x00, 0x06, 0x00, 0x04, 0x00, 0xb7, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6b, 0x65, 0x79, 0x00, 0x06, 0x00, 0x05, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x69, 0x74, - 0x65, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x67, 0x69, 0x64, 0x00, 0x05, - 0x00, 0x08, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x67, 0x6c, 0x5f, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x49, 0x6e, 0x76, - 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, - 0x00, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, 0x70, 0x5f, 0x74, 0x00, 0x00, 0x00, 0x06, 0x00, 0x05, 0x00, 0xe1, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x00, 0x06, 0x00, 0x05, 0x00, 0xe1, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x73, 0x69, 0x7a, 0x65, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, - 0xe1, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x00, 0x00, 0x00, - 0x00, 0x05, 0x00, 0x07, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x72, 0x5f, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x5f, 0x74, 0x00, 0x00, 0x00, 0x06, 0x00, 0x09, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x73, 0x68, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, - 0x64, 0x6c, 0x65, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x06, 0x00, 0x07, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x00, 0x00, 0x00, 0x06, 0x00, - 0x07, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, - 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, - 0x6e, 0x75, 0x6d, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x00, 0x05, 0x00, 0x06, 0x00, 0xe6, 0x00, 0x00, - 0x00, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, - 0x04, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0xe8, - 0x00, 0x00, 0x00, 0x73, 0x68, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, - 0x64, 0x6c, 0x65, 0x5f, 0x74, 0x00, 0x00, 0x00, 0x06, 0x00, 0x05, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x06, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x74, 0x72, 0x00, 0x00, 0x06, 0x00, 0x07, 0x00, 0xe9, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, - 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x70, 0x63, 0x00, 0x00, 0x06, 0x00, 0x05, - 0x00, 0xeb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x00, 0x00, 0x05, 0x00, - 0x03, 0x00, 0xed, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x05, 0x01, 0x00, 0x00, 0x61, 0x72, 0x67, 0x00, - 0x05, 0x00, 0x03, 0x00, 0x08, 0x01, 0x00, 0x00, 0x61, 0x72, 0x67, 0x00, 0x47, 0x00, 0x04, 0x00, 0x24, 0x00, 0x00, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0x26, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0x26, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0x27, 0x00, 0x00, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x28, 0x00, 0x00, 0x00, 0x02, 0x00, - 0x00, 0x00, 0x48, 0x00, 0x04, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x48, - 0x00, 0x05, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x47, 0x00, 0x04, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, - 0x00, 0xe1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, - 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x48, - 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, - 0x48, 0x00, 0x05, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x00, 0x05, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x10, 0x00, - 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x18, - 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, - 0x20, 0x00, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0xe5, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, - 0x00, 0x47, 0x00, 0x03, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x48, 0x00, 0x04, 0x00, 0xe6, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0xe7, 0x00, 0x00, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x02, 0x00, - 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0xea, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x47, 0x00, 0x03, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xeb, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x05, 0x01, - 0x00, 0x00, 0xec, 0x14, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0x19, 0x01, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x19, - 0x00, 0x00, 0x00, 0x13, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00, 0x00, 0x21, 0x00, 0x03, 0x00, 0x03, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x15, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x20, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x21, 0x00, - 0x04, 0x00, 0x08, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x04, 0x00, 0x0d, 0x00, 0x00, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x21, 0x00, 0x05, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x0d, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x0d, 0x00, - 0x00, 0x00, 0x14, 0x00, 0x02, 0x00, 0x14, 0x00, 0x00, 0x00, 0x21, 0x00, 0x05, 0x00, 0x15, 0x00, 0x00, 0x00, 0x14, - 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x21, 0x00, 0x04, 0x00, 0x1a, 0x00, 0x00, 0x00, - 0x14, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x21, 0x00, 0x04, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x13, 0x00, 0x00, 0x00, 0x27, 0x00, 0x03, 0x00, 0x22, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0x1e, 0x00, - 0x05, 0x00, 0x23, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x1c, - 0x00, 0x04, 0x00, 0x24, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x03, 0x00, - 0x25, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x04, 0x00, 0x26, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, - 0x00, 0x25, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x03, 0x00, 0x27, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x1e, 0x00, - 0x03, 0x00, 0x28, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x22, 0x00, 0x00, 0x00, 0xe5, - 0x14, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x21, 0x00, 0x05, 0x00, 0x29, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, - 0x23, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, - 0x00, 0x51, 0x2d, 0x9e, 0xcc, 0x15, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x2b, - 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x93, 0x35, 0x87, 0x1b, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x07, 0x00, - 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x0d, - 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, - 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x5d, 0x00, 0x00, 0x00, 0x64, 0x6b, 0x54, 0xe6, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, - 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x62, - 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x66, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6a, 0x00, 0x00, 0x00, 0x6b, 0xca, 0xeb, - 0x85, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x35, 0xae, 0xb2, 0xc2, 0x2b, 0x00, - 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x85, - 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x03, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x90, 0x00, 0x00, 0x00, 0x29, 0x00, 0x03, 0x00, 0x14, 0x00, 0x00, 0x00, 0x94, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x0b, - 0x00, 0x0d, 0x00, 0x00, 0x00, 0x97, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x42, 0x00, - 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x42, - 0x00, 0x00, 0x00, 0x2c, 0x00, 0x04, 0x00, 0x13, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x97, 0x00, 0x00, 0x00, - 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0xa5, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, - 0x00, 0x06, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x04, 0x00, 0xb7, 0x00, - 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xb8, 0x00, 0x00, 0x00, 0x07, - 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xbc, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, - 0x26, 0x00, 0x00, 0x00, 0x17, 0x00, 0x04, 0x00, 0xda, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, - 0x00, 0x20, 0x00, 0x04, 0x00, 0xdb, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, 0x3b, 0x00, - 0x04, 0x00, 0xdb, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xdd, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, - 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x27, 0x00, 0x03, 0x00, 0xe2, 0x00, 0x00, - 0x00, 0xe5, 0x14, 0x00, 0x00, 0x1e, 0x00, 0x06, 0x00, 0xe3, 0x00, 0x00, 0x00, 0xe1, 0x00, 0x00, 0x00, 0xe2, 0x00, - 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x27, 0x00, 0x03, 0x00, 0xe4, 0x00, 0x00, 0x00, 0xe5, - 0x14, 0x00, 0x00, 0x1d, 0x00, 0x03, 0x00, 0xe5, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x03, 0x00, - 0xe6, 0x00, 0x00, 0x00, 0xe5, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x04, 0x00, 0xe7, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x0c, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x03, 0x00, 0xe8, 0x00, 0x00, 0x00, 0xe7, 0x00, 0x00, 0x00, 0x1e, 0x00, - 0x03, 0x00, 0xe9, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xe4, 0x00, 0x00, 0x00, 0xe5, - 0x14, 0x00, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x03, 0x00, 0xea, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, - 0x20, 0x00, 0x04, 0x00, 0xe2, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x03, - 0x00, 0xeb, 0x00, 0x00, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xec, 0x00, 0x00, 0x00, 0x09, 0x00, - 0x00, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0xec, 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, 0x09, - 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0xee, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, - 0x20, 0x00, 0x04, 0x00, 0xef, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, - 0x00, 0xf7, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xfa, 0x00, - 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xfe, 0x00, 0x00, 0x00, 0xe5, - 0x14, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x01, 0x01, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, - 0xe8, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x04, 0x01, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, - 0x00, 0x2c, 0x00, 0x06, 0x00, 0xda, 0x00, 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0xb3, 0x00, - 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x36, 0x00, 0x05, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x05, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, - 0x07, 0x00, 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x85, 0x00, 0x00, - 0x00, 0xf6, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x04, 0x01, 0x00, 0x00, 0x05, 0x01, - 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x85, 0x00, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0xdd, 0x00, 0x00, 0x00, 0xde, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, - 0x42, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, 0xde, 0x00, 0x00, - 0x00, 0x3e, 0x00, 0x03, 0x00, 0xd9, 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, - 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xef, 0x00, 0x00, 0x00, 0xf0, - 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0xee, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, - 0x06, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, 0xae, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, - 0x00, 0xf2, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0xf4, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xf2, 0x00, 0x00, 0x00, 0xf3, 0x00, 0x00, 0x00, 0xf4, - 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xf3, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x01, 0x00, 0xf8, 0x00, 0x02, 0x00, - 0xf4, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xf7, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, - 0x00, 0x83, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0xe1, 0x00, 0x00, 0x00, 0xf9, 0x00, - 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xfa, 0x00, 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, 0xed, - 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0xe2, 0x00, 0x00, 0x00, - 0xfc, 0x00, 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, - 0x00, 0xd9, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xfe, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xfc, 0x00, - 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x06, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, - 0x01, 0x01, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x06, - 0x00, 0xe8, 0x00, 0x00, 0x00, 0x03, 0x01, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, - 0x00, 0x00, 0x90, 0x01, 0x04, 0x00, 0x23, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0xf9, 0x00, 0x00, 0x00, 0x3e, - 0x00, 0x03, 0x00, 0x05, 0x01, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x23, 0x00, 0x00, 0x00, - 0x07, 0x01, 0x00, 0x00, 0x05, 0x01, 0x00, 0x00, 0x90, 0x01, 0x04, 0x00, 0x13, 0x00, 0x00, 0x00, 0x09, 0x01, 0x00, - 0x00, 0x03, 0x01, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x08, 0x01, 0x00, 0x00, 0x09, 0x01, 0x00, 0x00, 0x3d, 0x00, - 0x04, 0x00, 0x13, 0x00, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x39, 0x00, 0x06, 0x00, 0x13, - 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, - 0x3e, 0x00, 0x03, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x13, 0x00, 0x00, - 0x00, 0x0c, 0x01, 0x00, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0d, 0x01, - 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x0c, 0x01, 0x00, 0x00, 0xa8, 0x00, 0x04, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0e, - 0x01, 0x00, 0x00, 0x0d, 0x01, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfa, 0x00, 0x04, 0x00, 0x0e, 0x01, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0xf8, 0x00, 0x02, - 0x00, 0x0f, 0x01, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0xed, 0x00, - 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0xa5, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x12, - 0x01, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, - 0xd9, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xfe, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, 0x00, 0x12, 0x01, 0x00, - 0x00, 0x83, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x3d, 0x00, 0x06, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x15, 0x01, - 0x00, 0x00, 0x14, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x13, - 0x00, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x01, 0x01, 0x00, 0x00, - 0x17, 0x01, 0x00, 0x00, 0x15, 0x01, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x90, 0x01, 0x04, 0x00, 0xe8, 0x00, 0x00, - 0x00, 0x18, 0x01, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, 0x3e, 0x00, 0x05, 0x00, 0x17, 0x01, 0x00, 0x00, 0x18, 0x01, - 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x10, 0x01, 0x00, 0x00, 0xf8, - 0x00, 0x02, 0x00, 0x10, 0x01, 0x00, 0x00, 0xfd, 0x00, 0x01, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, 0x05, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, - 0x00, 0x07, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x3d, 0x00, - 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, - 0x09, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, - 0x00, 0x09, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x31, 0x00, - 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, 0x09, - 0x00, 0x00, 0x00, 0xc2, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, - 0x36, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, - 0x00, 0x37, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x09, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x3d, 0x00, - 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, - 0x09, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, - 0x00, 0x09, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, - 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x37, - 0x00, 0x03, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x07, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x12, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, - 0x00, 0x3f, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x41, 0x00, - 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x4d, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, - 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, - 0x00, 0x3f, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, - 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x43, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x43, 0x00, 0x00, 0x00, 0xf6, - 0x00, 0x04, 0x00, 0x45, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, - 0x47, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x47, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x48, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x49, 0x00, - 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0x49, 0x00, 0x00, 0x00, 0x44, - 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x44, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x4d, 0x00, 0x00, - 0x00, 0x0f, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x07, 0x00, 0x00, 0x00, 0x4e, 0x00, 0x00, 0x00, 0x4d, 0x00, - 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x4e, - 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x05, 0x00, 0x06, 0x00, - 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, - 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, - 0x3f, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, - 0x00, 0x54, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x3f, 0x00, - 0x00, 0x00, 0xc2, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x57, - 0x00, 0x00, 0x00, 0xc5, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, - 0x58, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x5a, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, - 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x5a, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x5d, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, - 0x3f, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x46, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, - 0x00, 0x46, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, 0x41, 0x00, - 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, 0x60, - 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x41, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, - 0x43, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x45, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x63, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x64, 0x00, - 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x64, - 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, - 0xc2, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x66, 0x00, 0x00, - 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, - 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x3e, - 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, - 0x6b, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6c, 0x00, 0x00, - 0x00, 0x6b, 0x00, 0x00, 0x00, 0x6a, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x6c, 0x00, - 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc2, - 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, - 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x05, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x3e, 0x00, - 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x72, - 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, - 0x72, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, - 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc2, 0x00, - 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x66, 0x00, 0x00, 0x00, 0x3d, - 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x05, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x77, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, - 0x00, 0x3f, 0x00, 0x00, 0x00, 0x77, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x78, 0x00, - 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x78, 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, - 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, - 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, - 0x00, 0x17, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x19, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, - 0x00, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x85, 0x00, 0x00, 0x00, 0x86, - 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x85, 0x00, 0x00, 0x00, 0x8a, 0x00, 0x00, 0x00, - 0x07, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, - 0x00, 0x7c, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x7c, 0x00, 0x00, 0x00, 0xf6, 0x00, 0x04, 0x00, 0x7e, 0x00, - 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x80, 0x00, 0x00, 0x00, 0xf8, - 0x00, 0x02, 0x00, 0x80, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, 0x00, - 0x7b, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, - 0x00, 0x0c, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0x82, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x7e, 0x00, - 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x84, - 0x00, 0x00, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x86, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, - 0x41, 0x00, 0x06, 0x00, 0x07, 0x00, 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, - 0x00, 0x84, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x87, 0x00, - 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x3e, - 0x00, 0x03, 0x00, 0x8a, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0x07, 0x00, 0x00, 0x00, - 0x8b, 0x00, 0x00, 0x00, 0x8a, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, 0xab, 0x00, 0x05, 0x00, 0x14, 0x00, - 0x00, 0x00, 0x8d, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0x8f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0x8d, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, - 0x8f, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x8e, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x90, 0x00, 0x00, - 0x00, 0xf8, 0x00, 0x02, 0x00, 0x8f, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x7f, 0x00, 0x00, 0x00, 0xf8, 0x00, - 0x02, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, 0x7b, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x93, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, - 0x60, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x93, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, - 0x00, 0x7c, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x7e, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x94, 0x00, - 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, - 0xf8, 0x00, 0x02, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x39, 0x00, 0x06, 0x00, 0x14, 0x00, 0x00, 0x00, 0x99, 0x00, 0x00, - 0x00, 0x18, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x99, 0x00, - 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, - 0xf8, 0x00, 0x02, 0x00, 0x21, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x9d, 0x00, 0x00, - 0x00, 0x07, 0x00, 0x00, 0x00, 0x51, 0x00, 0x05, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, 0x1f, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x39, - 0x00, 0x06, 0x00, 0x06, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, - 0x9d, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, 0x05, - 0x00, 0x13, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x37, 0x00, - 0x03, 0x00, 0x23, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x2b, - 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, - 0xac, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0xb8, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, - 0x00, 0x07, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, 0x1c, 0x00, - 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0xa8, 0x00, 0x04, 0x00, 0x14, 0x00, 0x00, 0x00, 0xa2, 0x00, 0x00, 0x00, 0xa1, - 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0xa4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, - 0xa2, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xa3, 0x00, 0x00, - 0x00, 0x51, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x02, 0x00, - 0x00, 0x00, 0xaa, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, 0x42, - 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xa4, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xa4, 0x00, 0x00, 0x00, - 0xf5, 0x00, 0x07, 0x00, 0x14, 0x00, 0x00, 0x00, 0xa8, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, - 0x00, 0xa7, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xa8, 0x00, 0x00, 0x00, 0xa9, 0x00, 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0xf8, - 0x00, 0x02, 0x00, 0xa9, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x98, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, - 0xaa, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, - 0x00, 0x2b, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0xac, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, 0xf9, 0x00, - 0x02, 0x00, 0xae, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xae, 0x00, 0x00, 0x00, 0xf6, 0x00, 0x04, 0x00, 0xb0, - 0x00, 0x00, 0x00, 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xaf, 0x00, 0x00, 0x00, - 0xf8, 0x00, 0x02, 0x00, 0xaf, 0x00, 0x00, 0x00, 0x51, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xb2, 0x00, 0x00, - 0x00, 0x2a, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x82, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xb4, 0x00, - 0x00, 0x00, 0xb2, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xb5, - 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, - 0xb5, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0xac, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, - 0x00, 0x51, 0x00, 0x05, 0x00, 0x22, 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, 0x41, - 0x00, 0x06, 0x00, 0xbc, 0x00, 0x00, 0x00, 0xbd, 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, - 0xbb, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x06, 0x00, 0x26, 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, 0xbd, 0x00, 0x00, - 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x90, 0x01, 0x04, 0x00, 0xb7, 0x00, 0x00, 0x00, 0xbf, 0x00, - 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0xb9, 0x00, 0x00, 0x00, 0xbf, 0x00, 0x00, 0x00, 0x41, - 0x00, 0x05, 0x00, 0x85, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, - 0x3d, 0x00, 0x04, 0x00, 0x13, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, - 0x00, 0x14, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0xf7, 0x00, - 0x03, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xc2, 0x00, 0x00, 0x00, 0xc3, - 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, - 0x98, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xc6, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x85, 0x00, 0x00, - 0x00, 0xc7, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x13, 0x00, - 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x00, 0x00, 0x39, 0x00, 0x06, 0x00, 0x14, 0x00, 0x00, 0x00, 0xc9, - 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, - 0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xc9, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, - 0x00, 0xcb, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xca, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x85, 0x00, - 0x00, 0x00, 0xcc, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x13, - 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, 0xcc, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, - 0xce, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, 0xa8, 0x00, 0x04, 0x00, 0x14, 0x00, 0x00, - 0x00, 0xcf, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xcb, 0x00, 0x00, 0x00, 0xf8, 0x00, - 0x02, 0x00, 0xcb, 0x00, 0x00, 0x00, 0xf5, 0x00, 0x07, 0x00, 0x14, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, 0xc9, - 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0xcf, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, - 0xd2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xd0, 0x00, 0x00, 0x00, 0xd1, 0x00, 0x00, - 0x00, 0xd2, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xd1, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x85, 0x00, - 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x13, - 0x00, 0x00, 0x00, 0xd4, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0xd4, 0x00, 0x00, 0x00, - 0xf8, 0x00, 0x02, 0x00, 0xd2, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xc4, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, - 0x00, 0xc4, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xb1, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xb1, 0x00, - 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, - 0x3e, 0x00, 0x03, 0x00, 0xac, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xae, 0x00, 0x00, - 0x00, 0xf8, 0x00, 0x02, 0x00, 0xb0, 0x00, 0x00, 0x00, 0xff, 0x00, 0x01, 0x00, 0x38, 0x00, 0x01, 0x00 +static const std::array g_replacer_sbt_comp = { + 0x07230203, 0x00010300, 0x0008000b, 0x00000427, 0x00000000, 0x00020011, 0x00000001, 0x00020011, 0x000014b6, + 0x00020011, 0x000014e3, 0x0008000a, 0x5f565053, 0x5f545845, 0x63736564, 0x74706972, 0x695f726f, 0x7865646e, + 0x00676e69, 0x0009000a, 0x5f565053, 0x5f52484b, 0x73796870, 0x6c616369, 0x6f74735f, 0x65676172, 0x6675625f, + 0x00726566, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x000014e4, + 0x00000001, 0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x00000108, 0x00060010, 0x00000004, + 0x00000011, 0x00000020, 0x00000001, 0x00000001, 0x00030003, 0x00000002, 0x000001cc, 0x00070004, 0x455f4c47, + 0x625f5458, 0x65666675, 0x65725f72, 0x65726566, 0x0065636e, 0x00080004, 0x455f4c47, 0x625f5458, 0x65666675, + 0x65725f72, 0x65726566, 0x3265636e, 0x00000000, 0x000a0004, 0x475f4c47, 0x4c474f4f, 0x70635f45, 0x74735f70, + 0x5f656c79, 0x656e696c, 0x7269645f, 0x69746365, 0x00006576, 0x00080004, 0x475f4c47, 0x4c474f4f, 0x6e695f45, + 0x64756c63, 0x69645f65, 0x74636572, 0x00657669, 0x00040005, 0x00000004, 0x6e69616d, 0x00000000, 0x00080005, + 0x00000025, 0x64616873, 0x675f7265, 0x70756f72, 0x6e61685f, 0x5f656c64, 0x00000074, 0x00050006, 0x00000025, + 0x00000000, 0x61746164, 0x00000000, 0x00060005, 0x00000026, 0x68736168, 0x5f70616d, 0x6d657469, 0x0000745f, + 0x00040006, 0x00000026, 0x00000000, 0x0079656b, 0x00050006, 0x00000026, 0x00000001, 0x756c6176, 0x00000065, + 0x00060005, 0x00000028, 0x68736148, 0x5370614d, 0x61726f74, 0x00006567, 0x00040006, 0x00000028, 0x00000000, + 0x00000076, 0x00080005, 0x00000108, 0x475f6c67, 0x61626f6c, 0x766e496c, 0x7461636f, 0x496e6f69, 0x00000044, + 0x00050005, 0x0000010d, 0x68736168, 0x5f70616d, 0x00000074, 0x00050006, 0x0000010d, 0x00000000, 0x726f7473, + 0x00656761, 0x00050006, 0x0000010d, 0x00000001, 0x657a6973, 0x00000000, 0x00060006, 0x0000010d, 0x00000002, + 0x61706163, 0x79746963, 0x00000000, 0x00070005, 0x0000010f, 0x6c706572, 0x72656361, 0x7261705f, 0x5f736d61, + 0x00000074, 0x00090006, 0x0000010f, 0x00000000, 0x64616873, 0x675f7265, 0x70756f72, 0x6e61685f, 0x5f656c64, + 0x0070616d, 0x00070006, 0x0000010f, 0x00000001, 0x75706e69, 0x61685f74, 0x656c646e, 0x00000073, 0x00070006, + 0x0000010f, 0x00000002, 0x7074756f, 0x685f7475, 0x6c646e61, 0x00007365, 0x00060006, 0x0000010f, 0x00000003, + 0x5f6d756e, 0x646e6168, 0x0073656c, 0x00060005, 0x00000112, 0x72646441, 0x41737365, 0x79617272, 0x00000000, + 0x00040006, 0x00000112, 0x00000000, 0x00000076, 0x00080005, 0x00000114, 0x64616873, 0x675f7265, 0x70756f72, + 0x6e61685f, 0x5f656c64, 0x00000074, 0x00050006, 0x00000114, 0x00000000, 0x61746164, 0x00000000, 0x00060005, + 0x00000115, 0x756f7247, 0x6e614870, 0x50656c64, 0x00007274, 0x00070006, 0x00000115, 0x00000000, 0x756f7267, + 0x61685f70, 0x656c646e, 0x00000000, 0x00030005, 0x00000117, 0x00006370, 0x00050006, 0x00000117, 0x00000000, + 0x61726170, 0x0000736d, 0x00030005, 0x00000119, 0x00000000, 0x00040047, 0x00000024, 0x00000006, 0x00000004, + 0x00050048, 0x00000025, 0x00000000, 0x00000023, 0x00000000, 0x00050048, 0x00000026, 0x00000000, 0x00000023, + 0x00000000, 0x00050048, 0x00000026, 0x00000001, 0x00000023, 0x00000020, 0x00040047, 0x00000027, 0x00000006, + 0x00000040, 0x00030047, 0x00000028, 0x00000002, 0x00040048, 0x00000028, 0x00000000, 0x00000018, 0x00050048, + 0x00000028, 0x00000000, 0x00000023, 0x00000000, 0x00040047, 0x00000108, 0x0000000b, 0x0000001c, 0x00050048, + 0x0000010d, 0x00000000, 0x00000023, 0x00000000, 0x00050048, 0x0000010d, 0x00000001, 0x00000023, 0x00000008, + 0x00050048, 0x0000010d, 0x00000002, 0x00000023, 0x0000000c, 0x00050048, 0x0000010f, 0x00000000, 0x00000023, + 0x00000000, 0x00050048, 0x0000010f, 0x00000001, 0x00000023, 0x00000010, 0x00050048, 0x0000010f, 0x00000002, + 0x00000023, 0x00000018, 0x00050048, 0x0000010f, 0x00000003, 0x00000023, 0x00000020, 0x00040047, 0x00000111, + 0x00000006, 0x00000008, 0x00030047, 0x00000112, 0x00000002, 0x00040048, 0x00000112, 0x00000000, 0x00000018, + 0x00050048, 0x00000112, 0x00000000, 0x00000023, 0x00000000, 0x00040047, 0x00000113, 0x00000006, 0x00000004, + 0x00050048, 0x00000114, 0x00000000, 0x00000023, 0x00000000, 0x00030047, 0x00000115, 0x00000002, 0x00050048, + 0x00000115, 0x00000000, 0x00000023, 0x00000000, 0x00030047, 0x00000117, 0x00000002, 0x00050048, 0x00000117, + 0x00000000, 0x00000023, 0x00000000, 0x00040047, 0x0000016e, 0x0000000b, 0x00000019, 0x00020013, 0x00000002, + 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006, 0x00000020, 0x00000000, 0x00040020, 0x00000007, + 0x00000007, 0x00000006, 0x0004002b, 0x00000006, 0x0000000c, 0x00000008, 0x0004001c, 0x0000000d, 0x00000006, + 0x0000000c, 0x00020014, 0x00000014, 0x00030027, 0x00000022, 0x000014e5, 0x0004001c, 0x00000024, 0x00000006, + 0x0000000c, 0x0003001e, 0x00000025, 0x00000024, 0x0004001e, 0x00000026, 0x00000025, 0x00000025, 0x0003001d, + 0x00000027, 0x00000026, 0x0003001e, 0x00000028, 0x00000027, 0x00040020, 0x00000022, 0x000014e5, 0x00000028, + 0x0004002b, 0x00000006, 0x0000002e, 0xcc9e2d51, 0x00040015, 0x00000032, 0x00000020, 0x00000001, 0x0004002b, + 0x00000032, 0x00000033, 0x0000000f, 0x0004002b, 0x00000032, 0x00000036, 0x00000011, 0x0004002b, 0x00000006, + 0x00000039, 0x1b873593, 0x0004002b, 0x00000006, 0x00000042, 0x00000000, 0x00040020, 0x0000004c, 0x00000007, + 0x0000000d, 0x0004002b, 0x00000032, 0x00000054, 0x0000000d, 0x0004002b, 0x00000032, 0x00000057, 0x00000013, + 0x0004002b, 0x00000006, 0x0000005b, 0x00000005, 0x0004002b, 0x00000006, 0x0000005d, 0xe6546b64, 0x0004002b, + 0x00000032, 0x00000060, 0x00000001, 0x0004002b, 0x00000006, 0x00000062, 0x00000020, 0x0004002b, 0x00000032, + 0x00000066, 0x00000010, 0x0004002b, 0x00000006, 0x0000006a, 0x85ebca6b, 0x0004002b, 0x00000006, 0x00000071, + 0xc2b2ae35, 0x0004002b, 0x00000032, 0x00000083, 0x00000000, 0x0003002a, 0x00000014, 0x00000090, 0x00030029, + 0x00000014, 0x00000094, 0x000b002c, 0x0000000d, 0x00000097, 0x00000042, 0x00000042, 0x00000042, 0x00000042, + 0x00000042, 0x00000042, 0x00000042, 0x00000042, 0x0004002b, 0x00000032, 0x000000a5, 0x00000002, 0x0004002b, + 0x00000006, 0x000000b3, 0x00000001, 0x00040020, 0x000000bc, 0x000014e5, 0x00000026, 0x0004002b, 0x00000032, + 0x000000ca, 0x00000003, 0x0004002b, 0x00000032, 0x000000cd, 0x00000004, 0x0004002b, 0x00000032, 0x000000d0, + 0x00000005, 0x0004002b, 0x00000032, 0x000000d3, 0x00000006, 0x0004002b, 0x00000032, 0x000000d6, 0x00000007, + 0x00040017, 0x00000106, 0x00000006, 0x00000003, 0x00040020, 0x00000107, 0x00000001, 0x00000106, 0x0004003b, + 0x00000107, 0x00000108, 0x00000001, 0x00040020, 0x00000109, 0x00000001, 0x00000006, 0x0005001e, 0x0000010d, + 0x00000022, 0x00000006, 0x00000006, 0x00030027, 0x0000010e, 0x000014e5, 0x0006001e, 0x0000010f, 0x0000010d, + 0x0000010e, 0x0000010e, 0x00000006, 0x00030027, 0x00000110, 0x000014e5, 0x0003001d, 0x00000111, 0x00000110, + 0x0003001e, 0x00000112, 0x00000111, 0x0004001c, 0x00000113, 0x00000006, 0x0000000c, 0x0003001e, 0x00000114, + 0x00000113, 0x0003001e, 0x00000115, 0x00000114, 0x00040020, 0x00000110, 0x000014e5, 0x00000115, 0x00040020, + 0x0000010e, 0x000014e5, 0x00000112, 0x0003001e, 0x00000117, 0x0000010f, 0x00040020, 0x00000118, 0x00000009, + 0x00000117, 0x0004003b, 0x00000118, 0x00000119, 0x00000009, 0x00040020, 0x0000011a, 0x00000009, 0x00000006, + 0x00040020, 0x00000122, 0x00000009, 0x0000010d, 0x00040020, 0x00000125, 0x00000009, 0x0000010e, 0x00040020, + 0x00000129, 0x000014e5, 0x00000110, 0x00040020, 0x0000012c, 0x000014e5, 0x00000114, 0x00040020, 0x0000015b, + 0x000014e5, 0x00000113, 0x00040020, 0x0000015e, 0x000014e5, 0x00000006, 0x0006002c, 0x00000106, 0x0000016e, + 0x00000062, 0x000000b3, 0x000000b3, 0x00030001, 0x00000014, 0x000003b7, 0x00050036, 0x00000002, 0x00000004, + 0x00000000, 0x00000003, 0x000200f8, 0x00000005, 0x0004003b, 0x0000004c, 0x000002df, 0x00000007, 0x0004003b, + 0x0000004c, 0x000002dc, 0x00000007, 0x0004003b, 0x0000004c, 0x000002d9, 0x00000007, 0x0004003b, 0x0000004c, + 0x000002d6, 0x00000007, 0x0004003b, 0x0000004c, 0x000002d3, 0x00000007, 0x0004003b, 0x0000004c, 0x000002d0, + 0x00000007, 0x0004003b, 0x0000004c, 0x000002cd, 0x00000007, 0x0004003b, 0x0000004c, 0x000002ca, 0x00000007, + 0x0004003b, 0x0000004c, 0x000002c7, 0x00000007, 0x0004003b, 0x0000004c, 0x000002c4, 0x00000007, 0x0004003b, + 0x0000004c, 0x00000209, 0x00000007, 0x000300f7, 0x0000016f, 0x00000000, 0x000300fb, 0x00000042, 0x00000170, + 0x000200f8, 0x00000170, 0x00050041, 0x00000109, 0x0000010a, 0x00000108, 0x00000042, 0x0004003d, 0x00000006, + 0x0000010b, 0x0000010a, 0x00060041, 0x0000011a, 0x0000011b, 0x00000119, 0x00000083, 0x000000ca, 0x0004003d, + 0x00000006, 0x0000011c, 0x0000011b, 0x000500ae, 0x00000014, 0x0000011d, 0x0000010b, 0x0000011c, 0x000300f7, + 0x0000011f, 0x00000000, 0x000400fa, 0x0000011d, 0x0000011e, 0x0000011f, 0x000200f8, 0x0000011e, 0x000200f9, + 0x0000016f, 0x000200f8, 0x0000011f, 0x00060041, 0x00000122, 0x00000123, 0x00000119, 0x00000083, 0x00000083, + 0x0004003d, 0x0000010d, 0x00000124, 0x00000123, 0x00060041, 0x00000125, 0x00000126, 0x00000119, 0x00000083, + 0x00000060, 0x0004003d, 0x0000010e, 0x00000127, 0x00000126, 0x00060041, 0x00000129, 0x0000012a, 0x00000127, + 0x00000083, 0x0000010b, 0x0006003d, 0x00000110, 0x0000012b, 0x0000012a, 0x00000002, 0x00000008, 0x00050041, + 0x0000012c, 0x0000012d, 0x0000012b, 0x00000083, 0x0006003d, 0x00000114, 0x0000012e, 0x0000012d, 0x00000002, + 0x00000010, 0x00050051, 0x00000022, 0x00000131, 0x00000124, 0x00000000, 0x00050051, 0x00000006, 0x00000136, + 0x00000124, 0x00000002, 0x00050051, 0x00000113, 0x0000013a, 0x0000012e, 0x00000000, 0x00050051, 0x00000006, + 0x0000013c, 0x0000013a, 0x00000000, 0x00050051, 0x00000006, 0x0000013e, 0x0000013a, 0x00000001, 0x00050051, + 0x00000006, 0x00000140, 0x0000013a, 0x00000002, 0x00050051, 0x00000006, 0x00000142, 0x0000013a, 0x00000003, + 0x00050051, 0x00000006, 0x00000144, 0x0000013a, 0x00000004, 0x00050051, 0x00000006, 0x00000146, 0x0000013a, + 0x00000005, 0x00050051, 0x00000006, 0x00000148, 0x0000013a, 0x00000006, 0x00050051, 0x00000006, 0x0000014a, + 0x0000013a, 0x00000007, 0x000b0050, 0x0000000d, 0x0000037c, 0x0000013c, 0x0000013e, 0x00000140, 0x00000142, + 0x00000144, 0x00000146, 0x00000148, 0x0000014a, 0x000300f7, 0x000001de, 0x00000000, 0x000300fb, 0x00000042, + 0x00000187, 0x000200f8, 0x00000187, 0x000300f7, 0x000001ff, 0x00000000, 0x000300fb, 0x00000042, 0x000001ea, + 0x000200f8, 0x000001ea, 0x000200f9, 0x000001eb, 0x000200f8, 0x000001eb, 0x000700f5, 0x00000006, 0x000003b1, + 0x00000042, 0x000001ea, 0x000001fb, 0x000001f9, 0x000500b0, 0x00000014, 0x000001ee, 0x000003b1, 0x0000000c, + 0x000400f6, 0x000001fc, 0x000001f9, 0x00000000, 0x000400fa, 0x000001ee, 0x000001ef, 0x000001fc, 0x000200f8, + 0x000001ef, 0x0003003e, 0x000002dc, 0x0000037c, 0x00050041, 0x00000007, 0x000002de, 0x000002dc, 0x000003b1, + 0x0004003d, 0x00000006, 0x000001f2, 0x000002de, 0x0003003e, 0x000002df, 0x00000097, 0x00050041, 0x00000007, + 0x000002e1, 0x000002df, 0x000003b1, 0x0004003d, 0x00000006, 0x000001f5, 0x000002e1, 0x000500ab, 0x00000014, + 0x000001f6, 0x000001f2, 0x000001f5, 0x000300f7, 0x000001f8, 0x00000000, 0x000400fa, 0x000001f6, 0x000001f7, + 0x000001f8, 0x000200f8, 0x000001f7, 0x000200f9, 0x000001fc, 0x000200f8, 0x000001f8, 0x000200f9, 0x000001f9, + 0x000200f8, 0x000001f9, 0x00050080, 0x00000006, 0x000001fb, 0x000003b1, 0x00000060, 0x000200f9, 0x000001eb, + 0x000200f8, 0x000001fc, 0x000700f5, 0x00000014, 0x000003b5, 0x000003b7, 0x000001eb, 0x00000090, 0x000001f7, + 0x000700f5, 0x00000014, 0x000003b2, 0x00000090, 0x000001eb, 0x00000094, 0x000001f7, 0x000300f7, 0x000001fe, + 0x00000000, 0x000400fa, 0x000003b2, 0x000001ff, 0x000001fe, 0x000200f8, 0x000001fe, 0x000200f9, 0x000001ff, + 0x000200f8, 0x000001ff, 0x000700f5, 0x00000014, 0x000003b4, 0x000003b5, 0x000001fc, 0x00000094, 0x000001fe, + 0x000400a8, 0x00000014, 0x00000189, 0x000003b4, 0x000300f7, 0x0000018d, 0x00000000, 0x000400fa, 0x00000189, + 0x0000018a, 0x0000018d, 0x000200f8, 0x0000018a, 0x000500aa, 0x00000014, 0x0000018c, 0x00000136, 0x00000042, + 0x000200f9, 0x0000018d, 0x000200f8, 0x0000018d, 0x000700f5, 0x00000014, 0x0000018e, 0x000003b4, 0x000001ff, + 0x0000018c, 0x0000018a, 0x000300f7, 0x00000190, 0x00000000, 0x000400fa, 0x0000018e, 0x0000018f, 0x00000190, + 0x000200f8, 0x0000018f, 0x000200f9, 0x000001de, 0x000200f8, 0x00000190, 0x000200f9, 0x0000020d, 0x000200f8, + 0x0000020d, 0x000700f5, 0x00000006, 0x000003b9, 0x00000042, 0x00000190, 0x0000021f, 0x00000211, 0x000700f5, + 0x00000006, 0x000003b8, 0x00000042, 0x00000190, 0x00000222, 0x00000211, 0x000500b0, 0x00000014, 0x00000210, + 0x000003b8, 0x0000000c, 0x000400f6, 0x00000223, 0x00000211, 0x00000000, 0x000400fa, 0x00000210, 0x00000211, + 0x00000223, 0x000200f8, 0x00000211, 0x0003003e, 0x00000209, 0x0000037c, 0x00050041, 0x00000007, 0x00000213, + 0x00000209, 0x000003b8, 0x0004003d, 0x00000006, 0x00000214, 0x00000213, 0x00050084, 0x00000006, 0x0000023a, + 0x00000214, 0x0000002e, 0x000500c4, 0x00000006, 0x0000023c, 0x0000023a, 0x00000033, 0x000500c2, 0x00000006, + 0x0000023e, 0x0000023a, 0x00000036, 0x000500c5, 0x00000006, 0x0000023f, 0x0000023c, 0x0000023e, 0x00050084, + 0x00000006, 0x00000241, 0x0000023f, 0x00000039, 0x000500c6, 0x00000006, 0x00000217, 0x000003b9, 0x00000241, + 0x000500c4, 0x00000006, 0x00000219, 0x00000217, 0x00000054, 0x000500c2, 0x00000006, 0x0000021b, 0x00000217, + 0x00000057, 0x000500c5, 0x00000006, 0x0000021c, 0x00000219, 0x0000021b, 0x00050084, 0x00000006, 0x0000021e, + 0x0000021c, 0x0000005b, 0x00050080, 0x00000006, 0x0000021f, 0x0000021e, 0x0000005d, 0x00050080, 0x00000006, + 0x00000222, 0x000003b8, 0x00000060, 0x000200f9, 0x0000020d, 0x000200f8, 0x00000223, 0x000500c6, 0x00000006, + 0x00000225, 0x000003b9, 0x00000062, 0x000500c2, 0x00000006, 0x00000227, 0x00000225, 0x00000066, 0x000500c6, + 0x00000006, 0x00000229, 0x00000225, 0x00000227, 0x00050084, 0x00000006, 0x0000022b, 0x00000229, 0x0000006a, + 0x000500c2, 0x00000006, 0x0000022d, 0x0000022b, 0x00000054, 0x000500c6, 0x00000006, 0x0000022f, 0x0000022b, + 0x0000022d, 0x00050084, 0x00000006, 0x00000231, 0x0000022f, 0x00000071, 0x000500c2, 0x00000006, 0x00000233, + 0x00000231, 0x00000066, 0x000500c6, 0x00000006, 0x00000235, 0x00000231, 0x00000233, 0x000200f9, 0x00000192, + 0x000200f8, 0x00000192, 0x000700f5, 0x00000014, 0x000003e2, 0x000003b7, 0x00000223, 0x00000426, 0x000001d8, + 0x000700f5, 0x00000014, 0x000003d0, 0x000003b7, 0x00000223, 0x000003ca, 0x000001d8, 0x000700f5, 0x00000014, + 0x000003c1, 0x000003b7, 0x00000223, 0x000003be, 0x000001d8, 0x000700f5, 0x00000006, 0x000003ba, 0x00000235, + 0x00000223, 0x000001da, 0x000001d8, 0x000400f6, 0x000001db, 0x000001d8, 0x00000000, 0x000200f9, 0x00000193, + 0x000200f8, 0x00000193, 0x00050082, 0x00000006, 0x00000195, 0x00000136, 0x000000b3, 0x000500c7, 0x00000006, + 0x00000197, 0x000003ba, 0x00000195, 0x00060041, 0x000000bc, 0x0000019a, 0x00000131, 0x00000083, 0x00000197, + 0x0006003d, 0x00000026, 0x0000019b, 0x0000019a, 0x00000002, 0x00000010, 0x00050051, 0x00000025, 0x0000019c, + 0x0000019b, 0x00000000, 0x00050051, 0x00000024, 0x0000019e, 0x0000019c, 0x00000000, 0x00050051, 0x00000006, + 0x000001a0, 0x0000019e, 0x00000000, 0x00050051, 0x00000006, 0x000001a2, 0x0000019e, 0x00000001, 0x00050051, + 0x00000006, 0x000001a4, 0x0000019e, 0x00000002, 0x00050051, 0x00000006, 0x000001a6, 0x0000019e, 0x00000003, + 0x00050051, 0x00000006, 0x000001a8, 0x0000019e, 0x00000004, 0x00050051, 0x00000006, 0x000001aa, 0x0000019e, + 0x00000005, 0x00050051, 0x00000006, 0x000001ac, 0x0000019e, 0x00000006, 0x00050051, 0x00000006, 0x000001ae, + 0x0000019e, 0x00000007, 0x00050051, 0x00000025, 0x000001b0, 0x0000019b, 0x00000001, 0x00050051, 0x00000024, + 0x000001b2, 0x000001b0, 0x00000000, 0x00050051, 0x00000006, 0x000001b4, 0x000001b2, 0x00000000, 0x00050051, + 0x00000006, 0x000001b6, 0x000001b2, 0x00000001, 0x00050051, 0x00000006, 0x000001b8, 0x000001b2, 0x00000002, + 0x00050051, 0x00000006, 0x000001ba, 0x000001b2, 0x00000003, 0x00050051, 0x00000006, 0x000001bc, 0x000001b2, + 0x00000004, 0x00050051, 0x00000006, 0x000001be, 0x000001b2, 0x00000005, 0x00050051, 0x00000006, 0x000001c0, + 0x000001b2, 0x00000006, 0x00050051, 0x00000006, 0x000001c2, 0x000001b2, 0x00000007, 0x000b0050, 0x0000000d, + 0x0000038d, 0x000001a0, 0x000001a2, 0x000001a4, 0x000001a6, 0x000001a8, 0x000001aa, 0x000001ac, 0x000001ae, + 0x000300f7, 0x00000262, 0x00000000, 0x000300fb, 0x00000042, 0x0000024d, 0x000200f8, 0x0000024d, 0x000200f9, + 0x0000024e, 0x000200f8, 0x0000024e, 0x000700f5, 0x00000006, 0x000003bb, 0x00000042, 0x0000024d, 0x0000025e, + 0x0000025c, 0x000500b0, 0x00000014, 0x00000251, 0x000003bb, 0x0000000c, 0x000400f6, 0x0000025f, 0x0000025c, + 0x00000000, 0x000400fa, 0x00000251, 0x00000252, 0x0000025f, 0x000200f8, 0x00000252, 0x0003003e, 0x000002d6, + 0x0000038d, 0x00050041, 0x00000007, 0x000002d8, 0x000002d6, 0x000003bb, 0x0004003d, 0x00000006, 0x00000255, + 0x000002d8, 0x0003003e, 0x000002d9, 0x00000097, 0x00050041, 0x00000007, 0x000002db, 0x000002d9, 0x000003bb, + 0x0004003d, 0x00000006, 0x00000258, 0x000002db, 0x000500ab, 0x00000014, 0x00000259, 0x00000255, 0x00000258, + 0x000300f7, 0x0000025b, 0x00000000, 0x000400fa, 0x00000259, 0x0000025a, 0x0000025b, 0x000200f8, 0x0000025a, + 0x000200f9, 0x0000025f, 0x000200f8, 0x0000025b, 0x000200f9, 0x0000025c, 0x000200f8, 0x0000025c, 0x00050080, + 0x00000006, 0x0000025e, 0x000003bb, 0x00000060, 0x000200f9, 0x0000024e, 0x000200f8, 0x0000025f, 0x000700f5, + 0x00000014, 0x000003bf, 0x000003c1, 0x0000024e, 0x00000090, 0x0000025a, 0x000700f5, 0x00000014, 0x000003bc, + 0x00000090, 0x0000024e, 0x00000094, 0x0000025a, 0x000300f7, 0x00000261, 0x00000000, 0x000400fa, 0x000003bc, + 0x00000262, 0x00000261, 0x000200f8, 0x00000261, 0x000200f9, 0x00000262, 0x000200f8, 0x00000262, 0x000700f5, + 0x00000014, 0x000003be, 0x000003bf, 0x0000025f, 0x00000094, 0x00000261, 0x000300f7, 0x000001d7, 0x00000000, + 0x000400fa, 0x000003be, 0x000001c7, 0x000001c8, 0x000200f8, 0x000001c7, 0x000200f9, 0x000001db, 0x000200f8, + 0x000001c8, 0x000300f7, 0x00000280, 0x00000000, 0x000300fb, 0x00000042, 0x0000026b, 0x000200f8, 0x0000026b, + 0x000200f9, 0x0000026c, 0x000200f8, 0x0000026c, 0x000700f5, 0x00000006, 0x000003c7, 0x00000042, 0x0000026b, + 0x0000027c, 0x0000027a, 0x000500b0, 0x00000014, 0x0000026f, 0x000003c7, 0x0000000c, 0x000400f6, 0x0000027d, + 0x0000027a, 0x00000000, 0x000400fa, 0x0000026f, 0x00000270, 0x0000027d, 0x000200f8, 0x00000270, 0x0003003e, + 0x000002d0, 0x0000037c, 0x00050041, 0x00000007, 0x000002d2, 0x000002d0, 0x000003c7, 0x0004003d, 0x00000006, + 0x00000273, 0x000002d2, 0x0003003e, 0x000002d3, 0x0000038d, 0x00050041, 0x00000007, 0x000002d5, 0x000002d3, + 0x000003c7, 0x0004003d, 0x00000006, 0x00000276, 0x000002d5, 0x000500ab, 0x00000014, 0x00000277, 0x00000273, + 0x00000276, 0x000300f7, 0x00000279, 0x00000000, 0x000400fa, 0x00000277, 0x00000278, 0x00000279, 0x000200f8, + 0x00000278, 0x000200f9, 0x0000027d, 0x000200f8, 0x00000279, 0x000200f9, 0x0000027a, 0x000200f8, 0x0000027a, + 0x00050080, 0x00000006, 0x0000027c, 0x000003c7, 0x00000060, 0x000200f9, 0x0000026c, 0x000200f8, 0x0000027d, + 0x000700f5, 0x00000014, 0x000003cb, 0x000003d0, 0x0000026c, 0x00000090, 0x00000278, 0x000700f5, 0x00000014, + 0x000003c8, 0x00000090, 0x0000026c, 0x00000094, 0x00000278, 0x000300f7, 0x0000027f, 0x00000000, 0x000400fa, + 0x000003c8, 0x00000280, 0x0000027f, 0x000200f8, 0x0000027f, 0x000200f9, 0x00000280, 0x000200f8, 0x00000280, + 0x000700f5, 0x00000014, 0x000003ca, 0x000003cb, 0x0000027d, 0x00000094, 0x0000027f, 0x000300f7, 0x000001d1, + 0x00000000, 0x000400fa, 0x000003ca, 0x000001cc, 0x000001d1, 0x000200f8, 0x000001cc, 0x000b0050, 0x0000000d, + 0x000003a7, 0x000001b4, 0x000001b6, 0x000001b8, 0x000001ba, 0x000001bc, 0x000001be, 0x000001c0, 0x000001c2, + 0x000300f7, 0x000002a1, 0x00000000, 0x000300fb, 0x00000042, 0x0000028c, 0x000200f8, 0x0000028c, 0x000200f9, + 0x0000028d, 0x000200f8, 0x0000028d, 0x000700f5, 0x00000006, 0x000003d6, 0x00000042, 0x0000028c, 0x0000029d, + 0x0000029b, 0x000500b0, 0x00000014, 0x00000290, 0x000003d6, 0x0000000c, 0x000400f6, 0x0000029e, 0x0000029b, + 0x00000000, 0x000400fa, 0x00000290, 0x00000291, 0x0000029e, 0x000200f8, 0x00000291, 0x0003003e, 0x000002ca, + 0x000003a7, 0x00050041, 0x00000007, 0x000002cc, 0x000002ca, 0x000003d6, 0x0004003d, 0x00000006, 0x00000294, + 0x000002cc, 0x0003003e, 0x000002cd, 0x00000097, 0x00050041, 0x00000007, 0x000002cf, 0x000002cd, 0x000003d6, + 0x0004003d, 0x00000006, 0x00000297, 0x000002cf, 0x000500ab, 0x00000014, 0x00000298, 0x00000294, 0x00000297, + 0x000300f7, 0x0000029a, 0x00000000, 0x000400fa, 0x00000298, 0x00000299, 0x0000029a, 0x000200f8, 0x00000299, + 0x000200f9, 0x0000029e, 0x000200f8, 0x0000029a, 0x000200f9, 0x0000029b, 0x000200f8, 0x0000029b, 0x00050080, + 0x00000006, 0x0000029d, 0x000003d6, 0x00000060, 0x000200f9, 0x0000028d, 0x000200f8, 0x0000029e, 0x000700f5, + 0x00000014, 0x000003da, 0x000003e2, 0x0000028d, 0x00000090, 0x00000299, 0x000700f5, 0x00000014, 0x000003d7, + 0x00000090, 0x0000028d, 0x00000094, 0x00000299, 0x000300f7, 0x000002a0, 0x00000000, 0x000400fa, 0x000003d7, + 0x000002a1, 0x000002a0, 0x000200f8, 0x000002a0, 0x000200f9, 0x000002a1, 0x000200f8, 0x000002a1, 0x000700f5, + 0x00000014, 0x000003d9, 0x000003da, 0x0000029e, 0x00000094, 0x000002a0, 0x000400a8, 0x00000014, 0x000001d0, + 0x000003d9, 0x000200f9, 0x000001d1, 0x000200f8, 0x000001d1, 0x000700f5, 0x00000014, 0x00000426, 0x000003e2, + 0x00000280, 0x000003d9, 0x000002a1, 0x000700f5, 0x00000014, 0x000001d2, 0x000003ca, 0x00000280, 0x000001d0, + 0x000002a1, 0x000300f7, 0x000001d6, 0x00000000, 0x000400fa, 0x000001d2, 0x000001d3, 0x000001d6, 0x000200f8, + 0x000001d3, 0x000200f9, 0x000001db, 0x000200f8, 0x000001d6, 0x000200f9, 0x000001d7, 0x000200f8, 0x000001d7, + 0x000200f9, 0x000001d8, 0x000200f8, 0x000001d8, 0x00050080, 0x00000006, 0x000001da, 0x00000197, 0x00000060, + 0x000200f9, 0x00000192, 0x000200f8, 0x000001db, 0x000700f5, 0x00000006, 0x00000402, 0x00000042, 0x000001c7, + 0x000001b4, 0x000001d3, 0x000700f5, 0x00000006, 0x00000400, 0x00000042, 0x000001c7, 0x000001b6, 0x000001d3, + 0x000700f5, 0x00000006, 0x000003fe, 0x00000042, 0x000001c7, 0x000001b8, 0x000001d3, 0x000700f5, 0x00000006, + 0x000003fc, 0x00000042, 0x000001c7, 0x000001ba, 0x000001d3, 0x000700f5, 0x00000006, 0x000003fa, 0x00000042, + 0x000001c7, 0x000001bc, 0x000001d3, 0x000700f5, 0x00000006, 0x000003f8, 0x00000042, 0x000001c7, 0x000001be, + 0x000001d3, 0x000700f5, 0x00000006, 0x000003f6, 0x00000042, 0x000001c7, 0x000001c0, 0x000001d3, 0x000700f5, + 0x00000006, 0x000003f4, 0x00000042, 0x000001c7, 0x000001c2, 0x000001d3, 0x000300f7, 0x000001dd, 0x00000000, + 0x000400fa, 0x00000094, 0x000001de, 0x000001dd, 0x000200f8, 0x000001dd, 0x000200f9, 0x000001de, 0x000200f8, + 0x000001de, 0x000900f5, 0x00000006, 0x00000401, 0x00000042, 0x0000018f, 0x00000402, 0x000001db, 0x00000402, + 0x000001dd, 0x000900f5, 0x00000006, 0x000003ff, 0x00000042, 0x0000018f, 0x00000400, 0x000001db, 0x00000400, + 0x000001dd, 0x000900f5, 0x00000006, 0x000003fd, 0x00000042, 0x0000018f, 0x000003fe, 0x000001db, 0x000003fe, + 0x000001dd, 0x000900f5, 0x00000006, 0x000003fb, 0x00000042, 0x0000018f, 0x000003fc, 0x000001db, 0x000003fc, + 0x000001dd, 0x000900f5, 0x00000006, 0x000003f9, 0x00000042, 0x0000018f, 0x000003fa, 0x000001db, 0x000003fa, + 0x000001dd, 0x000900f5, 0x00000006, 0x000003f7, 0x00000042, 0x0000018f, 0x000003f8, 0x000001db, 0x000003f8, + 0x000001dd, 0x000900f5, 0x00000006, 0x000003f5, 0x00000042, 0x0000018f, 0x000003f6, 0x000001db, 0x000003f6, + 0x000001dd, 0x000900f5, 0x00000006, 0x000003f3, 0x00000042, 0x0000018f, 0x000003f4, 0x000001db, 0x000003f4, + 0x000001dd, 0x000b0050, 0x0000000d, 0x00000362, 0x00000401, 0x000003ff, 0x000003fd, 0x000003fb, 0x000003f9, + 0x000003f7, 0x000003f5, 0x000003f3, 0x000300f7, 0x000002c2, 0x00000000, 0x000300fb, 0x00000042, 0x000002ad, + 0x000200f8, 0x000002ad, 0x000200f9, 0x000002ae, 0x000200f8, 0x000002ae, 0x000700f5, 0x00000006, 0x00000403, + 0x00000042, 0x000002ad, 0x000002be, 0x000002bc, 0x000500b0, 0x00000014, 0x000002b1, 0x00000403, 0x0000000c, + 0x000400f6, 0x000002bf, 0x000002bc, 0x00000000, 0x000400fa, 0x000002b1, 0x000002b2, 0x000002bf, 0x000200f8, + 0x000002b2, 0x0003003e, 0x000002c4, 0x00000362, 0x00050041, 0x00000007, 0x000002c6, 0x000002c4, 0x00000403, + 0x0004003d, 0x00000006, 0x000002b5, 0x000002c6, 0x0003003e, 0x000002c7, 0x00000097, 0x00050041, 0x00000007, + 0x000002c9, 0x000002c7, 0x00000403, 0x0004003d, 0x00000006, 0x000002b8, 0x000002c9, 0x000500ab, 0x00000014, + 0x000002b9, 0x000002b5, 0x000002b8, 0x000300f7, 0x000002bb, 0x00000000, 0x000400fa, 0x000002b9, 0x000002ba, + 0x000002bb, 0x000200f8, 0x000002ba, 0x000200f9, 0x000002bf, 0x000200f8, 0x000002bb, 0x000200f9, 0x000002bc, + 0x000200f8, 0x000002bc, 0x00050080, 0x00000006, 0x000002be, 0x00000403, 0x00000060, 0x000200f9, 0x000002ae, + 0x000200f8, 0x000002bf, 0x000700f5, 0x00000014, 0x00000407, 0x000003b7, 0x000002ae, 0x00000090, 0x000002ba, + 0x000700f5, 0x00000014, 0x00000404, 0x00000090, 0x000002ae, 0x00000094, 0x000002ba, 0x000300f7, 0x000002c1, + 0x00000000, 0x000400fa, 0x00000404, 0x000002c2, 0x000002c1, 0x000200f8, 0x000002c1, 0x000200f9, 0x000002c2, + 0x000200f8, 0x000002c2, 0x000700f5, 0x00000014, 0x00000406, 0x00000407, 0x000002bf, 0x00000094, 0x000002c1, + 0x000400a8, 0x00000014, 0x00000150, 0x00000406, 0x000300f7, 0x00000152, 0x00000000, 0x000400fa, 0x00000150, + 0x00000151, 0x00000152, 0x000200f8, 0x00000151, 0x00060041, 0x00000125, 0x00000153, 0x00000119, 0x00000083, + 0x000000a5, 0x0004003d, 0x0000010e, 0x00000154, 0x00000153, 0x00060041, 0x00000129, 0x00000156, 0x00000154, + 0x00000083, 0x0000010b, 0x0006003d, 0x00000110, 0x00000157, 0x00000156, 0x00000002, 0x00000008, 0x00050041, + 0x0000012c, 0x00000159, 0x00000157, 0x00000083, 0x00050041, 0x0000015b, 0x0000015c, 0x00000159, 0x00000083, + 0x00050041, 0x0000015e, 0x0000015f, 0x0000015c, 0x00000083, 0x0005003e, 0x0000015f, 0x00000401, 0x00000002, + 0x00000004, 0x00050041, 0x0000015e, 0x00000161, 0x0000015c, 0x00000060, 0x0005003e, 0x00000161, 0x000003ff, + 0x00000002, 0x00000004, 0x00050041, 0x0000015e, 0x00000163, 0x0000015c, 0x000000a5, 0x0005003e, 0x00000163, + 0x000003fd, 0x00000002, 0x00000004, 0x00050041, 0x0000015e, 0x00000165, 0x0000015c, 0x000000ca, 0x0005003e, + 0x00000165, 0x000003fb, 0x00000002, 0x00000004, 0x00050041, 0x0000015e, 0x00000167, 0x0000015c, 0x000000cd, + 0x0005003e, 0x00000167, 0x000003f9, 0x00000002, 0x00000004, 0x00050041, 0x0000015e, 0x00000169, 0x0000015c, + 0x000000d0, 0x0005003e, 0x00000169, 0x000003f7, 0x00000002, 0x00000004, 0x00050041, 0x0000015e, 0x0000016b, + 0x0000015c, 0x000000d3, 0x0005003e, 0x0000016b, 0x000003f5, 0x00000002, 0x00000004, 0x00050041, 0x0000015e, + 0x0000016d, 0x0000015c, 0x000000d6, 0x0005003e, 0x0000016d, 0x000003f3, 0x00000002, 0x00000004, 0x000200f9, + 0x00000152, 0x000200f8, 0x00000152, 0x000200f9, 0x0000016f, 0x000200f8, 0x0000016f, 0x000100fd, 0x00010038 }; -const std::array g_replacer_bda_comp = { - 0x03, 0x02, 0x23, 0x07, 0x00, 0x05, 0x01, 0x00, 0x0b, 0x00, 0x08, 0x00, 0x1b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x11, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00, 0xb6, 0x14, 0x00, 0x00, 0x11, 0x00, - 0x02, 0x00, 0xe3, 0x14, 0x00, 0x00, 0x0b, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00, 0x47, 0x4c, 0x53, 0x4c, 0x2e, - 0x73, 0x74, 0x64, 0x2e, 0x34, 0x35, 0x30, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x03, 0x00, 0xe4, 0x14, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x07, 0x00, 0x05, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6d, 0x61, 0x69, - 0x6e, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, 0x10, 0x00, 0x06, 0x00, 0x04, 0x00, - 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x03, 0x00, 0x02, 0x00, 0x00, 0x00, 0xcc, 0x01, 0x00, 0x00, 0x04, 0x00, 0x07, 0x00, 0x47, 0x4c, 0x5f, 0x45, - 0x58, 0x54, 0x5f, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x00, 0x04, 0x00, 0x08, 0x00, 0x47, 0x4c, 0x5f, 0x45, 0x58, 0x54, 0x5f, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, - 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x32, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x0a, 0x00, 0x47, - 0x4c, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x63, 0x70, 0x70, 0x5f, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x5f, - 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x00, 0x00, 0x04, 0x00, 0x08, - 0x00, 0x47, 0x4c, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x00, 0x05, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6d, - 0x61, 0x69, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x6d, 0x75, 0x72, 0x6d, - 0x75, 0x72, 0x5f, 0x33, 0x32, 0x5f, 0x73, 0x63, 0x72, 0x61, 0x6d, 0x62, 0x6c, 0x65, 0x28, 0x75, 0x31, 0x3b, 0x00, - 0x00, 0x05, 0x00, 0x03, 0x00, 0x09, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0x11, 0x00, - 0x00, 0x00, 0x6d, 0x75, 0x72, 0x6d, 0x75, 0x72, 0x33, 0x5f, 0x33, 0x32, 0x28, 0x75, 0x31, 0x5b, 0x32, 0x5d, 0x3b, - 0x75, 0x31, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x6b, 0x65, 0x79, 0x00, - 0x05, 0x00, 0x04, 0x00, 0x10, 0x00, 0x00, 0x00, 0x73, 0x65, 0x65, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, - 0x00, 0x13, 0x00, 0x00, 0x00, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, - 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x00, 0x06, 0x00, 0x05, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x17, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x65, 0x71, 0x75, 0x61, 0x6c, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, - 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x2d, 0x75, - 0x31, 0x5b, 0x32, 0x5d, 0x31, 0x3b, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, - 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x2d, 0x75, - 0x31, 0x5b, 0x32, 0x5d, 0x31, 0x3b, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x16, 0x00, 0x00, 0x00, 0x6c, 0x68, 0x73, - 0x00, 0x05, 0x00, 0x03, 0x00, 0x17, 0x00, 0x00, 0x00, 0x72, 0x68, 0x73, 0x00, 0x05, 0x00, 0x0e, 0x00, 0x1c, 0x00, - 0x00, 0x00, 0x69, 0x73, 0x5f, 0x6e, 0x75, 0x6c, 0x6c, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x62, 0x75, - 0x66, 0x66, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x5f, 0x74, 0x2d, 0x75, 0x31, 0x5b, 0x32, 0x5d, 0x31, 0x3b, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x1b, 0x00, 0x00, - 0x00, 0x68, 0x00, 0x00, 0x00, 0x05, 0x00, 0x0d, 0x00, 0x20, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x28, 0x73, - 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x2d, 0x75, 0x31, 0x5b, 0x32, 0x5d, 0x31, 0x3b, 0x00, - 0x05, 0x00, 0x06, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x23, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, - 0x70, 0x5f, 0x74, 0x00, 0x00, 0x00, 0x06, 0x00, 0x05, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x00, 0x06, 0x00, 0x05, 0x00, 0x23, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x73, 0x69, 0x7a, 0x65, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, 0x23, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, - 0x00, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0x25, 0x00, - 0x00, 0x00, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x00, 0x06, 0x00, 0x05, 0x00, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x06, 0x00, 0x26, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, - 0x68, 0x6d, 0x61, 0x70, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x74, 0x00, 0x00, 0x06, 0x00, 0x04, 0x00, 0x26, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x65, 0x79, 0x00, 0x06, 0x00, 0x05, 0x00, 0x26, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, 0x06, 0x00, 0x28, 0x00, 0x00, 0x00, - 0x48, 0x61, 0x73, 0x68, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x00, 0x00, 0x06, 0x00, 0x04, - 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x05, 0x00, 0x14, 0x00, 0x2c, 0x00, - 0x00, 0x00, 0x67, 0x65, 0x74, 0x28, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2d, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, - 0x70, 0x5f, 0x74, 0x2d, 0x31, 0x2d, 0x75, 0x31, 0x2d, 0x75, 0x31, 0x31, 0x3b, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, - 0x2d, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x5f, 0x74, 0x2d, 0x75, 0x31, 0x5b, 0x32, 0x5d, 0x31, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, - 0x04, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, 0x70, 0x00, 0x05, 0x00, 0x03, 0x00, 0x2b, - 0x00, 0x00, 0x00, 0x6b, 0x65, 0x79, 0x00, 0x05, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, - 0x05, 0x00, 0x03, 0x00, 0x41, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0x4b, 0x00, 0x00, - 0x00, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x4d, 0x00, 0x00, 0x00, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x61, 0x62, 0x6c, 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x69, - 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x86, 0x00, 0x00, 0x00, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x61, 0x62, 0x6c, - 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x8a, 0x00, 0x00, 0x00, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x61, 0x62, - 0x6c, 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x00, - 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0xac, 0x00, 0x00, 0x00, 0x69, 0x64, 0x78, 0x00, 0x05, 0x00, 0x06, 0x00, 0xb7, - 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, 0x70, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x74, 0x00, 0x00, - 0x06, 0x00, 0x04, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x65, 0x79, 0x00, 0x06, 0x00, 0x05, - 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x00, 0x00, 0x00, 0x05, 0x00, - 0x04, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x69, 0x74, 0x65, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0xd9, - 0x00, 0x00, 0x00, 0x67, 0x69, 0x64, 0x00, 0x05, 0x00, 0x08, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x67, 0x6c, 0x5f, 0x47, - 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x49, 0x6e, 0x76, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x00, 0x00, - 0x00, 0x05, 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x68, 0x61, 0x73, 0x68, 0x6d, 0x61, 0x70, 0x5f, 0x74, 0x00, - 0x00, 0x00, 0x06, 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x00, 0x06, 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x73, 0x69, 0x7a, 0x65, - 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x63, 0x61, 0x70, - 0x61, 0x63, 0x69, 0x74, 0x79, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x07, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x72, 0x65, - 0x70, 0x6c, 0x61, 0x63, 0x65, 0x72, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x5f, 0x74, 0x00, 0x00, 0x00, 0x06, - 0x00, 0x0a, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x64, - 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x00, 0x00, - 0x00, 0x06, 0x00, 0x07, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, - 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x00, 0x00, 0x00, 0x06, 0x00, 0x07, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x73, 0x00, 0x00, - 0x06, 0x00, 0x06, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x6e, 0x75, 0x6d, 0x5f, 0x68, 0x61, 0x6e, - 0x64, 0x6c, 0x65, 0x73, 0x00, 0x05, 0x00, 0x06, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x41, 0x72, 0x72, 0x61, 0x79, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x04, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x62, 0x75, 0x66, 0x66, - 0x65, 0x72, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x74, - 0x00, 0x06, 0x00, 0x05, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, - 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x44, 0x65, 0x76, - 0x69, 0x63, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x50, 0x74, 0x72, 0x00, 0x00, 0x06, 0x00, 0x09, 0x00, - 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x76, 0x69, - 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0xeb, 0x00, - 0x00, 0x00, 0x70, 0x63, 0x00, 0x00, 0x06, 0x00, 0x05, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0xed, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x05, 0x00, 0x04, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x00, 0x00, 0x05, 0x00, 0x03, - 0x00, 0x05, 0x01, 0x00, 0x00, 0x61, 0x72, 0x67, 0x00, 0x05, 0x00, 0x03, 0x00, 0x08, 0x01, 0x00, 0x00, 0x61, 0x72, - 0x67, 0x00, 0x47, 0x00, 0x04, 0x00, 0x24, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x48, - 0x00, 0x05, 0x00, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x48, 0x00, 0x05, 0x00, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x00, 0x05, 0x00, 0x26, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x08, 0x00, - 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0x27, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x47, - 0x00, 0x03, 0x00, 0x28, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x48, 0x00, 0x04, 0x00, 0x28, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x0b, 0x00, - 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x23, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, - 0x00, 0x23, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe3, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe3, 0x00, 0x00, - 0x00, 0x03, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0xe5, 0x00, - 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x48, 0x00, 0x04, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x48, 0x00, 0x05, 0x00, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x47, 0x00, 0x04, 0x00, 0xe7, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x48, 0x00, - 0x05, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, - 0x00, 0x03, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xe9, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, 0xea, 0x00, 0x00, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x02, 0x00, - 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x05, 0x01, 0x00, 0x00, 0xec, 0x14, 0x00, 0x00, 0x47, 0x00, 0x04, 0x00, - 0x1a, 0x01, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x13, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00, - 0x00, 0x21, 0x00, 0x03, 0x00, 0x03, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x15, 0x00, 0x04, 0x00, 0x06, 0x00, - 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x21, 0x00, 0x04, 0x00, 0x08, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, - 0x07, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, - 0x00, 0x1c, 0x00, 0x04, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x21, 0x00, - 0x05, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x1e, - 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x14, 0x00, 0x02, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x21, 0x00, 0x05, 0x00, 0x15, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, - 0x00, 0x21, 0x00, 0x04, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x21, 0x00, - 0x04, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x27, 0x00, 0x03, 0x00, 0x22, - 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0x1e, 0x00, 0x05, 0x00, 0x23, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x04, 0x00, 0x24, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x0c, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x03, 0x00, 0x25, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x1e, 0x00, - 0x04, 0x00, 0x26, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x03, 0x00, 0x27, - 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x03, 0x00, 0x28, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, - 0x20, 0x00, 0x04, 0x00, 0x22, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x21, 0x00, 0x05, - 0x00, 0x29, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x2b, 0x00, - 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x51, 0x2d, 0x9e, 0xcc, 0x15, 0x00, 0x04, 0x00, 0x32, - 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, - 0x33, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, - 0x00, 0x11, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x93, 0x35, - 0x87, 0x1b, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, - 0x00, 0x04, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, - 0x32, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, - 0x00, 0x57, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x5b, 0x00, - 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x5d, 0x00, 0x00, 0x00, 0x64, - 0x6b, 0x54, 0xe6, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, - 0x00, 0x32, 0x00, 0x00, 0x00, 0x66, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, - 0x00, 0x00, 0x6a, 0x00, 0x00, 0x00, 0x6b, 0xca, 0xeb, 0x85, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x71, - 0x00, 0x00, 0x00, 0x35, 0xae, 0xb2, 0xc2, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x85, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, - 0x00, 0x2a, 0x00, 0x03, 0x00, 0x14, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x29, 0x00, 0x03, 0x00, 0x14, 0x00, - 0x00, 0x00, 0x94, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x05, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x97, 0x00, 0x00, 0x00, 0x42, - 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x04, 0x00, 0x13, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, - 0x97, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0xa5, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, - 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x1e, 0x00, - 0x04, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xb8, - 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xbc, 0x00, 0x00, 0x00, - 0xe5, 0x14, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x17, 0x00, 0x04, 0x00, 0xda, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x03, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xdb, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xda, 0x00, - 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0xdb, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, - 0x00, 0x04, 0x00, 0xdd, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x05, 0x00, - 0xe1, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x27, 0x00, 0x03, - 0x00, 0xe2, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0x1e, 0x00, 0x06, 0x00, 0xe3, 0x00, 0x00, 0x00, 0xe1, 0x00, - 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x27, 0x00, 0x03, 0x00, 0xe4, - 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0x1d, 0x00, 0x03, 0x00, 0xe5, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, - 0x1e, 0x00, 0x03, 0x00, 0xe6, 0x00, 0x00, 0x00, 0xe5, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x04, 0x00, 0xe7, 0x00, 0x00, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x03, 0x00, 0xe8, 0x00, 0x00, 0x00, 0xe7, 0x00, - 0x00, 0x00, 0x1e, 0x00, 0x03, 0x00, 0xe9, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xe4, - 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0xe9, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x03, 0x00, 0xea, 0x00, 0x00, 0x00, - 0xe4, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xe2, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0xe6, 0x00, 0x00, - 0x00, 0x1e, 0x00, 0x03, 0x00, 0xeb, 0x00, 0x00, 0x00, 0xe3, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xec, 0x00, - 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xeb, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0xec, 0x00, 0x00, 0x00, 0xed, - 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x32, 0x00, 0x00, 0x00, 0xee, 0x00, 0x00, 0x00, - 0x03, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xef, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x20, 0x00, 0x04, 0x00, 0xf7, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x20, 0x00, - 0x04, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0xfe, - 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x01, 0x01, 0x00, 0x00, - 0xe5, 0x14, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x04, 0x01, 0x00, 0x00, 0x07, 0x00, 0x00, - 0x00, 0x23, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, 0x20, 0x00, - 0x00, 0x00, 0x2c, 0x00, 0x06, 0x00, 0xda, 0x00, 0x00, 0x00, 0x1a, 0x01, 0x00, 0x00, 0x19, 0x01, 0x00, 0x00, 0xb3, - 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x36, 0x00, 0x05, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x05, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, - 0x00, 0x07, 0x00, 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x85, 0x00, - 0x00, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x04, 0x01, 0x00, 0x00, 0x05, - 0x01, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x85, 0x00, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, - 0x07, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0xdd, 0x00, 0x00, 0x00, 0xde, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, - 0x00, 0x42, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, 0xde, 0x00, - 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0xd9, 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, - 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xef, 0x00, 0x00, 0x00, - 0xf0, 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0xee, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, - 0x00, 0x06, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, 0xae, 0x00, 0x05, 0x00, 0x14, 0x00, - 0x00, 0x00, 0xf2, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0xf1, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0xf4, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xf2, 0x00, 0x00, 0x00, 0xf3, 0x00, 0x00, 0x00, - 0xf4, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xf3, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x01, 0x00, 0xf8, 0x00, 0x02, - 0x00, 0xf4, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xf7, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0xed, 0x00, - 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0xe1, 0x00, 0x00, 0x00, 0xf9, - 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xfa, 0x00, 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, - 0xed, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0xe2, 0x00, 0x00, - 0x00, 0xfc, 0x00, 0x00, 0x00, 0xfb, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xfd, 0x00, - 0x00, 0x00, 0xd9, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xfe, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xfc, - 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x06, 0x00, 0xe4, 0x00, 0x00, 0x00, - 0x00, 0x01, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, - 0x00, 0x01, 0x01, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x3d, 0x00, - 0x06, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x03, 0x01, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, - 0x00, 0x00, 0x00, 0x90, 0x01, 0x04, 0x00, 0x23, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0xf9, 0x00, 0x00, 0x00, - 0x3e, 0x00, 0x03, 0x00, 0x05, 0x01, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x23, 0x00, 0x00, - 0x00, 0x07, 0x01, 0x00, 0x00, 0x05, 0x01, 0x00, 0x00, 0x90, 0x01, 0x04, 0x00, 0x13, 0x00, 0x00, 0x00, 0x09, 0x01, - 0x00, 0x00, 0x03, 0x01, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x08, 0x01, 0x00, 0x00, 0x09, 0x01, 0x00, 0x00, 0x3d, - 0x00, 0x04, 0x00, 0x13, 0x00, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x39, 0x00, 0x06, 0x00, - 0x13, 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0a, 0x01, 0x00, - 0x00, 0x3e, 0x00, 0x03, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x13, 0x00, - 0x00, 0x00, 0x0c, 0x01, 0x00, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0d, - 0x01, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x0c, 0x01, 0x00, 0x00, 0xa8, 0x00, 0x04, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x0e, 0x01, 0x00, 0x00, 0x0d, 0x01, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xfa, 0x00, 0x04, 0x00, 0x0e, 0x01, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0xf8, 0x00, - 0x02, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0xed, - 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0xa5, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0xe2, 0x00, 0x00, 0x00, - 0x12, 0x01, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, - 0x00, 0xd9, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0xfe, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, 0x00, 0x12, 0x01, - 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x3d, 0x00, 0x06, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x15, - 0x01, 0x00, 0x00, 0x14, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, - 0x13, 0x00, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, 0xf6, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x01, 0x01, 0x00, - 0x00, 0x17, 0x01, 0x00, 0x00, 0x15, 0x01, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x90, 0x01, 0x04, 0x00, 0xe8, 0x00, - 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, 0x16, 0x01, 0x00, 0x00, 0x3e, 0x00, 0x05, 0x00, 0x17, 0x01, 0x00, 0x00, 0x18, - 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x10, 0x01, 0x00, 0x00, - 0xf8, 0x00, 0x02, 0x00, 0x10, 0x01, 0x00, 0x00, 0xfd, 0x00, 0x01, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, 0x05, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x37, 0x00, - 0x03, 0x00, 0x07, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x3d, - 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, - 0x00, 0x09, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x31, 0x00, - 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x31, - 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, - 0x09, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, - 0x00, 0x36, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x34, 0x00, - 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x09, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x3d, - 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, - 0x00, 0x09, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x3c, 0x00, - 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, - 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, - 0x37, 0x00, 0x03, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x07, 0x00, 0x00, - 0x00, 0x10, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x12, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, - 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x41, - 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, - 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x4d, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, - 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x3e, 0x00, - 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x41, 0x00, 0x00, 0x00, 0x42, - 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x43, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x43, 0x00, 0x00, 0x00, - 0xf6, 0x00, 0x04, 0x00, 0x45, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, - 0x00, 0x47, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x47, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, - 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x49, - 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0x49, 0x00, 0x00, 0x00, - 0x44, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x44, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x4d, 0x00, - 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x07, 0x00, 0x00, 0x00, 0x4e, 0x00, 0x00, 0x00, 0x4d, - 0x00, 0x00, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, - 0x4e, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x3d, 0x00, - 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x05, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, - 0x3f, 0x00, 0x00, 0x00, 0x52, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, - 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, 0x53, 0x00, - 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x3f, - 0x00, 0x00, 0x00, 0xc2, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, - 0x57, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, - 0x00, 0x58, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, 0x3d, 0x00, - 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x5a, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x5a, 0x00, 0x00, 0x00, 0x5b, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x5d, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, - 0x00, 0x3f, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x46, 0x00, 0x00, 0x00, 0xf8, 0x00, - 0x02, 0x00, 0x46, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, 0x41, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x5f, 0x00, 0x00, 0x00, - 0x60, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x41, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, - 0x00, 0x43, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x45, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, - 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x64, - 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, - 0x64, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, - 0x00, 0xc2, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x66, 0x00, - 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, - 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, - 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, - 0x00, 0x6b, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6c, 0x00, - 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x6a, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x6c, - 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, - 0xc2, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, - 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, - 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x3e, - 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, - 0x72, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, - 0x00, 0x72, 0x00, 0x00, 0x00, 0x71, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x73, 0x00, - 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc2, - 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x66, 0x00, 0x00, 0x00, - 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x05, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x77, 0x00, 0x00, 0x00, 0x76, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, 0x3e, 0x00, - 0x03, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x77, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x78, - 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x78, 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, - 0x36, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, - 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, - 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x19, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x85, 0x00, 0x00, 0x00, - 0x86, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x85, 0x00, 0x00, 0x00, 0x8a, 0x00, 0x00, - 0x00, 0x07, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0xf9, 0x00, - 0x02, 0x00, 0x7c, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x7c, 0x00, 0x00, 0x00, 0xf6, 0x00, 0x04, 0x00, 0x7e, - 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x80, 0x00, 0x00, 0x00, - 0xf8, 0x00, 0x02, 0x00, 0x80, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, - 0x00, 0x7b, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0x81, 0x00, - 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0x82, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x7e, - 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, - 0x84, 0x00, 0x00, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x86, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, - 0x00, 0x41, 0x00, 0x06, 0x00, 0x07, 0x00, 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, 0x83, 0x00, - 0x00, 0x00, 0x84, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x87, - 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, 0x7b, 0x00, 0x00, 0x00, - 0x3e, 0x00, 0x03, 0x00, 0x8a, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0x07, 0x00, 0x00, - 0x00, 0x8b, 0x00, 0x00, 0x00, 0x8a, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, 0x3d, 0x00, - 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, 0xab, 0x00, 0x05, 0x00, 0x14, - 0x00, 0x00, 0x00, 0x8d, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, - 0x8f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0x8d, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, - 0x00, 0x8f, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x8e, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x90, 0x00, - 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x8f, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0x7f, 0x00, 0x00, 0x00, 0xf8, - 0x00, 0x02, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, 0x00, - 0x7b, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x93, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, - 0x00, 0x60, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x93, 0x00, 0x00, 0x00, 0xf9, 0x00, - 0x02, 0x00, 0x7c, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x7e, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x94, - 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, - 0x00, 0xf8, 0x00, 0x02, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x39, 0x00, 0x06, 0x00, 0x14, 0x00, 0x00, 0x00, 0x99, 0x00, - 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x99, - 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, - 0x00, 0xf8, 0x00, 0x02, 0x00, 0x21, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x9d, 0x00, - 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x51, 0x00, 0x05, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, 0x1f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, - 0x39, 0x00, 0x06, 0x00, 0x06, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, - 0x00, 0x9d, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00, 0x36, 0x00, - 0x05, 0x00, 0x13, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x37, - 0x00, 0x03, 0x00, 0x23, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x37, 0x00, 0x03, 0x00, 0x13, 0x00, 0x00, 0x00, - 0x2b, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, - 0x00, 0xac, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x04, 0x00, 0xb8, 0x00, 0x00, 0x00, 0xb9, 0x00, - 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, 0x1c, - 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0xa8, 0x00, 0x04, 0x00, 0x14, 0x00, 0x00, 0x00, 0xa2, 0x00, 0x00, 0x00, - 0xa1, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0xa4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, - 0x00, 0xa2, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xa3, 0x00, - 0x00, 0x00, 0x51, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0xaa, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x00, - 0x42, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xa4, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xa4, 0x00, 0x00, - 0x00, 0xf5, 0x00, 0x07, 0x00, 0x14, 0x00, 0x00, 0x00, 0xa8, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, 0x00, 0x2d, 0x00, - 0x00, 0x00, 0xa7, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xa8, 0x00, 0x00, 0x00, 0xa9, 0x00, 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, - 0xf8, 0x00, 0x02, 0x00, 0xa9, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0x98, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, - 0x00, 0xaa, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, 0x20, 0x00, - 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0xac, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, 0x00, 0xf9, - 0x00, 0x02, 0x00, 0xae, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xae, 0x00, 0x00, 0x00, 0xf6, 0x00, 0x04, 0x00, - 0xb0, 0x00, 0x00, 0x00, 0xb1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xaf, 0x00, 0x00, - 0x00, 0xf8, 0x00, 0x02, 0x00, 0xaf, 0x00, 0x00, 0x00, 0x51, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xb2, 0x00, - 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x82, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xb4, - 0x00, 0x00, 0x00, 0xb2, 0x00, 0x00, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, - 0xb5, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xb6, 0x00, 0x00, - 0x00, 0xb5, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0xac, 0x00, 0x00, 0x00, 0xb6, 0x00, - 0x00, 0x00, 0x51, 0x00, 0x05, 0x00, 0x22, 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, - 0x41, 0x00, 0x06, 0x00, 0xbc, 0x00, 0x00, 0x00, 0xbd, 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, - 0x00, 0xbb, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x06, 0x00, 0x26, 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, 0xbd, 0x00, - 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x90, 0x01, 0x04, 0x00, 0xb7, 0x00, 0x00, 0x00, 0xbf, - 0x00, 0x00, 0x00, 0xbe, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x03, 0x00, 0xb9, 0x00, 0x00, 0x00, 0xbf, 0x00, 0x00, 0x00, - 0x41, 0x00, 0x05, 0x00, 0x85, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, - 0x00, 0x3d, 0x00, 0x04, 0x00, 0x13, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x39, 0x00, - 0x05, 0x00, 0x14, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0xf7, - 0x00, 0x03, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xc2, 0x00, 0x00, 0x00, - 0xc3, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, - 0x00, 0x98, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xc6, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x85, 0x00, - 0x00, 0x00, 0xc7, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x13, - 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xc7, 0x00, 0x00, 0x00, 0x39, 0x00, 0x06, 0x00, 0x14, 0x00, 0x00, 0x00, - 0xc9, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, - 0x00, 0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xc9, 0x00, 0x00, 0x00, 0xca, 0x00, - 0x00, 0x00, 0xcb, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xca, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x85, - 0x00, 0x00, 0x00, 0xcc, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, - 0x13, 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, 0xcc, 0x00, 0x00, 0x00, 0x39, 0x00, 0x05, 0x00, 0x14, 0x00, 0x00, - 0x00, 0xce, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0xcd, 0x00, 0x00, 0x00, 0xa8, 0x00, 0x04, 0x00, 0x14, 0x00, - 0x00, 0x00, 0xcf, 0x00, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xcb, 0x00, 0x00, 0x00, 0xf8, - 0x00, 0x02, 0x00, 0xcb, 0x00, 0x00, 0x00, 0xf5, 0x00, 0x07, 0x00, 0x14, 0x00, 0x00, 0x00, 0xd0, 0x00, 0x00, 0x00, - 0xc9, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x00, 0xcf, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00, 0xf7, 0x00, 0x03, - 0x00, 0xd2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x04, 0x00, 0xd0, 0x00, 0x00, 0x00, 0xd1, 0x00, - 0x00, 0x00, 0xd2, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xd1, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x85, - 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0xb9, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, - 0x13, 0x00, 0x00, 0x00, 0xd4, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x02, 0x00, 0xd4, 0x00, 0x00, - 0x00, 0xf8, 0x00, 0x02, 0x00, 0xd2, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xc4, 0x00, 0x00, 0x00, 0xf8, 0x00, - 0x02, 0x00, 0xc4, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xb1, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xb1, - 0x00, 0x00, 0x00, 0x3d, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, - 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0xd6, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, - 0x00, 0x3e, 0x00, 0x03, 0x00, 0xac, 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0xf9, 0x00, 0x02, 0x00, 0xae, 0x00, - 0x00, 0x00, 0xf8, 0x00, 0x02, 0x00, 0xb0, 0x00, 0x00, 0x00, 0xff, 0x00, 0x01, 0x00, 0x38, 0x00, 0x01, 0x00 +const std::array g_replacer_bda_comp = { + 0x07230203, 0x00010300, 0x0008000b, 0x00000358, 0x00000000, 0x00020011, 0x00000001, 0x00020011, 0x000014b6, + 0x00020011, 0x000014e3, 0x0008000a, 0x5f565053, 0x5f545845, 0x63736564, 0x74706972, 0x695f726f, 0x7865646e, + 0x00676e69, 0x0009000a, 0x5f565053, 0x5f52484b, 0x73796870, 0x6c616369, 0x6f74735f, 0x65676172, 0x6675625f, + 0x00726566, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x000014e4, + 0x00000001, 0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x000000eb, 0x00060010, 0x00000004, + 0x00000011, 0x00000020, 0x00000001, 0x00000001, 0x00030003, 0x00000002, 0x000001cc, 0x00070004, 0x455f4c47, + 0x625f5458, 0x65666675, 0x65725f72, 0x65726566, 0x0065636e, 0x00080004, 0x455f4c47, 0x625f5458, 0x65666675, + 0x65725f72, 0x65726566, 0x3265636e, 0x00000000, 0x000a0004, 0x475f4c47, 0x4c474f4f, 0x70635f45, 0x74735f70, + 0x5f656c79, 0x656e696c, 0x7269645f, 0x69746365, 0x00006576, 0x00080004, 0x475f4c47, 0x4c474f4f, 0x6e695f45, + 0x64756c63, 0x69645f65, 0x74636572, 0x00657669, 0x00040005, 0x00000004, 0x6e69616d, 0x00000000, 0x00080005, + 0x00000025, 0x66667562, 0x645f7265, 0x63697665, 0x64615f65, 0x73657264, 0x00745f73, 0x00050006, 0x00000025, + 0x00000000, 0x61746164, 0x00000000, 0x00060005, 0x00000026, 0x68736168, 0x5f70616d, 0x6d657469, 0x0000745f, + 0x00040006, 0x00000026, 0x00000000, 0x0079656b, 0x00050006, 0x00000026, 0x00000001, 0x756c6176, 0x00000065, + 0x00060005, 0x00000028, 0x68736148, 0x5370614d, 0x61726f74, 0x00006567, 0x00040006, 0x00000028, 0x00000000, + 0x00000076, 0x00080005, 0x000000eb, 0x475f6c67, 0x61626f6c, 0x766e496c, 0x7461636f, 0x496e6f69, 0x00000044, + 0x00050005, 0x000000f0, 0x68736168, 0x5f70616d, 0x00000074, 0x00050006, 0x000000f0, 0x00000000, 0x726f7473, + 0x00656761, 0x00050006, 0x000000f0, 0x00000001, 0x657a6973, 0x00000000, 0x00060006, 0x000000f0, 0x00000002, + 0x61706163, 0x79746963, 0x00000000, 0x00070005, 0x000000f2, 0x6c706572, 0x72656361, 0x7261705f, 0x5f736d61, + 0x00000074, 0x000a0006, 0x000000f2, 0x00000000, 0x66667562, 0x645f7265, 0x63697665, 0x64615f65, 0x73657264, + 0x616d5f73, 0x00000070, 0x00070006, 0x000000f2, 0x00000001, 0x75706e69, 0x61685f74, 0x656c646e, 0x00000073, + 0x00070006, 0x000000f2, 0x00000002, 0x7074756f, 0x685f7475, 0x6c646e61, 0x00007365, 0x00060006, 0x000000f2, + 0x00000003, 0x5f6d756e, 0x646e6168, 0x0073656c, 0x00060005, 0x000000f5, 0x72646441, 0x41737365, 0x79617272, + 0x00000000, 0x00040006, 0x000000f5, 0x00000000, 0x00000076, 0x00080005, 0x000000f7, 0x66667562, 0x645f7265, + 0x63697665, 0x64615f65, 0x73657264, 0x00745f73, 0x00050006, 0x000000f7, 0x00000000, 0x61746164, 0x00000000, + 0x00080005, 0x000000f8, 0x66667542, 0x65447265, 0x65636976, 0x72646441, 0x50737365, 0x00007274, 0x00090006, + 0x000000f8, 0x00000000, 0x66667562, 0x645f7265, 0x63697665, 0x64615f65, 0x73657264, 0x00000073, 0x00030005, + 0x000000fa, 0x00006370, 0x00050006, 0x000000fa, 0x00000000, 0x61726170, 0x0000736d, 0x00030005, 0x000000fc, + 0x00000000, 0x00040047, 0x00000024, 0x00000006, 0x00000004, 0x00050048, 0x00000025, 0x00000000, 0x00000023, + 0x00000000, 0x00050048, 0x00000026, 0x00000000, 0x00000023, 0x00000000, 0x00050048, 0x00000026, 0x00000001, + 0x00000023, 0x00000008, 0x00040047, 0x00000027, 0x00000006, 0x00000010, 0x00030047, 0x00000028, 0x00000002, + 0x00040048, 0x00000028, 0x00000000, 0x00000018, 0x00050048, 0x00000028, 0x00000000, 0x00000023, 0x00000000, + 0x00040047, 0x000000eb, 0x0000000b, 0x0000001c, 0x00050048, 0x000000f0, 0x00000000, 0x00000023, 0x00000000, + 0x00050048, 0x000000f0, 0x00000001, 0x00000023, 0x00000008, 0x00050048, 0x000000f0, 0x00000002, 0x00000023, + 0x0000000c, 0x00050048, 0x000000f2, 0x00000000, 0x00000023, 0x00000000, 0x00050048, 0x000000f2, 0x00000001, + 0x00000023, 0x00000010, 0x00050048, 0x000000f2, 0x00000002, 0x00000023, 0x00000018, 0x00050048, 0x000000f2, + 0x00000003, 0x00000023, 0x00000020, 0x00040047, 0x000000f4, 0x00000006, 0x00000008, 0x00030047, 0x000000f5, + 0x00000002, 0x00040048, 0x000000f5, 0x00000000, 0x00000018, 0x00050048, 0x000000f5, 0x00000000, 0x00000023, + 0x00000000, 0x00040047, 0x000000f6, 0x00000006, 0x00000004, 0x00050048, 0x000000f7, 0x00000000, 0x00000023, + 0x00000000, 0x00030047, 0x000000f8, 0x00000002, 0x00050048, 0x000000f8, 0x00000000, 0x00000023, 0x00000000, + 0x00030047, 0x000000fa, 0x00000002, 0x00050048, 0x000000fa, 0x00000000, 0x00000023, 0x00000000, 0x00040047, + 0x0000013b, 0x0000000b, 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00040015, + 0x00000006, 0x00000020, 0x00000000, 0x00040020, 0x00000007, 0x00000007, 0x00000006, 0x0004002b, 0x00000006, + 0x0000000c, 0x00000002, 0x0004001c, 0x0000000d, 0x00000006, 0x0000000c, 0x00020014, 0x00000014, 0x00030027, + 0x00000022, 0x000014e5, 0x0004001c, 0x00000024, 0x00000006, 0x0000000c, 0x0003001e, 0x00000025, 0x00000024, + 0x0004001e, 0x00000026, 0x00000025, 0x00000025, 0x0003001d, 0x00000027, 0x00000026, 0x0003001e, 0x00000028, + 0x00000027, 0x00040020, 0x00000022, 0x000014e5, 0x00000028, 0x0004002b, 0x00000006, 0x0000002e, 0xcc9e2d51, + 0x00040015, 0x00000032, 0x00000020, 0x00000001, 0x0004002b, 0x00000032, 0x00000033, 0x0000000f, 0x0004002b, + 0x00000032, 0x00000036, 0x00000011, 0x0004002b, 0x00000006, 0x00000039, 0x1b873593, 0x0004002b, 0x00000006, + 0x00000042, 0x00000000, 0x00040020, 0x0000004c, 0x00000007, 0x0000000d, 0x0004002b, 0x00000032, 0x00000054, + 0x0000000d, 0x0004002b, 0x00000032, 0x00000057, 0x00000013, 0x0004002b, 0x00000006, 0x0000005b, 0x00000005, + 0x0004002b, 0x00000006, 0x0000005d, 0xe6546b64, 0x0004002b, 0x00000032, 0x00000060, 0x00000001, 0x0004002b, + 0x00000006, 0x00000062, 0x00000008, 0x0004002b, 0x00000032, 0x00000066, 0x00000010, 0x0004002b, 0x00000006, + 0x0000006a, 0x85ebca6b, 0x0004002b, 0x00000006, 0x00000071, 0xc2b2ae35, 0x0004002b, 0x00000032, 0x00000083, + 0x00000000, 0x0003002a, 0x00000014, 0x00000090, 0x00030029, 0x00000014, 0x00000094, 0x0005002c, 0x0000000d, + 0x00000097, 0x00000042, 0x00000042, 0x0004002b, 0x00000032, 0x000000a5, 0x00000002, 0x0004002b, 0x00000006, + 0x000000b3, 0x00000001, 0x00040020, 0x000000bc, 0x000014e5, 0x00000026, 0x00040017, 0x000000e9, 0x00000006, + 0x00000003, 0x00040020, 0x000000ea, 0x00000001, 0x000000e9, 0x0004003b, 0x000000ea, 0x000000eb, 0x00000001, + 0x00040020, 0x000000ec, 0x00000001, 0x00000006, 0x0005001e, 0x000000f0, 0x00000022, 0x00000006, 0x00000006, + 0x00030027, 0x000000f1, 0x000014e5, 0x0006001e, 0x000000f2, 0x000000f0, 0x000000f1, 0x000000f1, 0x00000006, + 0x00030027, 0x000000f3, 0x000014e5, 0x0003001d, 0x000000f4, 0x000000f3, 0x0003001e, 0x000000f5, 0x000000f4, + 0x0004001c, 0x000000f6, 0x00000006, 0x0000000c, 0x0003001e, 0x000000f7, 0x000000f6, 0x0003001e, 0x000000f8, + 0x000000f7, 0x00040020, 0x000000f3, 0x000014e5, 0x000000f8, 0x00040020, 0x000000f1, 0x000014e5, 0x000000f5, + 0x0003001e, 0x000000fa, 0x000000f2, 0x00040020, 0x000000fb, 0x00000009, 0x000000fa, 0x0004003b, 0x000000fb, + 0x000000fc, 0x00000009, 0x0004002b, 0x00000032, 0x000000fd, 0x00000003, 0x00040020, 0x000000fe, 0x00000009, + 0x00000006, 0x00040020, 0x00000106, 0x00000009, 0x000000f0, 0x00040020, 0x00000109, 0x00000009, 0x000000f1, + 0x00040020, 0x0000010d, 0x000014e5, 0x000000f3, 0x00040020, 0x00000110, 0x000014e5, 0x000000f7, 0x00040020, + 0x00000133, 0x000014e5, 0x000000f6, 0x00040020, 0x00000136, 0x000014e5, 0x00000006, 0x0004002b, 0x00000006, + 0x0000013a, 0x00000020, 0x0006002c, 0x000000e9, 0x0000013b, 0x0000013a, 0x000000b3, 0x000000b3, 0x00030001, + 0x00000014, 0x000002f4, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, 0x000200f8, 0x00000005, + 0x0004003b, 0x0000004c, 0x00000294, 0x00000007, 0x0004003b, 0x0000004c, 0x00000291, 0x00000007, 0x0004003b, + 0x0000004c, 0x0000028e, 0x00000007, 0x0004003b, 0x0000004c, 0x0000028b, 0x00000007, 0x0004003b, 0x0000004c, + 0x00000288, 0x00000007, 0x0004003b, 0x0000004c, 0x00000285, 0x00000007, 0x0004003b, 0x0000004c, 0x00000282, + 0x00000007, 0x0004003b, 0x0000004c, 0x0000027f, 0x00000007, 0x0004003b, 0x0000004c, 0x0000027c, 0x00000007, + 0x0004003b, 0x0000004c, 0x00000279, 0x00000007, 0x0004003b, 0x0000004c, 0x000001be, 0x00000007, 0x000300f7, + 0x0000013c, 0x00000000, 0x000300fb, 0x00000042, 0x0000013d, 0x000200f8, 0x0000013d, 0x00050041, 0x000000ec, + 0x000000ed, 0x000000eb, 0x00000042, 0x0004003d, 0x00000006, 0x000000ee, 0x000000ed, 0x00060041, 0x000000fe, + 0x000000ff, 0x000000fc, 0x00000083, 0x000000fd, 0x0004003d, 0x00000006, 0x00000100, 0x000000ff, 0x000500ae, + 0x00000014, 0x00000101, 0x000000ee, 0x00000100, 0x000300f7, 0x00000103, 0x00000000, 0x000400fa, 0x00000101, + 0x00000102, 0x00000103, 0x000200f8, 0x00000102, 0x000200f9, 0x0000013c, 0x000200f8, 0x00000103, 0x00060041, + 0x00000106, 0x00000107, 0x000000fc, 0x00000083, 0x00000083, 0x0004003d, 0x000000f0, 0x00000108, 0x00000107, + 0x00060041, 0x00000109, 0x0000010a, 0x000000fc, 0x00000083, 0x00000060, 0x0004003d, 0x000000f1, 0x0000010b, + 0x0000010a, 0x00060041, 0x0000010d, 0x0000010e, 0x0000010b, 0x00000083, 0x000000ee, 0x0006003d, 0x000000f3, + 0x0000010f, 0x0000010e, 0x00000002, 0x00000008, 0x00050041, 0x00000110, 0x00000111, 0x0000010f, 0x00000083, + 0x0006003d, 0x000000f7, 0x00000112, 0x00000111, 0x00000002, 0x00000010, 0x00050051, 0x00000022, 0x00000115, + 0x00000108, 0x00000000, 0x00050051, 0x00000006, 0x0000011a, 0x00000108, 0x00000002, 0x00050051, 0x000000f6, + 0x0000011e, 0x00000112, 0x00000000, 0x00050051, 0x00000006, 0x00000120, 0x0000011e, 0x00000000, 0x00050051, + 0x00000006, 0x00000122, 0x0000011e, 0x00000001, 0x00050050, 0x0000000d, 0x000002dd, 0x00000120, 0x00000122, + 0x000300f7, 0x00000193, 0x00000000, 0x000300fb, 0x00000042, 0x00000154, 0x000200f8, 0x00000154, 0x000300f7, + 0x000001b4, 0x00000000, 0x000300fb, 0x00000042, 0x0000019f, 0x000200f8, 0x0000019f, 0x000200f9, 0x000001a0, + 0x000200f8, 0x000001a0, 0x000700f5, 0x00000006, 0x000002ee, 0x00000042, 0x0000019f, 0x000001b0, 0x000001ae, + 0x000500b0, 0x00000014, 0x000001a3, 0x000002ee, 0x0000000c, 0x000400f6, 0x000001b1, 0x000001ae, 0x00000000, + 0x000400fa, 0x000001a3, 0x000001a4, 0x000001b1, 0x000200f8, 0x000001a4, 0x0003003e, 0x00000291, 0x000002dd, + 0x00050041, 0x00000007, 0x00000293, 0x00000291, 0x000002ee, 0x0004003d, 0x00000006, 0x000001a7, 0x00000293, + 0x0003003e, 0x00000294, 0x00000097, 0x00050041, 0x00000007, 0x00000296, 0x00000294, 0x000002ee, 0x0004003d, + 0x00000006, 0x000001aa, 0x00000296, 0x000500ab, 0x00000014, 0x000001ab, 0x000001a7, 0x000001aa, 0x000300f7, + 0x000001ad, 0x00000000, 0x000400fa, 0x000001ab, 0x000001ac, 0x000001ad, 0x000200f8, 0x000001ac, 0x000200f9, + 0x000001b1, 0x000200f8, 0x000001ad, 0x000200f9, 0x000001ae, 0x000200f8, 0x000001ae, 0x00050080, 0x00000006, + 0x000001b0, 0x000002ee, 0x00000060, 0x000200f9, 0x000001a0, 0x000200f8, 0x000001b1, 0x000700f5, 0x00000014, + 0x000002f2, 0x000002f4, 0x000001a0, 0x00000090, 0x000001ac, 0x000700f5, 0x00000014, 0x000002ef, 0x00000090, + 0x000001a0, 0x00000094, 0x000001ac, 0x000300f7, 0x000001b3, 0x00000000, 0x000400fa, 0x000002ef, 0x000001b4, + 0x000001b3, 0x000200f8, 0x000001b3, 0x000200f9, 0x000001b4, 0x000200f8, 0x000001b4, 0x000700f5, 0x00000014, + 0x000002f1, 0x000002f2, 0x000001b1, 0x00000094, 0x000001b3, 0x000400a8, 0x00000014, 0x00000156, 0x000002f1, + 0x000300f7, 0x0000015a, 0x00000000, 0x000400fa, 0x00000156, 0x00000157, 0x0000015a, 0x000200f8, 0x00000157, + 0x000500aa, 0x00000014, 0x00000159, 0x0000011a, 0x00000042, 0x000200f9, 0x0000015a, 0x000200f8, 0x0000015a, + 0x000700f5, 0x00000014, 0x0000015b, 0x000002f1, 0x000001b4, 0x00000159, 0x00000157, 0x000300f7, 0x0000015d, + 0x00000000, 0x000400fa, 0x0000015b, 0x0000015c, 0x0000015d, 0x000200f8, 0x0000015c, 0x000200f9, 0x00000193, + 0x000200f8, 0x0000015d, 0x000200f9, 0x000001c2, 0x000200f8, 0x000001c2, 0x000700f5, 0x00000006, 0x000002f6, + 0x00000042, 0x0000015d, 0x000001d4, 0x000001c6, 0x000700f5, 0x00000006, 0x000002f5, 0x00000042, 0x0000015d, + 0x000001d7, 0x000001c6, 0x000500b0, 0x00000014, 0x000001c5, 0x000002f5, 0x0000000c, 0x000400f6, 0x000001d8, + 0x000001c6, 0x00000000, 0x000400fa, 0x000001c5, 0x000001c6, 0x000001d8, 0x000200f8, 0x000001c6, 0x0003003e, + 0x000001be, 0x000002dd, 0x00050041, 0x00000007, 0x000001c8, 0x000001be, 0x000002f5, 0x0004003d, 0x00000006, + 0x000001c9, 0x000001c8, 0x00050084, 0x00000006, 0x000001ef, 0x000001c9, 0x0000002e, 0x000500c4, 0x00000006, + 0x000001f1, 0x000001ef, 0x00000033, 0x000500c2, 0x00000006, 0x000001f3, 0x000001ef, 0x00000036, 0x000500c5, + 0x00000006, 0x000001f4, 0x000001f1, 0x000001f3, 0x00050084, 0x00000006, 0x000001f6, 0x000001f4, 0x00000039, + 0x000500c6, 0x00000006, 0x000001cc, 0x000002f6, 0x000001f6, 0x000500c4, 0x00000006, 0x000001ce, 0x000001cc, + 0x00000054, 0x000500c2, 0x00000006, 0x000001d0, 0x000001cc, 0x00000057, 0x000500c5, 0x00000006, 0x000001d1, + 0x000001ce, 0x000001d0, 0x00050084, 0x00000006, 0x000001d3, 0x000001d1, 0x0000005b, 0x00050080, 0x00000006, + 0x000001d4, 0x000001d3, 0x0000005d, 0x00050080, 0x00000006, 0x000001d7, 0x000002f5, 0x00000060, 0x000200f9, + 0x000001c2, 0x000200f8, 0x000001d8, 0x000500c6, 0x00000006, 0x000001da, 0x000002f6, 0x00000062, 0x000500c2, + 0x00000006, 0x000001dc, 0x000001da, 0x00000066, 0x000500c6, 0x00000006, 0x000001de, 0x000001da, 0x000001dc, + 0x00050084, 0x00000006, 0x000001e0, 0x000001de, 0x0000006a, 0x000500c2, 0x00000006, 0x000001e2, 0x000001e0, + 0x00000054, 0x000500c6, 0x00000006, 0x000001e4, 0x000001e0, 0x000001e2, 0x00050084, 0x00000006, 0x000001e6, + 0x000001e4, 0x00000071, 0x000500c2, 0x00000006, 0x000001e8, 0x000001e6, 0x00000066, 0x000500c6, 0x00000006, + 0x000001ea, 0x000001e6, 0x000001e8, 0x000200f9, 0x0000015f, 0x000200f8, 0x0000015f, 0x000700f5, 0x00000014, + 0x0000031f, 0x000002f4, 0x000001d8, 0x00000357, 0x0000018d, 0x000700f5, 0x00000014, 0x0000030d, 0x000002f4, + 0x000001d8, 0x00000307, 0x0000018d, 0x000700f5, 0x00000014, 0x000002fe, 0x000002f4, 0x000001d8, 0x000002fb, + 0x0000018d, 0x000700f5, 0x00000006, 0x000002f7, 0x000001ea, 0x000001d8, 0x0000018f, 0x0000018d, 0x000400f6, + 0x00000190, 0x0000018d, 0x00000000, 0x000200f9, 0x00000160, 0x000200f8, 0x00000160, 0x00050082, 0x00000006, + 0x00000162, 0x0000011a, 0x000000b3, 0x000500c7, 0x00000006, 0x00000164, 0x000002f7, 0x00000162, 0x00060041, + 0x000000bc, 0x00000167, 0x00000115, 0x00000083, 0x00000164, 0x0006003d, 0x00000026, 0x00000168, 0x00000167, + 0x00000002, 0x00000010, 0x00050051, 0x00000025, 0x00000169, 0x00000168, 0x00000000, 0x00050051, 0x00000024, + 0x0000016b, 0x00000169, 0x00000000, 0x00050051, 0x00000006, 0x0000016d, 0x0000016b, 0x00000000, 0x00050051, + 0x00000006, 0x0000016f, 0x0000016b, 0x00000001, 0x00050051, 0x00000025, 0x00000171, 0x00000168, 0x00000001, + 0x00050051, 0x00000024, 0x00000173, 0x00000171, 0x00000000, 0x00050051, 0x00000006, 0x00000175, 0x00000173, + 0x00000000, 0x00050051, 0x00000006, 0x00000177, 0x00000173, 0x00000001, 0x00050050, 0x0000000d, 0x000002e2, + 0x0000016d, 0x0000016f, 0x000300f7, 0x00000217, 0x00000000, 0x000300fb, 0x00000042, 0x00000202, 0x000200f8, + 0x00000202, 0x000200f9, 0x00000203, 0x000200f8, 0x00000203, 0x000700f5, 0x00000006, 0x000002f8, 0x00000042, + 0x00000202, 0x00000213, 0x00000211, 0x000500b0, 0x00000014, 0x00000206, 0x000002f8, 0x0000000c, 0x000400f6, + 0x00000214, 0x00000211, 0x00000000, 0x000400fa, 0x00000206, 0x00000207, 0x00000214, 0x000200f8, 0x00000207, + 0x0003003e, 0x0000028b, 0x000002e2, 0x00050041, 0x00000007, 0x0000028d, 0x0000028b, 0x000002f8, 0x0004003d, + 0x00000006, 0x0000020a, 0x0000028d, 0x0003003e, 0x0000028e, 0x00000097, 0x00050041, 0x00000007, 0x00000290, + 0x0000028e, 0x000002f8, 0x0004003d, 0x00000006, 0x0000020d, 0x00000290, 0x000500ab, 0x00000014, 0x0000020e, + 0x0000020a, 0x0000020d, 0x000300f7, 0x00000210, 0x00000000, 0x000400fa, 0x0000020e, 0x0000020f, 0x00000210, + 0x000200f8, 0x0000020f, 0x000200f9, 0x00000214, 0x000200f8, 0x00000210, 0x000200f9, 0x00000211, 0x000200f8, + 0x00000211, 0x00050080, 0x00000006, 0x00000213, 0x000002f8, 0x00000060, 0x000200f9, 0x00000203, 0x000200f8, + 0x00000214, 0x000700f5, 0x00000014, 0x000002fc, 0x000002fe, 0x00000203, 0x00000090, 0x0000020f, 0x000700f5, + 0x00000014, 0x000002f9, 0x00000090, 0x00000203, 0x00000094, 0x0000020f, 0x000300f7, 0x00000216, 0x00000000, + 0x000400fa, 0x000002f9, 0x00000217, 0x00000216, 0x000200f8, 0x00000216, 0x000200f9, 0x00000217, 0x000200f8, + 0x00000217, 0x000700f5, 0x00000014, 0x000002fb, 0x000002fc, 0x00000214, 0x00000094, 0x00000216, 0x000300f7, + 0x0000018c, 0x00000000, 0x000400fa, 0x000002fb, 0x0000017c, 0x0000017d, 0x000200f8, 0x0000017c, 0x000200f9, + 0x00000190, 0x000200f8, 0x0000017d, 0x000300f7, 0x00000235, 0x00000000, 0x000300fb, 0x00000042, 0x00000220, + 0x000200f8, 0x00000220, 0x000200f9, 0x00000221, 0x000200f8, 0x00000221, 0x000700f5, 0x00000006, 0x00000304, + 0x00000042, 0x00000220, 0x00000231, 0x0000022f, 0x000500b0, 0x00000014, 0x00000224, 0x00000304, 0x0000000c, + 0x000400f6, 0x00000232, 0x0000022f, 0x00000000, 0x000400fa, 0x00000224, 0x00000225, 0x00000232, 0x000200f8, + 0x00000225, 0x0003003e, 0x00000285, 0x000002dd, 0x00050041, 0x00000007, 0x00000287, 0x00000285, 0x00000304, + 0x0004003d, 0x00000006, 0x00000228, 0x00000287, 0x0003003e, 0x00000288, 0x000002e2, 0x00050041, 0x00000007, + 0x0000028a, 0x00000288, 0x00000304, 0x0004003d, 0x00000006, 0x0000022b, 0x0000028a, 0x000500ab, 0x00000014, + 0x0000022c, 0x00000228, 0x0000022b, 0x000300f7, 0x0000022e, 0x00000000, 0x000400fa, 0x0000022c, 0x0000022d, + 0x0000022e, 0x000200f8, 0x0000022d, 0x000200f9, 0x00000232, 0x000200f8, 0x0000022e, 0x000200f9, 0x0000022f, + 0x000200f8, 0x0000022f, 0x00050080, 0x00000006, 0x00000231, 0x00000304, 0x00000060, 0x000200f9, 0x00000221, + 0x000200f8, 0x00000232, 0x000700f5, 0x00000014, 0x00000308, 0x0000030d, 0x00000221, 0x00000090, 0x0000022d, + 0x000700f5, 0x00000014, 0x00000305, 0x00000090, 0x00000221, 0x00000094, 0x0000022d, 0x000300f7, 0x00000234, + 0x00000000, 0x000400fa, 0x00000305, 0x00000235, 0x00000234, 0x000200f8, 0x00000234, 0x000200f9, 0x00000235, + 0x000200f8, 0x00000235, 0x000700f5, 0x00000014, 0x00000307, 0x00000308, 0x00000232, 0x00000094, 0x00000234, + 0x000300f7, 0x00000186, 0x00000000, 0x000400fa, 0x00000307, 0x00000181, 0x00000186, 0x000200f8, 0x00000181, + 0x00050050, 0x0000000d, 0x000002ea, 0x00000175, 0x00000177, 0x000300f7, 0x00000256, 0x00000000, 0x000300fb, + 0x00000042, 0x00000241, 0x000200f8, 0x00000241, 0x000200f9, 0x00000242, 0x000200f8, 0x00000242, 0x000700f5, + 0x00000006, 0x00000313, 0x00000042, 0x00000241, 0x00000252, 0x00000250, 0x000500b0, 0x00000014, 0x00000245, + 0x00000313, 0x0000000c, 0x000400f6, 0x00000253, 0x00000250, 0x00000000, 0x000400fa, 0x00000245, 0x00000246, + 0x00000253, 0x000200f8, 0x00000246, 0x0003003e, 0x0000027f, 0x000002ea, 0x00050041, 0x00000007, 0x00000281, + 0x0000027f, 0x00000313, 0x0004003d, 0x00000006, 0x00000249, 0x00000281, 0x0003003e, 0x00000282, 0x00000097, + 0x00050041, 0x00000007, 0x00000284, 0x00000282, 0x00000313, 0x0004003d, 0x00000006, 0x0000024c, 0x00000284, + 0x000500ab, 0x00000014, 0x0000024d, 0x00000249, 0x0000024c, 0x000300f7, 0x0000024f, 0x00000000, 0x000400fa, + 0x0000024d, 0x0000024e, 0x0000024f, 0x000200f8, 0x0000024e, 0x000200f9, 0x00000253, 0x000200f8, 0x0000024f, + 0x000200f9, 0x00000250, 0x000200f8, 0x00000250, 0x00050080, 0x00000006, 0x00000252, 0x00000313, 0x00000060, + 0x000200f9, 0x00000242, 0x000200f8, 0x00000253, 0x000700f5, 0x00000014, 0x00000317, 0x0000031f, 0x00000242, + 0x00000090, 0x0000024e, 0x000700f5, 0x00000014, 0x00000314, 0x00000090, 0x00000242, 0x00000094, 0x0000024e, + 0x000300f7, 0x00000255, 0x00000000, 0x000400fa, 0x00000314, 0x00000256, 0x00000255, 0x000200f8, 0x00000255, + 0x000200f9, 0x00000256, 0x000200f8, 0x00000256, 0x000700f5, 0x00000014, 0x00000316, 0x00000317, 0x00000253, + 0x00000094, 0x00000255, 0x000400a8, 0x00000014, 0x00000185, 0x00000316, 0x000200f9, 0x00000186, 0x000200f8, + 0x00000186, 0x000700f5, 0x00000014, 0x00000357, 0x0000031f, 0x00000235, 0x00000316, 0x00000256, 0x000700f5, + 0x00000014, 0x00000187, 0x00000307, 0x00000235, 0x00000185, 0x00000256, 0x000300f7, 0x0000018b, 0x00000000, + 0x000400fa, 0x00000187, 0x00000188, 0x0000018b, 0x000200f8, 0x00000188, 0x000200f9, 0x00000190, 0x000200f8, + 0x0000018b, 0x000200f9, 0x0000018c, 0x000200f8, 0x0000018c, 0x000200f9, 0x0000018d, 0x000200f8, 0x0000018d, + 0x00050080, 0x00000006, 0x0000018f, 0x00000164, 0x00000060, 0x000200f9, 0x0000015f, 0x000200f8, 0x00000190, + 0x000700f5, 0x00000006, 0x00000333, 0x00000042, 0x0000017c, 0x00000175, 0x00000188, 0x000700f5, 0x00000006, + 0x00000331, 0x00000042, 0x0000017c, 0x00000177, 0x00000188, 0x000300f7, 0x00000192, 0x00000000, 0x000400fa, + 0x00000094, 0x00000193, 0x00000192, 0x000200f8, 0x00000192, 0x000200f9, 0x00000193, 0x000200f8, 0x00000193, + 0x000900f5, 0x00000006, 0x00000332, 0x00000042, 0x0000015c, 0x00000333, 0x00000190, 0x00000333, 0x00000192, + 0x000900f5, 0x00000006, 0x00000330, 0x00000042, 0x0000015c, 0x00000331, 0x00000190, 0x00000331, 0x00000192, + 0x00050050, 0x0000000d, 0x000002d5, 0x00000332, 0x00000330, 0x000300f7, 0x00000277, 0x00000000, 0x000300fb, + 0x00000042, 0x00000262, 0x000200f8, 0x00000262, 0x000200f9, 0x00000263, 0x000200f8, 0x00000263, 0x000700f5, + 0x00000006, 0x00000334, 0x00000042, 0x00000262, 0x00000273, 0x00000271, 0x000500b0, 0x00000014, 0x00000266, + 0x00000334, 0x0000000c, 0x000400f6, 0x00000274, 0x00000271, 0x00000000, 0x000400fa, 0x00000266, 0x00000267, + 0x00000274, 0x000200f8, 0x00000267, 0x0003003e, 0x00000279, 0x000002d5, 0x00050041, 0x00000007, 0x0000027b, + 0x00000279, 0x00000334, 0x0004003d, 0x00000006, 0x0000026a, 0x0000027b, 0x0003003e, 0x0000027c, 0x00000097, + 0x00050041, 0x00000007, 0x0000027e, 0x0000027c, 0x00000334, 0x0004003d, 0x00000006, 0x0000026d, 0x0000027e, + 0x000500ab, 0x00000014, 0x0000026e, 0x0000026a, 0x0000026d, 0x000300f7, 0x00000270, 0x00000000, 0x000400fa, + 0x0000026e, 0x0000026f, 0x00000270, 0x000200f8, 0x0000026f, 0x000200f9, 0x00000274, 0x000200f8, 0x00000270, + 0x000200f9, 0x00000271, 0x000200f8, 0x00000271, 0x00050080, 0x00000006, 0x00000273, 0x00000334, 0x00000060, + 0x000200f9, 0x00000263, 0x000200f8, 0x00000274, 0x000700f5, 0x00000014, 0x00000338, 0x000002f4, 0x00000263, + 0x00000090, 0x0000026f, 0x000700f5, 0x00000014, 0x00000335, 0x00000090, 0x00000263, 0x00000094, 0x0000026f, + 0x000300f7, 0x00000276, 0x00000000, 0x000400fa, 0x00000335, 0x00000277, 0x00000276, 0x000200f8, 0x00000276, + 0x000200f9, 0x00000277, 0x000200f8, 0x00000277, 0x000700f5, 0x00000014, 0x00000337, 0x00000338, 0x00000274, + 0x00000094, 0x00000276, 0x000400a8, 0x00000014, 0x00000128, 0x00000337, 0x000300f7, 0x0000012a, 0x00000000, + 0x000400fa, 0x00000128, 0x00000129, 0x0000012a, 0x000200f8, 0x00000129, 0x00060041, 0x00000109, 0x0000012b, + 0x000000fc, 0x00000083, 0x000000a5, 0x0004003d, 0x000000f1, 0x0000012c, 0x0000012b, 0x00060041, 0x0000010d, + 0x0000012e, 0x0000012c, 0x00000083, 0x000000ee, 0x0006003d, 0x000000f3, 0x0000012f, 0x0000012e, 0x00000002, + 0x00000008, 0x00050041, 0x00000110, 0x00000131, 0x0000012f, 0x00000083, 0x00050041, 0x00000133, 0x00000134, + 0x00000131, 0x00000083, 0x00050041, 0x00000136, 0x00000137, 0x00000134, 0x00000083, 0x0005003e, 0x00000137, + 0x00000332, 0x00000002, 0x00000004, 0x00050041, 0x00000136, 0x00000139, 0x00000134, 0x00000060, 0x0005003e, + 0x00000139, 0x00000330, 0x00000002, 0x00000004, 0x000200f9, 0x0000012a, 0x000200f8, 0x0000012a, 0x000200f9, + 0x0000013c, 0x000200f8, 0x0000013c, 0x000100fd, 0x00010038 }; GFXRECON_END_NAMESPACE(decode) From e878682a4b728e2991ef01f541ec230f0f2dced3 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 14 Jan 2025 15:07:42 +0100 Subject: [PATCH 17/23] Guard address_replacer.ProcessCmdTraceRays(), only for '-m rebind' - cleanup obsolete warnings --- framework/decode/vulkan_address_replacer.cpp | 3 +-- .../decode/vulkan_replay_consumer_base.cpp | 27 ++++++++++--------- .../encode/custom_vulkan_encoder_commands.h | 10 ------- framework/encode/vulkan_capture_manager.cpp | 19 ------------- framework/encode/vulkan_capture_manager.h | 3 --- 5 files changed, 16 insertions(+), 46 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index a1b24d88bf..7beb2107f7 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -965,7 +965,7 @@ void VulkanAddressReplacer::ProcessGetQueryPoolResults(VkDevice device VkQueryResultFlags flags) { // intercept queries containing acceleration-structure compaction-sizes - // if (!_as_compact_queries.empty()) + if (!_as_compact_queries.empty()) { bool is_synced = flags & VK_QUERY_RESULT_WAIT_BIT; @@ -977,7 +977,6 @@ void VulkanAddressReplacer::ProcessGetQueryPoolResults(VkDevice device for (const auto& [as, query_index] : it->second) { - GFXRECON_LOG_INFO("query-index %d: %d", query_index, result_array[query_index]); _as_compact_sizes[as] = result_array[query_index]; } } diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index e23a7f4103..14721c3ecd 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -8641,20 +8641,23 @@ void VulkanReplayConsumerBase::OverrideCmdTraceRaysKHR( VkStridedDeviceAddressRegionKHR* in_pHitShaderBindingTable = pHitShaderBindingTable->GetPointer(); VkStridedDeviceAddressRegionKHR* in_pCallableShaderBindingTable = pCallableShaderBindingTable->GetPointer(); - // identify buffer(s) by their device-address - const auto& address_tracker = GetDeviceAddressTracker(device_info); - auto& address_replacer = GetDeviceAddressReplacer(device_info); + if (!device_info->allocator->SupportsOpaqueDeviceAddresses()) + { + // identify buffer(s) by their device-address + const auto& address_tracker = GetDeviceAddressTracker(device_info); + auto& address_replacer = GetDeviceAddressReplacer(device_info); - auto bound_pipeline = GetObjectInfoTable().GetVkPipelineInfo(command_buffer_info->bound_pipeline_id); - GFXRECON_ASSERT(bound_pipeline != nullptr) + auto bound_pipeline = GetObjectInfoTable().GetVkPipelineInfo(command_buffer_info->bound_pipeline_id); + GFXRECON_ASSERT(bound_pipeline != nullptr) - address_replacer.ProcessCmdTraceRays(command_buffer_info, - in_pRaygenShaderBindingTable, - in_pMissShaderBindingTable, - in_pHitShaderBindingTable, - in_pCallableShaderBindingTable, - address_tracker, - bound_pipeline->shader_group_handle_map); + address_replacer.ProcessCmdTraceRays(command_buffer_info, + in_pRaygenShaderBindingTable, + in_pMissShaderBindingTable, + in_pHitShaderBindingTable, + in_pCallableShaderBindingTable, + address_tracker, + bound_pipeline->shader_group_handle_map); + } func(commandBuffer, in_pRaygenShaderBindingTable, diff --git a/framework/encode/custom_vulkan_encoder_commands.h b/framework/encode/custom_vulkan_encoder_commands.h index e3660a94e5..42d73156d8 100644 --- a/framework/encode/custom_vulkan_encoder_commands.h +++ b/framework/encode/custom_vulkan_encoder_commands.h @@ -1020,16 +1020,6 @@ struct CustomEncoderPreCall -struct CustomEncoderPreCall -{ - template - static void Dispatch(VulkanCaptureManager* manager, Args... args) - { - manager->PreProcess_vkGetRayTracingShaderGroupHandlesKHR(args...); - } -}; - template <> struct CustomEncoderPostCall { diff --git a/framework/encode/vulkan_capture_manager.cpp b/framework/encode/vulkan_capture_manager.cpp index b211fe249b..c572b81f18 100644 --- a/framework/encode/vulkan_capture_manager.cpp +++ b/framework/encode/vulkan_capture_manager.cpp @@ -1317,11 +1317,6 @@ VulkanCaptureManager::OverrideCreateRayTracingPipelinesKHR(VkDevice } else { - GFXRECON_LOG_ERROR_ONCE( - "The capturing application used vkCreateRayTracingPipelinesKHR, which may require the " - "rayTracingPipelineShaderGroupHandleCaptureReplay feature for accurate capture and replay. The capturing " - "device does not support this feature, so replay may fail."); - if (deferred_operation_wrapper) { deferred_operation_wrapper->create_infos.clear(); @@ -1408,7 +1403,6 @@ VulkanCaptureManager::OverrideCreateRayTracingPipelinesKHR(VkDevice } } } - return result; } @@ -2524,19 +2518,6 @@ void VulkanCaptureManager::PreProcess_vkGetAccelerationStructureDeviceAddressKHR } } -void VulkanCaptureManager::PreProcess_vkGetRayTracingShaderGroupHandlesKHR( - VkDevice device, VkPipeline pipeline, uint32_t firstGroup, uint32_t groupCount, size_t dataSize, void* pData) -{ - auto device_wrapper = vulkan_wrappers::GetWrapper(device); - if (!device_wrapper->property_feature_info.feature_rayTracingPipelineShaderGroupHandleCaptureReplay) - { - GFXRECON_LOG_WARNING_ONCE( - "The application is using vkGetRayTracingShaderGroupHandlesKHR, which may require the " - "rayTracingPipelineShaderGroupHandleCaptureReplay feature for accurate capture and replay. The capture " - "device does not support this feature, so replay of the captured file may fail."); - } -} - void VulkanCaptureManager::PreProcess_vkGetAndroidHardwareBufferPropertiesANDROID( VkDevice device, const struct AHardwareBuffer* hardware_buffer, diff --git a/framework/encode/vulkan_capture_manager.h b/framework/encode/vulkan_capture_manager.h index 50e69c4af9..b928248475 100644 --- a/framework/encode/vulkan_capture_manager.h +++ b/framework/encode/vulkan_capture_manager.h @@ -1270,9 +1270,6 @@ class VulkanCaptureManager : public ApiCaptureManager PreProcess_vkGetAccelerationStructureDeviceAddressKHR(VkDevice device, const VkAccelerationStructureDeviceAddressInfoKHR* pInfo); - void PreProcess_vkGetRayTracingShaderGroupHandlesKHR( - VkDevice device, VkPipeline pipeline, uint32_t firstGroup, uint32_t groupCount, size_t dataSize, void* pData); - void PreProcess_vkGetAndroidHardwareBufferPropertiesANDROID(VkDevice device, const struct AHardwareBuffer* buffer, VkAndroidHardwareBufferPropertiesANDROID* pProperties); From 5fc94a5b6c44050afcdd8f605a352f96f0e09f5e Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Tue, 14 Jan 2025 18:57:35 +0100 Subject: [PATCH 18/23] review comments, cleanup --- framework/decode/vulkan_address_replacer.cpp | 355 +++++++++--------- framework/decode/vulkan_address_replacer.h | 58 +-- .../decode/vulkan_device_address_tracker.cpp | 52 +-- .../decode/vulkan_device_address_tracker.h | 10 +- 4 files changed, 241 insertions(+), 234 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 7beb2107f7..9d27be98ae 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -109,7 +109,7 @@ inline uint32_t aligned_size(uint32_t size, uint32_t alignment) inline uint32_t div_up(uint32_t nom, uint32_t denom) { - GFXRECON_ASSERT(denom > 0) + GFXRECON_ASSERT(denom > 0); return (nom + denom - 1) / denom; } @@ -161,39 +161,44 @@ decode::VulkanAddressReplacer::acceleration_structure_asset_t::~acceleration_str VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* device_info, const encode::VulkanDeviceTable* device_table, const decode::CommonObjectInfoTable& object_table) : - _device_table(device_table) + device_table_(device_table) { - GFXRECON_ASSERT(device_info != nullptr && device_table != nullptr) + GFXRECON_ASSERT(device_info != nullptr && device_table != nullptr); const VulkanPhysicalDeviceInfo* physical_device_info = object_table.GetVkPhysicalDeviceInfo(device_info->parent_id); - _device = device_info->handle; - _resource_allocator = device_info->allocator.get(); - _get_device_address_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_2 - ? device_table->GetBufferDeviceAddress - : device_table->GetBufferDeviceAddressKHR; + device_ = device_info->handle; + resource_allocator_ = device_info->allocator.get(); + get_device_address_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_2 + ? device_table->GetBufferDeviceAddress + : device_table->GetBufferDeviceAddressKHR; if (physical_device_info != nullptr && physical_device_info->capture_raytracing_properties && - physical_device_info->replay_device_info->raytracing_properties) + physical_device_info->replay_device_info->raytracing_properties && + physical_device_info->replay_device_info->acceleration_structure_properties) { - _capture_ray_properties = *physical_device_info->capture_raytracing_properties; - _replay_ray_properties = *physical_device_info->replay_device_info->raytracing_properties; - - if (physical_device_info->replay_device_info->acceleration_structure_properties) - { - _replay_acceleration_structure_properties = - *physical_device_info->replay_device_info->acceleration_structure_properties; - } + SetRaytracingProperties(*physical_device_info->capture_raytracing_properties, + *physical_device_info->replay_device_info->raytracing_properties, + *physical_device_info->replay_device_info->acceleration_structure_properties); + } + GFXRECON_ASSERT(physical_device_info->replay_device_info != nullptr); + GFXRECON_ASSERT(physical_device_info->replay_device_info->memory_properties.has_value()); + memory_properties_ = *physical_device_info->replay_device_info->memory_properties; +} - if (_capture_ray_properties.shaderGroupHandleSize != _replay_ray_properties.shaderGroupHandleSize || - _capture_ray_properties.shaderGroupHandleAlignment != _replay_ray_properties.shaderGroupHandleAlignment || - _capture_ray_properties.shaderGroupBaseAlignment != _replay_ray_properties.shaderGroupBaseAlignment) - { - _valid_sbt_alignment = false; - } +void VulkanAddressReplacer::SetRaytracingProperties( + const VkPhysicalDeviceRayTracingPipelinePropertiesKHR& capture_properties, + const VkPhysicalDeviceRayTracingPipelinePropertiesKHR& replay_properties, + const VkPhysicalDeviceAccelerationStructurePropertiesKHR& replay_as_properties) +{ + capture_ray_properties_ = capture_properties; + replay_ray_properties_ = replay_properties; + replay_acceleration_structure_properties_ = replay_as_properties; - GFXRECON_ASSERT(physical_device_info->replay_device_info != nullptr); - GFXRECON_ASSERT(physical_device_info->replay_device_info->memory_properties.has_value()); - _memory_properties = *physical_device_info->replay_device_info->memory_properties; + if (capture_ray_properties_.shaderGroupHandleSize != replay_ray_properties_.shaderGroupHandleSize || + capture_ray_properties_.shaderGroupHandleAlignment != replay_ray_properties_.shaderGroupHandleAlignment || + capture_ray_properties_.shaderGroupBaseAlignment != replay_ray_properties_.shaderGroupBaseAlignment) + { + valid_sbt_alignment_ = false; } } @@ -202,41 +207,41 @@ VulkanAddressReplacer::~VulkanAddressReplacer() mark_injected_commands_helper_t mark_injected_commands_helper; // explicitly free resources here, in order to mark destruction API-calls as injected - _pipeline_sbt_context_map = {}; - _build_as_context_map = {}; + pipeline_sbt_context_map_ = {}; + build_as_context_map_ = {}; - _shadow_sbt_map = {}; - _shadow_as_map = {}; + shadow_sbt_map_ = {}; + shadow_as_map_ = {}; - if (_pipeline_bda != VK_NULL_HANDLE) + if (pipeline_bda_ != VK_NULL_HANDLE) { - _device_table->DestroyPipeline(_device, _pipeline_bda, nullptr); + device_table_->DestroyPipeline(device_, pipeline_bda_, nullptr); } - if (_pipeline_sbt != VK_NULL_HANDLE) + if (pipeline_sbt_ != VK_NULL_HANDLE) { - _device_table->DestroyPipeline(_device, _pipeline_sbt, nullptr); + device_table_->DestroyPipeline(device_, pipeline_sbt_, nullptr); } - if (_pipeline_layout != VK_NULL_HANDLE) + if (pipeline_layout_ != VK_NULL_HANDLE) { - _device_table->DestroyPipelineLayout(_device, _pipeline_layout, nullptr); + device_table_->DestroyPipelineLayout(device_, pipeline_layout_, nullptr); } - if (_query_pool != VK_NULL_HANDLE) + if (query_pool_ != VK_NULL_HANDLE) { - _device_table->DestroyQueryPool(_device, _query_pool, nullptr); + device_table_->DestroyQueryPool(device_, query_pool_, nullptr); } - if (_fence != VK_NULL_HANDLE) + if (fence_ != VK_NULL_HANDLE) { - _device_table->DestroyFence(_device, _fence, nullptr); + device_table_->DestroyFence(device_, fence_, nullptr); } - if (_command_buffer != VK_NULL_HANDLE) + if (command_buffer_ != VK_NULL_HANDLE) { - GFXRECON_ASSERT(_command_pool != VK_NULL_HANDLE) - _device_table->FreeCommandBuffers(_device, _command_pool, 1, &_command_buffer); + GFXRECON_ASSERT(command_pool_ != VK_NULL_HANDLE); + device_table_->FreeCommandBuffers(device_, command_pool_, 1, &command_buffer_); } - if (_command_pool != VK_NULL_HANDLE) + if (command_pool_ != VK_NULL_HANDLE) { - _device_table->DestroyCommandPool(_device, _command_pool, nullptr); + device_table_->DestroyCommandPool(device_, command_pool_, nullptr); } } @@ -249,7 +254,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( const decode::VulkanDeviceAddressTracker& address_tracker, const std::unordered_map& group_handle_map) { - GFXRECON_ASSERT(_device_table != nullptr); + GFXRECON_ASSERT(device_table_ != nullptr); // NOTE: we expect this map to be populated here, but not for older captures (before #1844) using trimming. if (group_handle_map.empty()) @@ -306,12 +311,12 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( address_remap(hit_sbt); address_remap(callable_sbt); - if (!_valid_sbt_alignment || !valid_group_handles) + if (!valid_sbt_alignment_ || !valid_group_handles) { // mark injected commands mark_injected_commands_helper_t mark_injected_commands_helper; - if (_pipeline_sbt == VK_NULL_HANDLE) + if (pipeline_sbt_ == VK_NULL_HANDLE) { if (!init_pipeline()) { @@ -321,21 +326,21 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } // prepare linear hashmap - _hashmap_sbt.clear(); + hashmap_sbt_.clear(); for (const auto& [lhs, rhs] : group_handle_map) { - _hashmap_sbt.put(lhs, rhs); + hashmap_sbt_.put(lhs, rhs); } // get a context for this command-buffer - auto& pipeline_context_sbt = _pipeline_sbt_context_map[command_buffer_info->handle]; + auto& pipeline_context_sbt = pipeline_sbt_context_map_[command_buffer_info->handle]; - if (!create_buffer(pipeline_context_sbt.hashmap_storage, _hashmap_sbt.get_storage(nullptr))) + if (!create_buffer(pipeline_context_sbt.hashmap_storage, hashmap_sbt_.get_storage(nullptr))) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); } - _hashmap_sbt.get_storage(pipeline_context_sbt.hashmap_storage.mapped_data); + hashmap_sbt_.get_storage(pipeline_context_sbt.hashmap_storage.mapped_data); // input-handles constexpr uint32_t max_num_handles = 4; @@ -356,13 +361,13 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( replacer_params_t replacer_params = {}; replacer_params.hashmap.storage = pipeline_context_sbt.hashmap_storage.device_address; - replacer_params.hashmap.size = _hashmap_sbt.size(); - replacer_params.hashmap.capacity = _hashmap_sbt.capacity(); + replacer_params.hashmap.size = hashmap_sbt_.size(); + replacer_params.hashmap.capacity = hashmap_sbt_.capacity(); replacer_params.input_handles = pipeline_context_sbt.input_handle_buffer.device_address; replacer_params.num_handles = num_addresses; - if (_valid_sbt_alignment) + if (valid_sbt_alignment_) { GFXRECON_LOG_INFO_ONCE("VulkanAddressReplacer::ProcessCmdTraceRays: Replay adjusted mismatching raytracing " "shader-group-handles"); @@ -397,10 +402,10 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // find/create shadow-SBT-buffer uint32_t sbt_offset = 0; - auto& shadow_buf_context = _shadow_sbt_map[command_buffer_info->handle]; + auto& shadow_buf_context = shadow_sbt_map_[command_buffer_info->handle]; - const uint32_t handle_size_aligned = aligned_size(_replay_ray_properties.shaderGroupHandleSize, - _replay_ray_properties.shaderGroupHandleAlignment); + const uint32_t handle_size_aligned = aligned_size(replay_ray_properties_.shaderGroupHandleSize, + replay_ray_properties_.shaderGroupHandleAlignment); for (auto& region : { raygen_sbt, miss_sbt, hit_sbt, callable_sbt }) { @@ -408,7 +413,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( { uint32_t num_handles_limit = region->size / region->stride; uint32_t group_size = aligned_size(num_handles_limit * handle_size_aligned, - _replay_ray_properties.shaderGroupBaseAlignment); + replay_ray_properties_.shaderGroupBaseAlignment); sbt_offset += group_size; // adjust group-size @@ -417,7 +422,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } } // raygen: stride == size - raygen_sbt->size = raygen_sbt->stride = _replay_ray_properties.shaderGroupBaseAlignment; + raygen_sbt->size = raygen_sbt->stride = replay_ray_properties_.shaderGroupBaseAlignment; if (!create_buffer(shadow_buf_context, sbt_offset, VK_BUFFER_USAGE_SHADER_BINDING_TABLE_BIT_KHR)) { @@ -441,18 +446,18 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } } - _device_table->CmdBindPipeline(command_buffer_info->handle, VK_PIPELINE_BIND_POINT_COMPUTE, _pipeline_sbt); + device_table_->CmdBindPipeline(command_buffer_info->handle, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline_sbt_); // NOTE: using push-constants here requires us to re-establish the previous data, if any - _device_table->CmdPushConstants(command_buffer_info->handle, - _pipeline_layout, + device_table_->CmdPushConstants(command_buffer_info->handle, + pipeline_layout_, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(replacer_params_t), &replacer_params); // run a single workgroup constexpr uint32_t wg_size = 32; - _device_table->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); + device_table_->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); // post memory-barrier for (const auto& buf : buffer_set) @@ -468,7 +473,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // set previous push-constant data if (!command_buffer_info->push_constant_data.empty()) { - _device_table->CmdPushConstants(command_buffer_info->handle, + device_table_->CmdPushConstants(command_buffer_info->handle, command_buffer_info->push_constant_pipeline_layout, command_buffer_info->push_constant_stage_flags, 0, @@ -485,7 +490,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( VkAccelerationStructureBuildRangeInfoKHR** build_range_infos, const VulkanDeviceAddressTracker& address_tracker) { - GFXRECON_ASSERT(_device_table != nullptr); + GFXRECON_ASSERT(device_table_ != nullptr); bool force_replace = false; @@ -545,7 +550,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( } mark_injected_commands_helper_t mark_injected_commands_helper; - _device_table->GetAccelerationStructureBuildSizesKHR(_device, + device_table_->GetAccelerationStructureBuildSizesKHR(device_, VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, &build_geometry_info, primitive_counts.data(), @@ -555,11 +560,9 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // retrieve VkAccelerationStructureKHR -> VkBuffer -> check/correct size auto* acceleration_structure_info = address_tracker.GetAccelerationStructureByHandle(build_geometry_info.dstAccelerationStructure); - GFXRECON_ASSERT(acceleration_structure_info != nullptr) + GFXRECON_ASSERT(acceleration_structure_info != nullptr); auto* buffer_info = address_tracker.GetBufferByHandle(acceleration_structure_info->buffer); - GFXRECON_ASSERT(buffer_info != nullptr) - - bool as_buffer_usable = + bool as_buffer_usable = buffer_info != nullptr && buffer_info->size >= build_size_info.accelerationStructureSize; // determine required size of scratch-buffer @@ -577,7 +580,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // now definitely requiring address-replacement force_replace = true; - auto& replacement_as = _shadow_as_map[build_geometry_info.dstAccelerationStructure]; + auto& replacement_as = shadow_as_map_[build_geometry_info.dstAccelerationStructure]; if (replacement_as.handle == VK_NULL_HANDLE) { @@ -589,13 +592,17 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( GFXRECON_ASSERT(accel_info != nullptr && accel_info->replay_address != 0); replacement_as.address = accel_info->replay_address; } - else if (!create_acceleration_asset(replacement_as, - build_geometry_info.type, - build_size_info.accelerationStructureSize, - scratch_size)) + else { - GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: creation of shadow " - "acceleration-structure failed"); + auto ret = create_acceleration_asset(replacement_as, + build_geometry_info.type, + build_size_info.accelerationStructureSize, + scratch_size); + if (!ret) + { + GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: creation of shadow " + "acceleration-structure failed"); + } } } @@ -610,7 +617,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( replacement_as.scratch, scratch_size, 0, - _replay_acceleration_structure_properties.minAccelerationStructureScratchOffsetAlignment, + replay_acceleration_structure_properties_.minAccelerationStructureScratchOffsetAlignment, false)) { GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: scratch-buffer creation failed"); @@ -670,12 +677,12 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (!addresses_to_replace.empty()) { // prepare linear hashmap - _hashmap_bda.clear(); + hashmap_bda_.clear(); auto acceleration_structure_map = address_tracker.GetAccelerationStructureDeviceAddressMap(); for (const auto& [capture_address, replay_address] : acceleration_structure_map) { auto* accel_info = address_tracker.GetAccelerationStructureByCaptureDeviceAddress(capture_address); - GFXRECON_ASSERT(accel_info != nullptr) + GFXRECON_ASSERT(accel_info != nullptr); if (force_replace || capture_address != replay_address) { @@ -684,8 +691,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // extra look-up required for potentially replaced AS if (accel_info != nullptr) { - auto shadow_as_it = _shadow_as_map.find(accel_info->handle); - if (shadow_as_it != _shadow_as_map.end()) + auto shadow_as_it = shadow_as_map_.find(accel_info->handle); + if (shadow_as_it != shadow_as_map_.end()) { new_address = shadow_as_it->second.address; } @@ -693,29 +700,29 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // store addresses we will need to replace GFXRECON_ASSERT(new_address != 0); - _hashmap_bda.put(capture_address, new_address); + hashmap_bda_.put(capture_address, new_address); } } - if (!_hashmap_bda.empty()) + if (!hashmap_bda_.empty()) { // mark injected commands mark_injected_commands_helper_t mark_injected_commands_helper; - if (_pipeline_bda == VK_NULL_HANDLE && !init_pipeline()) + if (pipeline_bda_ == VK_NULL_HANDLE && !init_pipeline()) { GFXRECON_LOG_WARNING_ONCE("ProcessCmdBuildAccelerationStructuresKHR: internal pipeline-creation failed") return; } - auto& pipeline_context_bda = _build_as_context_map[command_buffer_info->handle]; + auto& pipeline_context_bda = build_as_context_map_[command_buffer_info->handle]; - if (!create_buffer(pipeline_context_bda.hashmap_storage, _hashmap_bda.get_storage(nullptr))) + if (!create_buffer(pipeline_context_bda.hashmap_storage, hashmap_bda_.get_storage(nullptr))) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: hashmap-storage-buffer creation failed"); return; } - _hashmap_bda.get_storage(pipeline_context_bda.hashmap_storage.mapped_data); + hashmap_bda_.get_storage(pipeline_context_bda.hashmap_storage.mapped_data); uint32_t num_bytes = addresses_to_replace.size() * sizeof(VkDeviceAddress); @@ -728,8 +735,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( replacer_params_t replacer_params = {}; replacer_params.hashmap.storage = pipeline_context_bda.hashmap_storage.device_address; - replacer_params.hashmap.size = _hashmap_bda.size(); - replacer_params.hashmap.capacity = _hashmap_bda.capacity(); + replacer_params.hashmap.size = hashmap_bda_.size(); + replacer_params.hashmap.capacity = hashmap_bda_.capacity(); // in-place replacer_params.input_handles = pipeline_context_bda.input_handle_buffer.device_address; @@ -737,18 +744,18 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( replacer_params.num_handles = addresses_to_replace.size(); - _device_table->CmdBindPipeline(command_buffer_info->handle, VK_PIPELINE_BIND_POINT_COMPUTE, _pipeline_bda); + device_table_->CmdBindPipeline(command_buffer_info->handle, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline_bda_); // NOTE: using push-constants here requires us to re-establish the previous data, if any - _device_table->CmdPushConstants(command_buffer_info->handle, - _pipeline_layout, + device_table_->CmdPushConstants(command_buffer_info->handle, + pipeline_layout_, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(replacer_params_t), &replacer_params); // dispatch workgroups constexpr uint32_t wg_size = 32; - _device_table->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); + device_table_->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); // post memory-barrier for (const auto& buf : buffer_set) @@ -764,7 +771,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // set previous push-constant data if (!command_buffer_info->push_constant_data.empty()) { - _device_table->CmdPushConstants(command_buffer_info->handle, + device_table_->CmdPushConstants(command_buffer_info->handle, command_buffer_info->push_constant_pipeline_layout, command_buffer_info->push_constant_stage_flags, 0, @@ -781,8 +788,8 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR( if (info != nullptr) { auto swap_acceleration_structure = [this](VkAccelerationStructureKHR& as) { - auto shadow_as_it = _shadow_as_map.find(as); - if (shadow_as_it != _shadow_as_map.end()) + auto shadow_as_it = shadow_as_map_.find(as); + if (shadow_as_it != shadow_as_map_.end()) { as = shadow_as_it->second.handle; } @@ -793,14 +800,16 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR( swap_acceleration_structure(info->dst); VkDeviceSize compact_size = 0; - auto compact_size_it = _as_compact_sizes.find(info->dst); - if (compact_size_it != _as_compact_sizes.end()) + auto compact_size_it = as_compact_sizes_.find(info->dst); + if (compact_size_it != as_compact_sizes_.end()) { compact_size = compact_size_it->second; - _as_compact_sizes.erase(info->dst); - GFXRECON_LOG_INFO( - "VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR: found compacted AS-size: %ul", - compact_size); + as_compact_sizes_.erase(info->dst); + + // tmp/debug: cleanup before merge +// GFXRECON_LOG_INFO( +// "VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR: found compacted AS-size: %ul", +// compact_size); } } } @@ -814,8 +823,8 @@ void VulkanAddressReplacer::ProcessCmdWriteAccelerationStructuresPropertiesKHR( { for (uint32_t i = 0; i < count; ++i) { - auto shadow_as_it = _shadow_as_map.find(acceleration_structures[i]); - if (shadow_as_it != _shadow_as_map.end()) + auto shadow_as_it = shadow_as_map_.find(acceleration_structures[i]); + if (shadow_as_it != shadow_as_map_.end()) { acceleration_structures[i] = shadow_as_it->second.handle; } @@ -823,7 +832,7 @@ void VulkanAddressReplacer::ProcessCmdWriteAccelerationStructuresPropertiesKHR( if (query_type == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR) { // read back compacted size later - _as_compact_queries[pool][acceleration_structures[i]] = first_query + i; + as_compact_queries_[pool][acceleration_structures[i]] = first_query + i; } } } @@ -849,8 +858,8 @@ void VulkanAddressReplacer::ProcessUpdateDescriptorSets(uint32_t de { for (uint32_t j = 0; j < write_as->accelerationStructureCount; ++j) { - auto acceleration_structure_it = _shadow_as_map.find(write_as->pAccelerationStructures[j]); - if (acceleration_structure_it != _shadow_as_map.end()) + auto acceleration_structure_it = shadow_as_map_.find(write_as->pAccelerationStructures[j]); + if (acceleration_structure_it != shadow_as_map_.end()) { // we found an existing replacement-structure -> swap auto* out_array = const_cast(write_as->pAccelerationStructures); @@ -873,17 +882,17 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( if (info_count > 0 && init_queue_assets()) { // reset/submit/sync command-buffer - queue_submit_helper_t queue_submit_helper(_device_table, _device, _command_buffer, _queue, _fence); + queue_submit_helper_t queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); // dummy-wrapper VulkanCommandBufferInfo command_buffer_info = {}; - command_buffer_info.handle = _command_buffer; + command_buffer_info.handle = command_buffer_; ProcessCmdBuildAccelerationStructuresKHR( &command_buffer_info, info_count, geometry_infos, range_infos, address_tracker); // issue build-command mark_injected_commands_helper_t mark_injected_commands_helper; - _device_table->CmdBuildAccelerationStructuresKHR(_command_buffer, info_count, geometry_infos, range_infos); + device_table_->CmdBuildAccelerationStructuresKHR(command_buffer_, info_count, geometry_infos, range_infos); } } @@ -892,10 +901,10 @@ void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( VkCopyAccelerationStructureInfoKHR* copy_infos, const decode::VulkanDeviceAddressTracker& address_tracker) { - if (info_count > 0 && init_queue_assets()) + if (copy_infos != nullptr && info_count > 0 && init_queue_assets()) { // reset/submit/sync command-buffer - queue_submit_helper_t queue_submit_helper(_device_table, _device, _command_buffer, _queue, _fence); + queue_submit_helper_t queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); for (uint32_t i = 0; i < info_count; ++i) { @@ -907,7 +916,7 @@ void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( // issue copy command mark_injected_commands_helper_t mark_injected_commands_helper; - _device_table->CmdCopyAccelerationStructureKHR(_command_buffer, copy_info); + device_table_->CmdCopyAccelerationStructureKHR(command_buffer_, copy_info); } else { @@ -923,30 +932,30 @@ void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMe if (init_queue_assets()) { // reset/submit/sync command-buffer - queue_submit_helper_t queue_submit_helper(_device_table, _device, _command_buffer, _queue, _fence); + queue_submit_helper_t queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); - ProcessCmdWriteAccelerationStructuresPropertiesKHR(1, &acceleration_structure, query_type, _query_pool, 0); + ProcessCmdWriteAccelerationStructuresPropertiesKHR(1, &acceleration_structure, query_type, query_pool_, 0); // issue vkCmdWriteAccelerationStructuresPropertiesKHR mark_injected_commands_helper_t mark_injected_commands_helper; - _device_table->CmdWriteAccelerationStructuresPropertiesKHR( - _command_buffer, 1, &acceleration_structure, query_type, _query_pool, 0); + device_table_->CmdWriteAccelerationStructuresPropertiesKHR( + command_buffer_, 1, &acceleration_structure, query_type, query_pool_, 0); } VkDeviceSize compact_size = 0; // issue vkCmdWriteAccelerationStructuresPropertiesKHR mark_injected_commands_helper_t mark_injected_commands_helper; - _device_table->GetQueryPoolResults(_device, - _query_pool, + device_table_->GetQueryPoolResults(device_, + query_pool_, 0, 1, sizeof(VkDeviceSize), &compact_size, sizeof(VkDeviceSize), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); - ProcessGetQueryPoolResults(_device, - _query_pool, + ProcessGetQueryPoolResults(device_, + query_pool_, 0, 1, sizeof(VkDeviceSize), @@ -965,28 +974,28 @@ void VulkanAddressReplacer::ProcessGetQueryPoolResults(VkDevice device VkQueryResultFlags flags) { // intercept queries containing acceleration-structure compaction-sizes - if (!_as_compact_queries.empty()) + if (!as_compact_queries_.empty()) { bool is_synced = flags & VK_QUERY_RESULT_WAIT_BIT; - auto it = _as_compact_queries.find(query_pool); - if (is_synced && it != _as_compact_queries.end()) + auto it = as_compact_queries_.find(query_pool); + if (is_synced && it != as_compact_queries_.end()) { // assuming post-processing here, pData was already written auto* result_array = reinterpret_cast(pData); for (const auto& [as, query_index] : it->second) { - _as_compact_sizes[as] = result_array[query_index]; + as_compact_sizes_[as] = result_array[query_index]; } } - _as_compact_queries.erase(query_pool); + as_compact_queries_.erase(query_pool); } } bool VulkanAddressReplacer::init_pipeline() { - if (_pipeline_sbt != VK_NULL_HANDLE) + if (pipeline_sbt_ != VK_NULL_HANDLE) { // assume already initialized return true; @@ -1005,7 +1014,7 @@ bool VulkanAddressReplacer::init_pipeline() pipeline_layout_info.pushConstantRangeCount = 1; pipeline_layout_info.pPushConstantRanges = &push_constant_range; - VkResult result = _device_table->CreatePipelineLayout(_device, &pipeline_layout_info, nullptr, &_pipeline_layout); + VkResult result = device_table_->CreatePipelineLayout(device_, &pipeline_layout_info, nullptr, &pipeline_layout_); if (result != VK_SUCCESS) { @@ -1023,7 +1032,7 @@ bool VulkanAddressReplacer::init_pipeline() shader_module_create_info.pCode = reinterpret_cast(spirv.data()); VkResult result = - _device_table->CreateShaderModule(_device, &shader_module_create_info, nullptr, &compute_module); + device_table_->CreateShaderModule(device_, &shader_module_create_info, nullptr, &compute_module); if (result != VK_SUCCESS) { @@ -1044,8 +1053,8 @@ bool VulkanAddressReplacer::init_pipeline() pipeline_create_info.layout = layout; pipeline_create_info.stage = stage_info; - result = _device_table->CreateComputePipelines( - _device, VK_NULL_HANDLE, 1, &pipeline_create_info, VK_NULL_HANDLE, &out_pipeline); + result = device_table_->CreateComputePipelines( + device_, VK_NULL_HANDLE, 1, &pipeline_create_info, VK_NULL_HANDLE, &out_pipeline); if (result != VK_SUCCESS) { @@ -1054,19 +1063,19 @@ bool VulkanAddressReplacer::init_pipeline() if (compute_module != VK_NULL_HANDLE) { - _device_table->DestroyShaderModule(_device, compute_module, nullptr); + device_table_->DestroyShaderModule(device_, compute_module, nullptr); } return result; }; // create SBT pipeline - if (create_pipeline(_pipeline_layout, g_replacer_sbt_comp, _pipeline_sbt) != VK_SUCCESS) + if (create_pipeline(pipeline_layout_, g_replacer_sbt_comp, pipeline_sbt_) != VK_SUCCESS) { return false; } // create BDA pipeline - if (create_pipeline(_pipeline_layout, g_replacer_bda_comp, _pipeline_bda) != VK_SUCCESS) + if (create_pipeline(pipeline_layout_, g_replacer_bda_comp, pipeline_bda_) != VK_SUCCESS) { return false; } @@ -1075,7 +1084,7 @@ bool VulkanAddressReplacer::init_pipeline() bool VulkanAddressReplacer::init_queue_assets() { - if (_queue != VK_NULL_HANDLE) + if (queue_ != VK_NULL_HANDLE) { return true; }; @@ -1086,7 +1095,7 @@ bool VulkanAddressReplacer::init_queue_assets() create_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; create_info.queueFamilyIndex = 0; - VkResult result = _device_table->CreateCommandPool(_device, &create_info, nullptr, &_command_pool); + VkResult result = device_table_->CreateCommandPool(device_, &create_info, nullptr, &command_pool_); if (result != VK_SUCCESS) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: internal command-pool creation failed"); @@ -1096,10 +1105,10 @@ bool VulkanAddressReplacer::init_queue_assets() VkCommandBufferAllocateInfo alloc_info = {}; alloc_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; alloc_info.pNext = nullptr; - alloc_info.commandPool = _command_pool; + alloc_info.commandPool = command_pool_; alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; alloc_info.commandBufferCount = 1; - result = _device_table->AllocateCommandBuffers(_device, &alloc_info, &_command_buffer); + result = device_table_->AllocateCommandBuffers(device_, &alloc_info, &command_buffer_); if (result != VK_SUCCESS) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: internal command-buffer creation failed"); @@ -1110,7 +1119,7 @@ bool VulkanAddressReplacer::init_queue_assets() fence_create_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; fence_create_info.pNext = nullptr; fence_create_info.flags = VK_FENCE_CREATE_SIGNALED_BIT; - result = _device_table->CreateFence(_device, &fence_create_info, nullptr, &_fence); + result = device_table_->CreateFence(device_, &fence_create_info, nullptr, &fence_); if (result != VK_SUCCESS) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: internal fence creation failed"); @@ -1124,16 +1133,16 @@ bool VulkanAddressReplacer::init_queue_assets() pool_info.queryType = VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR; pool_info.queryCount = 1; pool_info.pipelineStatistics = 0; - result = _device_table->CreateQueryPool(_device, &pool_info, nullptr, &_query_pool); + result = device_table_->CreateQueryPool(device_, &pool_info, nullptr, &query_pool_); if (result != VK_SUCCESS) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: internal query-pool creation failed"); return false; } - _device_table->GetDeviceQueue(_device, 0, 0, &_queue); - GFXRECON_ASSERT(_queue != VK_NULL_HANDLE); - return _queue != VK_NULL_HANDLE; + device_table_->GetDeviceQueue(device_, 0, 0, &queue_); + GFXRECON_ASSERT(queue_ != VK_NULL_HANDLE); + return queue_ != VK_NULL_HANDLE; } bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_t& buffer_context, @@ -1156,7 +1165,7 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ // free previous resources buffer_context = {}; - buffer_context.resource_allocator = _resource_allocator; + buffer_context.resource_allocator = resource_allocator_; buffer_context.num_bytes = num_bytes; VkBufferCreateInfo buffer_create_info = {}; @@ -1167,7 +1176,7 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ buffer_create_info.queueFamilyIndexCount = 0; buffer_create_info.size = num_bytes; - VkResult result = _resource_allocator->CreateBufferDirect( + VkResult result = resource_allocator_->CreateBufferDirect( &buffer_create_info, nullptr, &buffer_context.buffer, &buffer_context.allocator_data); if (result != VK_SUCCESS) { @@ -1175,20 +1184,20 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ } VkMemoryRequirements memory_requirements; - _device_table->GetBufferMemoryRequirements(_device, buffer_context.buffer, &memory_requirements); + device_table_->GetBufferMemoryRequirements(device_, buffer_context.buffer, &memory_requirements); VkMemoryPropertyFlags memory_property_flags = use_host_mem ? VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT : VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; uint32_t memory_type_index = - graphics::GetMemoryTypeIndex(_memory_properties, memory_requirements.memoryTypeBits, memory_property_flags); + graphics::GetMemoryTypeIndex(memory_properties_, memory_requirements.memoryTypeBits, memory_property_flags); if (memory_type_index == std::numeric_limits::max()) { /* fallback to coherent */ memory_type_index = - graphics::GetMemoryTypeIndex(_memory_properties, memory_requirements.memoryTypeBits, memory_property_flags); + graphics::GetMemoryTypeIndex(memory_properties_, memory_requirements.memoryTypeBits, memory_property_flags); } GFXRECON_ASSERT(memory_type_index != std::numeric_limits::max()); @@ -1203,7 +1212,7 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ alloc_flags_info.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT; alloc_info.pNext = &alloc_flags_info; - result = _resource_allocator->AllocateMemoryDirect( + result = resource_allocator_->AllocateMemoryDirect( &alloc_info, nullptr, &buffer_context.device_memory, &buffer_context.memory_data); if (result != VK_SUCCESS) @@ -1212,7 +1221,7 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ } VkMemoryPropertyFlags memory_flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - result = _resource_allocator->BindBufferMemory(buffer_context.buffer, + result = resource_allocator_->BindBufferMemory(buffer_context.buffer, buffer_context.device_memory, 0, buffer_context.allocator_data, @@ -1227,7 +1236,7 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ VkBufferDeviceAddressInfo address_info = {}; address_info.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO; address_info.buffer = buffer_context.buffer; - buffer_context.device_address = _get_device_address_fn_(_device, &address_info); + buffer_context.device_address = get_device_address_fn_(device_, &address_info); GFXRECON_ASSERT(buffer_context.device_address != 0); // ensure alignment for returned address @@ -1236,7 +1245,7 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ if (use_host_mem) { // map buffer - result = _resource_allocator->MapResourceMemoryDirect( + result = resource_allocator_->MapResourceMemoryDirect( VK_WHOLE_SIZE, 0, &buffer_context.mapped_data, buffer_context.allocator_data); return result == VK_SUCCESS; } @@ -1259,7 +1268,7 @@ void VulkanAddressReplacer::barrier(VkCommandBuffer command_buffer, barrier.srcAccessMask = src_access; barrier.dstAccessMask = dst_access; - _device_table->CmdPipelineBarrier( + device_table_->CmdPipelineBarrier( command_buffer, src_stage, dst_stage, VkDependencyFlags(0), 0, nullptr, 1, &barrier, 0, nullptr); } @@ -1267,12 +1276,12 @@ void VulkanAddressReplacer::DestroyShadowResources(VkAccelerationStructureKHR ha { if (handle != VK_NULL_HANDLE) { - auto remove_as_it = _shadow_as_map.find(handle); + auto remove_as_it = shadow_as_map_.find(handle); - if (remove_as_it != _shadow_as_map.end()) + if (remove_as_it != shadow_as_map_.end()) { mark_injected_commands_helper_t mark_injected_commands_helper; - _shadow_as_map.erase(remove_as_it); + shadow_as_map_.erase(remove_as_it); } } } @@ -1281,28 +1290,28 @@ void VulkanAddressReplacer::DestroyShadowResources(VkCommandBuffer handle) { if (handle != VK_NULL_HANDLE) { - auto remove_context_it = _build_as_context_map.find(handle); + auto remove_context_it = build_as_context_map_.find(handle); - if (remove_context_it != _build_as_context_map.end()) + if (remove_context_it != build_as_context_map_.end()) { mark_injected_commands_helper_t mark_injected_commands_helper; - _build_as_context_map.erase(remove_context_it); + build_as_context_map_.erase(remove_context_it); } - auto shadow_sbt_it = _shadow_sbt_map.find(handle); + auto shadow_sbt_it = shadow_sbt_map_.find(handle); - if (shadow_sbt_it != _shadow_sbt_map.end()) + if (shadow_sbt_it != shadow_sbt_map_.end()) { mark_injected_commands_helper_t mark_injected_commands_helper; - _shadow_sbt_map.erase(shadow_sbt_it); + shadow_sbt_map_.erase(shadow_sbt_it); } - auto pipeline_sbt_it = _pipeline_sbt_context_map.find(handle); + auto pipeline_sbt_it = pipeline_sbt_context_map_.find(handle); - if (pipeline_sbt_it != _pipeline_sbt_context_map.end()) + if (pipeline_sbt_it != pipeline_sbt_context_map_.end()) { mark_injected_commands_helper_t mark_injected_commands_helper; - _pipeline_sbt_context_map.erase(pipeline_sbt_it); + pipeline_sbt_context_map_.erase(pipeline_sbt_it); } } } @@ -1312,8 +1321,8 @@ bool VulkanAddressReplacer::create_acceleration_asset(VulkanAddressReplacer::acc size_t num_buffer_bytes, size_t num_scratch_bytes) { - as_asset.device = _device; - as_asset.destroy_fn = _device_table->DestroyAccelerationStructureKHR; + as_asset.device = device_; + as_asset.destroy_fn = device_table_->DestroyAccelerationStructureKHR; // create a replacement acceleration-structure with proper sized buffer bool success = create_buffer( @@ -1332,7 +1341,7 @@ bool VulkanAddressReplacer::create_acceleration_asset(VulkanAddressReplacer::acc as_create_info.size = num_buffer_bytes; as_create_info.type = type; - VkResult res = _device_table->CreateAccelerationStructureKHR(_device, &as_create_info, nullptr, &as_asset.handle); + VkResult res = device_table_->CreateAccelerationStructureKHR(device_, &as_create_info, nullptr, &as_asset.handle); if (res != VK_SUCCESS || as_asset.handle == VK_NULL_HANDLE) { @@ -1343,8 +1352,8 @@ bool VulkanAddressReplacer::create_acceleration_asset(VulkanAddressReplacer::acc VkAccelerationStructureDeviceAddressInfoKHR acceleration_address_info = {}; acceleration_address_info.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR; acceleration_address_info.accelerationStructure = as_asset.handle; - as_asset.address = _device_table->GetAccelerationStructureDeviceAddressKHR(_device, &acceleration_address_info); - GFXRECON_ASSERT(as_asset.address != 0) + as_asset.address = device_table_->GetAccelerationStructureDeviceAddressKHR(device_, &acceleration_address_info); + GFXRECON_ASSERT(as_asset.address != 0); return true; } diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 325f8f8e5a..5a1a313ed2 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -55,6 +55,16 @@ class VulkanAddressReplacer ~VulkanAddressReplacer(); + /** + * @brief Set raytracing-related properties + * + * @param capture_properties capture-time raytracing pipeline properties + * @param replay_properties replay-time raytracing pipeline properties + * @param replay_as_properties replay-time acceleration-structure properties + */ + void SetRaytracingProperties(const VkPhysicalDeviceRayTracingPipelinePropertiesKHR& capture_properties, + const VkPhysicalDeviceRayTracingPipelinePropertiesKHR& replay_properties, + const VkPhysicalDeviceAccelerationStructurePropertiesKHR& replay_as_properties); /** * @brief ProcessCmdTraceRays will check and potentially correct input-parameters to 'vkCmdTraceRays', * like buffer-device-addresses and shader-group-handles. @@ -272,50 +282,50 @@ class VulkanAddressReplacer VkPipelineStageFlags dst_stage, VkAccessFlags dst_access); - const encode::VulkanDeviceTable* _device_table = nullptr; - VkPhysicalDeviceMemoryProperties _memory_properties = {}; - VkPhysicalDeviceRayTracingPipelinePropertiesKHR _capture_ray_properties{}, _replay_ray_properties{}; - VkPhysicalDeviceAccelerationStructurePropertiesKHR _replay_acceleration_structure_properties{}; - bool _valid_sbt_alignment = true; + const encode::VulkanDeviceTable* device_table_ = nullptr; + VkPhysicalDeviceMemoryProperties memory_properties_ = {}; + VkPhysicalDeviceRayTracingPipelinePropertiesKHR capture_ray_properties_{}, replay_ray_properties_{}; + VkPhysicalDeviceAccelerationStructurePropertiesKHR replay_acceleration_structure_properties_{}; + bool valid_sbt_alignment_ = true; - VkDevice _device = VK_NULL_HANDLE; - decode::VulkanResourceAllocator* _resource_allocator = nullptr; + VkDevice device_ = VK_NULL_HANDLE; + decode::VulkanResourceAllocator* resource_allocator_ = nullptr; // common layout used for all pipelines - VkPipelineLayout _pipeline_layout = VK_NULL_HANDLE; + VkPipelineLayout pipeline_layout_ = VK_NULL_HANDLE; // pipeline dealing with shader-binding-table (SBT), replacing group-handles - VkPipeline _pipeline_sbt = VK_NULL_HANDLE; + VkPipeline pipeline_sbt_ = VK_NULL_HANDLE; // pipeline dealing with buffer-device-addresses (BDA), replacing addresses - VkPipeline _pipeline_bda = VK_NULL_HANDLE; + VkPipeline pipeline_bda_ = VK_NULL_HANDLE; // required assets for submitting meta-commands - VkCommandPool _command_pool = VK_NULL_HANDLE; - VkCommandBuffer _command_buffer = VK_NULL_HANDLE; - VkFence _fence = VK_NULL_HANDLE; - VkQueue _queue = VK_NULL_HANDLE; - VkQueryPool _query_pool = VK_NULL_HANDLE; + VkCommandPool command_pool_ = VK_NULL_HANDLE; + VkCommandBuffer command_buffer_ = VK_NULL_HANDLE; + VkFence fence_ = VK_NULL_HANDLE; + VkQueue queue_ = VK_NULL_HANDLE; + VkQueryPool query_pool_ = VK_NULL_HANDLE; - util::linear_hashmap _hashmap_sbt; - util::linear_hashmap _hashmap_bda; - std::unordered_map _shadow_sbt_map; + util::linear_hashmap hashmap_sbt_; + util::linear_hashmap hashmap_bda_; + std::unordered_map shadow_sbt_map_; // pipeline-contexts dealing with shader-binding-tables, per command-buffer - std::unordered_map _pipeline_sbt_context_map; + std::unordered_map pipeline_sbt_context_map_; // resources related to acceleration-structures - std::unordered_map _shadow_as_map; + std::unordered_map shadow_as_map_; // pipeline-contexts dealing with acceleration-structure builds, per command-buffer - std::unordered_map _build_as_context_map; + std::unordered_map build_as_context_map_; // currently running compaction queries. pool -> AS -> query-pool-index - std::unordered_map> _as_compact_queries; - std::unordered_map _as_compact_sizes; + std::unordered_map> as_compact_queries_; + std::unordered_map as_compact_sizes_; // required function pointers - PFN_vkGetBufferDeviceAddress _get_device_address_fn_ = nullptr; + PFN_vkGetBufferDeviceAddress get_device_address_fn_ = nullptr; }; GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_device_address_tracker.cpp b/framework/decode/vulkan_device_address_tracker.cpp index 0200daa5aa..56d9935640 100644 --- a/framework/decode/vulkan_device_address_tracker.cpp +++ b/framework/decode/vulkan_device_address_tracker.cpp @@ -27,15 +27,15 @@ GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(decode) VulkanDeviceAddressTracker::VulkanDeviceAddressTracker(const VulkanObjectInfoTable& object_info_table) : - _object_info_table(object_info_table) + object_info_table_(object_info_table) {} void decode::VulkanDeviceAddressTracker::TrackBuffer(const decode::VulkanBufferInfo* buffer_info) { if (buffer_info != nullptr && buffer_info->capture_address != 0) { - _buffer_capture_addresses[buffer_info->capture_address] = buffer_info->capture_id; - _buffer_handles[buffer_info->handle] = buffer_info->capture_id; + buffer_capture_addresses_[buffer_info->capture_address] = buffer_info->capture_id; + buffer_handles_[buffer_info->handle] = buffer_info->capture_id; } } @@ -43,7 +43,7 @@ void VulkanDeviceAddressTracker::RemoveBuffer(const VulkanBufferInfo* buffer_inf { if (buffer_info != nullptr) { - _buffer_capture_addresses.erase(buffer_info->capture_address); + buffer_capture_addresses_.erase(buffer_info->capture_address); } } @@ -53,11 +53,11 @@ void VulkanDeviceAddressTracker::TrackAccelerationStructure( if (acceleration_structure_info != nullptr && acceleration_structure_info->capture_address != 0) { // track device address - _acceleration_structure_capture_addresses[acceleration_structure_info->capture_address] = + acceleration_structure_capture_addresses_[acceleration_structure_info->capture_address] = acceleration_structure_info->capture_id; // track vulkan-handle - _acceleration_structure_handles[acceleration_structure_info->handle] = acceleration_structure_info->capture_id; + acceleration_structure_handles_[acceleration_structure_info->handle] = acceleration_structure_info->capture_id; } } @@ -66,23 +66,23 @@ void VulkanDeviceAddressTracker::RemoveAccelerationStructure( { if (acceleration_structure_info != nullptr) { - _acceleration_structure_capture_addresses.erase(acceleration_structure_info->capture_id); + acceleration_structure_capture_addresses_.erase(acceleration_structure_info->capture_id); } } const decode::VulkanBufferInfo* decode::VulkanDeviceAddressTracker::GetBufferByCaptureDeviceAddress(VkDeviceAddress capture_address) const { - return GetBufferInfo(capture_address, _buffer_capture_addresses); + return GetBufferInfo(capture_address, buffer_capture_addresses_); } const VulkanBufferInfo* VulkanDeviceAddressTracker::GetBufferByHandle(VkBuffer handle) const { - auto handle_it = _buffer_handles.find(handle); - if (handle_it != _buffer_handles.end()) + auto handle_it = buffer_handles_.find(handle); + if (handle_it != buffer_handles_.end()) { const auto& [h, handle_id] = *handle_it; - const VulkanBufferInfo* found_buffer_info = _object_info_table.GetVkBufferInfo(handle_id); + const VulkanBufferInfo* found_buffer_info = object_info_table_.GetVkBufferInfo(handle_id); if (found_buffer_info != nullptr) { @@ -114,7 +114,7 @@ VulkanDeviceAddressTracker::GetBufferInfo(VkDeviceAddress } // found_address is lower or equal to device_address const auto& [found_address, buffer_handle] = *address_it; - const VulkanBufferInfo* found_buffer = _object_info_table.GetVkBufferInfo(buffer_handle); + const VulkanBufferInfo* found_buffer = object_info_table_.GetVkBufferInfo(buffer_handle); if (found_buffer != nullptr) { @@ -130,17 +130,11 @@ VulkanDeviceAddressTracker::GetBufferInfo(VkDeviceAddress const VulkanAccelerationStructureKHRInfo* VulkanDeviceAddressTracker::GetAccelerationStructureByCaptureDeviceAddress(VkDeviceAddress capture_address) const { - auto address_it = _acceleration_structure_capture_addresses.find(capture_address); - if (address_it != _acceleration_structure_capture_addresses.end()) + auto address_it = acceleration_structure_capture_addresses_.find(capture_address); + if (address_it != acceleration_structure_capture_addresses_.end()) { const auto& [found_address, acceleration_structure_handle] = *address_it; - const VulkanAccelerationStructureKHRInfo* found_acceleration_structure_info = - _object_info_table.GetVkAccelerationStructureKHRInfo(acceleration_structure_handle); - - if (found_acceleration_structure_info != nullptr) - { - return found_acceleration_structure_info; - } + return object_info_table_.GetVkAccelerationStructureKHRInfo(acceleration_structure_handle); } return nullptr; } @@ -148,17 +142,11 @@ VulkanDeviceAddressTracker::GetAccelerationStructureByCaptureDeviceAddress(VkDev [[nodiscard]] const VulkanAccelerationStructureKHRInfo* VulkanDeviceAddressTracker::GetAccelerationStructureByHandle(VkAccelerationStructureKHR handle) const { - auto handle_it = _acceleration_structure_handles.find(handle); - if (handle_it != _acceleration_structure_handles.end()) + auto handle_it = acceleration_structure_handles_.find(handle); + if (handle_it != acceleration_structure_handles_.end()) { const auto& [h, handle_id] = *handle_it; - const VulkanAccelerationStructureKHRInfo* found_acceleration_structure_info = - _object_info_table.GetVkAccelerationStructureKHRInfo(handle_id); - - if (found_acceleration_structure_info != nullptr) - { - return found_acceleration_structure_info; - } + return object_info_table_.GetVkAccelerationStructureKHRInfo(handle_id); } return nullptr; } @@ -167,10 +155,10 @@ std::unordered_map VulkanDeviceAddressTracker::GetAccelerationStructureDeviceAddressMap() const { std::unordered_map ret; - for (const auto& [address, handleId] : _acceleration_structure_capture_addresses) + for (const auto& [address, handleId] : acceleration_structure_capture_addresses_) { const VulkanAccelerationStructureKHRInfo* acceleration_structure_info = - _object_info_table.GetVkAccelerationStructureKHRInfo(handleId); + object_info_table_.GetVkAccelerationStructureKHRInfo(handleId); if (acceleration_structure_info != nullptr && acceleration_structure_info->replay_address != 0) { diff --git a/framework/decode/vulkan_device_address_tracker.h b/framework/decode/vulkan_device_address_tracker.h index c35fd09665..f5b4c04f66 100644 --- a/framework/decode/vulkan_device_address_tracker.h +++ b/framework/decode/vulkan_device_address_tracker.h @@ -121,12 +121,12 @@ class VulkanDeviceAddressTracker [[nodiscard]] const VulkanBufferInfo* GetBufferInfo(VkDeviceAddress device_address, const buffer_address_map_t& address_map) const; - const VulkanObjectInfoTable& _object_info_table; - buffer_address_map_t _buffer_capture_addresses; - std::unordered_map _acceleration_structure_capture_addresses; + const VulkanObjectInfoTable& object_info_table_; + buffer_address_map_t buffer_capture_addresses_; + std::unordered_map acceleration_structure_capture_addresses_; - std::unordered_map _buffer_handles; - std::unordered_map _acceleration_structure_handles; + std::unordered_map buffer_handles_; + std::unordered_map acceleration_structure_handles_; }; GFXRECON_END_NAMESPACE(decode) From 58a5de062e7df4d94b2e62d7c5d29b4d9e037fb2 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Wed, 15 Jan 2025 12:32:37 +0100 Subject: [PATCH 19/23] Extend state-tracking for VkPhysicalDeviceAccelerationStructurePropertiesKHR --- framework/decode/vulkan_address_replacer.cpp | 128 +++++++++++-------- framework/decode/vulkan_address_replacer.h | 30 +++-- framework/encode/vulkan_capture_manager.cpp | 9 ++ framework/encode/vulkan_handle_wrappers.h | 5 +- framework/encode/vulkan_state_tracker.cpp | 16 ++- framework/encode/vulkan_state_tracker.h | 4 + framework/encode/vulkan_state_writer.cpp | 14 +- framework/encode/vulkan_state_writer.h | 2 +- 8 files changed, 132 insertions(+), 76 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 9d27be98ae..4e6acfa94e 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -30,15 +30,15 @@ GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(decode) //! RAII helper to mark injected commands in scope -struct mark_injected_commands_helper_t +struct MarkInjectedCommandsHelper { - mark_injected_commands_helper_t() + MarkInjectedCommandsHelper() { // mark injected commands decode::BeginInjectedCommands(); } - ~mark_injected_commands_helper_t() + ~MarkInjectedCommandsHelper() { // mark end of injected commands decode::EndInjectedCommands(); @@ -46,7 +46,7 @@ struct mark_injected_commands_helper_t }; //! RAII helper submit a command-buffer to a queue and synchronize via fence -struct queue_submit_helper_t +struct QueueSubmitHelper { const encode::VulkanDeviceTable* device_table = nullptr; VkDevice device = VK_NULL_HANDLE; @@ -54,14 +54,14 @@ struct queue_submit_helper_t VkFence fence = VK_NULL_HANDLE; VkQueue queue = VK_NULL_HANDLE; - queue_submit_helper_t(const encode::VulkanDeviceTable* device_table_, - VkDevice device_, - VkCommandBuffer command_buffer_, - VkQueue queue_, - VkFence fence_) : + QueueSubmitHelper(const encode::VulkanDeviceTable* device_table_, + VkDevice device_, + VkCommandBuffer command_buffer_, + VkQueue queue_, + VkFence fence_) : device(device_), device_table(device_table_), command_buffer(command_buffer_), fence(fence_), queue(queue_) { - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; device_table->ResetFences(device, 1, &fence); @@ -73,9 +73,9 @@ struct queue_submit_helper_t device_table->BeginCommandBuffer(command_buffer, &command_buffer_begin_info); } - ~queue_submit_helper_t() + ~QueueSubmitHelper() { - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; device_table->EndCommandBuffer(command_buffer); @@ -166,33 +166,43 @@ VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* GFXRECON_ASSERT(device_info != nullptr && device_table != nullptr); const VulkanPhysicalDeviceInfo* physical_device_info = object_table.GetVkPhysicalDeviceInfo(device_info->parent_id); - device_ = device_info->handle; - resource_allocator_ = device_info->allocator.get(); + GFXRECON_ASSERT(physical_device_info != nullptr); + device_ = device_info->handle; + resource_allocator_ = device_info->allocator.get(); get_device_address_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_2 ? device_table->GetBufferDeviceAddress : device_table->GetBufferDeviceAddressKHR; - if (physical_device_info != nullptr && physical_device_info->capture_raytracing_properties && - physical_device_info->replay_device_info->raytracing_properties && - physical_device_info->replay_device_info->acceleration_structure_properties) + // get_physical_device_properties_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_1 ? + + if (physical_device_info != nullptr) { - SetRaytracingProperties(*physical_device_info->capture_raytracing_properties, - *physical_device_info->replay_device_info->raytracing_properties, - *physical_device_info->replay_device_info->acceleration_structure_properties); + GFXRECON_ASSERT(physical_device_info->replay_device_info != nullptr); + SetRaytracingProperties(physical_device_info->capture_raytracing_properties, + physical_device_info->replay_device_info->raytracing_properties, + physical_device_info->replay_device_info->acceleration_structure_properties); } - GFXRECON_ASSERT(physical_device_info->replay_device_info != nullptr); GFXRECON_ASSERT(physical_device_info->replay_device_info->memory_properties.has_value()); memory_properties_ = *physical_device_info->replay_device_info->memory_properties; } void VulkanAddressReplacer::SetRaytracingProperties( - const VkPhysicalDeviceRayTracingPipelinePropertiesKHR& capture_properties, - const VkPhysicalDeviceRayTracingPipelinePropertiesKHR& replay_properties, - const VkPhysicalDeviceAccelerationStructurePropertiesKHR& replay_as_properties) + const std::optional& capture_properties, + const std::optional& replay_properties, + const std::optional& replay_as_properties) { - capture_ray_properties_ = capture_properties; - replay_ray_properties_ = replay_properties; - replay_acceleration_structure_properties_ = replay_as_properties; + if (capture_properties) + { + capture_ray_properties_ = *capture_properties; + } + if (replay_properties) + { + replay_ray_properties_ = *replay_properties; + } + if (replay_as_properties) + { + replay_acceleration_structure_properties_ = *replay_as_properties; + } if (capture_ray_properties_.shaderGroupHandleSize != replay_ray_properties_.shaderGroupHandleSize || capture_ray_properties_.shaderGroupHandleAlignment != replay_ray_properties_.shaderGroupHandleAlignment || @@ -200,11 +210,16 @@ void VulkanAddressReplacer::SetRaytracingProperties( { valid_sbt_alignment_ = false; } + + if (!capture_properties || !replay_properties || !replay_as_properties) + { + GFXRECON_LOG_WARNING("VulkanAddressReplacer::SetRaytracingProperties: missing device-information") + } } VulkanAddressReplacer::~VulkanAddressReplacer() { - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; // explicitly free resources here, in order to mark destruction API-calls as injected pipeline_sbt_context_map_ = {}; @@ -314,7 +329,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( if (!valid_sbt_alignment_ || !valid_group_handles) { // mark injected commands - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; if (pipeline_sbt_ == VK_NULL_HANDLE) { @@ -424,7 +439,10 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // raygen: stride == size raygen_sbt->size = raygen_sbt->stride = replay_ray_properties_.shaderGroupBaseAlignment; - if (!create_buffer(shadow_buf_context, sbt_offset, VK_BUFFER_USAGE_SHADER_BINDING_TABLE_BIT_KHR)) + if (!create_buffer(shadow_buf_context, + sbt_offset, + VK_BUFFER_USAGE_SHADER_BINDING_TABLE_BIT_KHR, + replay_ray_properties_.shaderGroupBaseAlignment)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: shadow shader-binding-table creation failed"); return; @@ -549,7 +567,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( primitive_counts[j] = range_info->primitiveCount; } - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; device_table_->GetAccelerationStructureBuildSizesKHR(device_, VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, &build_geometry_info, @@ -612,13 +630,13 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // hot swap acceleration-structure handle build_geometry_info.dstAccelerationStructure = replacement_as.handle; + uint32_t min_scratch_offset_alignment = + replay_acceleration_structure_properties_ + ? replay_acceleration_structure_properties_->minAccelerationStructureScratchOffsetAlignment + : 128; + // create a replacement scratch-buffer - if (!create_buffer( - replacement_as.scratch, - scratch_size, - 0, - replay_acceleration_structure_properties_.minAccelerationStructureScratchOffsetAlignment, - false)) + if (!create_buffer(replacement_as.scratch, scratch_size, 0, min_scratch_offset_alignment, false)) { GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: scratch-buffer creation failed"); return; @@ -707,7 +725,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (!hashmap_bda_.empty()) { // mark injected commands - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; if (pipeline_bda_ == VK_NULL_HANDLE && !init_pipeline()) { @@ -807,9 +825,9 @@ void VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR( as_compact_sizes_.erase(info->dst); // tmp/debug: cleanup before merge -// GFXRECON_LOG_INFO( -// "VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR: found compacted AS-size: %ul", -// compact_size); + // GFXRECON_LOG_INFO( + // "VulkanAddressReplacer::ProcessCmdCopyAccelerationStructuresKHR: found compacted AS-size: + // %ul", compact_size); } } } @@ -845,6 +863,12 @@ void VulkanAddressReplacer::ProcessUpdateDescriptorSets(uint32_t de GFXRECON_UNREFERENCED_PARAMETER(descriptor_copy_count); GFXRECON_UNREFERENCED_PARAMETER(descriptor_copies); + // bail out if we're not tracking any shadow acceleration-structures + if (shadow_as_map_.empty()) + { + return; + } + for (uint32_t i = 0; i < descriptor_write_count; ++i) { VkWriteDescriptorSet& write = descriptor_writes[i]; @@ -882,7 +906,7 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( if (info_count > 0 && init_queue_assets()) { // reset/submit/sync command-buffer - queue_submit_helper_t queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); + QueueSubmitHelper queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); // dummy-wrapper VulkanCommandBufferInfo command_buffer_info = {}; @@ -891,7 +915,7 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand( &command_buffer_info, info_count, geometry_infos, range_infos, address_tracker); // issue build-command - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; device_table_->CmdBuildAccelerationStructuresKHR(command_buffer_, info_count, geometry_infos, range_infos); } } @@ -904,7 +928,7 @@ void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( if (copy_infos != nullptr && info_count > 0 && init_queue_assets()) { // reset/submit/sync command-buffer - queue_submit_helper_t queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); + QueueSubmitHelper queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); for (uint32_t i = 0; i < info_count; ++i) { @@ -915,7 +939,7 @@ void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand( ProcessCmdCopyAccelerationStructuresKHR(copy_info, address_tracker); // issue copy command - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; device_table_->CmdCopyAccelerationStructureKHR(command_buffer_, copy_info); } else @@ -932,12 +956,12 @@ void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMe if (init_queue_assets()) { // reset/submit/sync command-buffer - queue_submit_helper_t queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); + QueueSubmitHelper queue_submit_helper(device_table_, device_, command_buffer_, queue_, fence_); ProcessCmdWriteAccelerationStructuresPropertiesKHR(1, &acceleration_structure, query_type, query_pool_, 0); // issue vkCmdWriteAccelerationStructuresPropertiesKHR - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; device_table_->CmdWriteAccelerationStructuresPropertiesKHR( command_buffer_, 1, &acceleration_structure, query_type, query_pool_, 0); } @@ -945,7 +969,7 @@ void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMe VkDeviceSize compact_size = 0; // issue vkCmdWriteAccelerationStructuresPropertiesKHR - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; device_table_->GetQueryPoolResults(device_, query_pool_, 0, @@ -1280,7 +1304,7 @@ void VulkanAddressReplacer::DestroyShadowResources(VkAccelerationStructureKHR ha if (remove_as_it != shadow_as_map_.end()) { - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; shadow_as_map_.erase(remove_as_it); } } @@ -1294,7 +1318,7 @@ void VulkanAddressReplacer::DestroyShadowResources(VkCommandBuffer handle) if (remove_context_it != build_as_context_map_.end()) { - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; build_as_context_map_.erase(remove_context_it); } @@ -1302,7 +1326,7 @@ void VulkanAddressReplacer::DestroyShadowResources(VkCommandBuffer handle) if (shadow_sbt_it != shadow_sbt_map_.end()) { - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; shadow_sbt_map_.erase(shadow_sbt_it); } @@ -1310,7 +1334,7 @@ void VulkanAddressReplacer::DestroyShadowResources(VkCommandBuffer handle) if (pipeline_sbt_it != pipeline_sbt_context_map_.end()) { - mark_injected_commands_helper_t mark_injected_commands_helper; + MarkInjectedCommandsHelper mark_injected_commands_helper; pipeline_sbt_context_map_.erase(pipeline_sbt_it); } } diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 5a1a313ed2..673fb09e14 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -58,13 +58,14 @@ class VulkanAddressReplacer /** * @brief Set raytracing-related properties * - * @param capture_properties capture-time raytracing pipeline properties - * @param replay_properties replay-time raytracing pipeline properties - * @param replay_as_properties replay-time acceleration-structure properties + * @param capture_properties optional capture-time raytracing pipeline properties + * @param replay_properties optional replay-time raytracing pipeline properties + * @param replay_as_properties optional replay-time acceleration-structure properties */ - void SetRaytracingProperties(const VkPhysicalDeviceRayTracingPipelinePropertiesKHR& capture_properties, - const VkPhysicalDeviceRayTracingPipelinePropertiesKHR& replay_properties, - const VkPhysicalDeviceAccelerationStructurePropertiesKHR& replay_as_properties); + void SetRaytracingProperties( + const std::optional& capture_properties, + const std::optional& replay_properties, + const std::optional& replay_as_properties); /** * @brief ProcessCmdTraceRays will check and potentially correct input-parameters to 'vkCmdTraceRays', * like buffer-device-addresses and shader-group-handles. @@ -103,10 +104,10 @@ class VulkanAddressReplacer * * Depending on capture- and replay-device-properties this includes the following: * - * if replaying on same device/driver using the default-allocator (no --rebind): + * if replaying on same device/driver using the default-allocator (no -m rebind): * - happy day, nothing to do! * - * if replaying on a different device/driver OR using the rebind-allocator (via --rebind): + * if replaying on a different device/driver OR using the rebind-allocator (via -m rebind): * - remap buffer-device-addresses for triangle-, aabb- and instance-geometries referenced in `build_geometry_infos` * - check buffer-sizes for acceleration-structures and scratch-buffers * - if necessary, create shadow acceleration-structures and -buffers, adjust references @@ -282,11 +283,11 @@ class VulkanAddressReplacer VkPipelineStageFlags dst_stage, VkAccessFlags dst_access); - const encode::VulkanDeviceTable* device_table_ = nullptr; - VkPhysicalDeviceMemoryProperties memory_properties_ = {}; - VkPhysicalDeviceRayTracingPipelinePropertiesKHR capture_ray_properties_{}, replay_ray_properties_{}; - VkPhysicalDeviceAccelerationStructurePropertiesKHR replay_acceleration_structure_properties_{}; - bool valid_sbt_alignment_ = true; + const encode::VulkanDeviceTable* device_table_ = nullptr; + VkPhysicalDeviceMemoryProperties memory_properties_ = {}; + VkPhysicalDeviceRayTracingPipelinePropertiesKHR capture_ray_properties_{}, replay_ray_properties_{}; + std::optional replay_acceleration_structure_properties_{}; + bool valid_sbt_alignment_ = true; VkDevice device_ = VK_NULL_HANDLE; decode::VulkanResourceAllocator* resource_allocator_ = nullptr; @@ -325,7 +326,8 @@ class VulkanAddressReplacer std::unordered_map as_compact_sizes_; // required function pointers - PFN_vkGetBufferDeviceAddress get_device_address_fn_ = nullptr; + PFN_vkGetBufferDeviceAddress get_device_address_fn_ = nullptr; + PFN_vkGetPhysicalDeviceProperties2 get_physical_device_properties_fn_ = nullptr; }; GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/encode/vulkan_capture_manager.cpp b/framework/encode/vulkan_capture_manager.cpp index c572b81f18..8dfa7b6af1 100644 --- a/framework/encode/vulkan_capture_manager.cpp +++ b/framework/encode/vulkan_capture_manager.cpp @@ -1160,6 +1160,15 @@ void VulkanCaptureManager::OverrideGetPhysicalDeviceProperties2(VkPhysicalDevice state_tracker_->TrackRayTracingPipelineProperties(physicalDevice, raytracing_props); } } + + if (auto acceleration_props = + graphics::vulkan_struct_get_pnext(pProperties)) + { + if (IsCaptureModeTrack()) + { + state_tracker_->TrackAccelerationStructureProperties(physicalDevice, acceleration_props); + } + } } VkResult VulkanCaptureManager::OverrideGetPhysicalDeviceToolPropertiesEXT( diff --git a/framework/encode/vulkan_handle_wrappers.h b/framework/encode/vulkan_handle_wrappers.h index 940f394341..71859385be 100644 --- a/framework/encode/vulkan_handle_wrappers.h +++ b/framework/encode/vulkan_handle_wrappers.h @@ -146,8 +146,9 @@ struct PhysicalDeviceWrapper : public HandleWrapper std::unique_ptr queue_family_properties2; std::vector> queue_family_checkpoint_properties; - // Track RayTracingPipelinePropertiesKHR - std::optional ray_tracing_pipeline_properties; + // Track RayTracingPipeline / AccelerationStructure properties + std::optional ray_tracing_pipeline_properties; + std::optional acceleration_structure_properties; }; struct InstanceWrapper : public HandleWrapper diff --git a/framework/encode/vulkan_state_tracker.cpp b/framework/encode/vulkan_state_tracker.cpp index 8406bb4955..065436c6ef 100644 --- a/framework/encode/vulkan_state_tracker.cpp +++ b/framework/encode/vulkan_state_tracker.cpp @@ -1856,16 +1856,24 @@ void VulkanStateTracker::TrackRayTracingPipelineProperties( wrapper->ray_tracing_pipeline_properties->pNext = nullptr; } +void VulkanStateTracker::TrackAccelerationStructureProperties( + VkPhysicalDevice physicalDevice, + VkPhysicalDeviceAccelerationStructurePropertiesKHR* acceleration_structure_properties) +{ + auto wrapper = vulkan_wrappers::GetWrapper(physicalDevice); + wrapper->acceleration_structure_properties = *acceleration_structure_properties; + wrapper->acceleration_structure_properties->pNext = nullptr; +} + void VulkanStateTracker::TrackRayTracingShaderGroupHandles(VkDevice device, VkPipeline pipeline, size_t data_size, const void* data) { assert((device != VK_NULL_HANDLE) && (pipeline != VK_NULL_HANDLE)); - - auto wrapper = vulkan_wrappers::GetWrapper(pipeline); - const uint8_t* byte_data = reinterpret_cast(data); - wrapper->device_id = vulkan_wrappers::GetWrappedId(device); + auto wrapper = vulkan_wrappers::GetWrapper(pipeline); + auto* byte_data = reinterpret_cast(data); + wrapper->device_id = vulkan_wrappers::GetWrappedId(device); wrapper->shader_group_handle_data.assign(byte_data, byte_data + data_size); } diff --git a/framework/encode/vulkan_state_tracker.h b/framework/encode/vulkan_state_tracker.h index 652fe5f7e1..4b9a9b20c7 100644 --- a/framework/encode/vulkan_state_tracker.h +++ b/framework/encode/vulkan_state_tracker.h @@ -469,6 +469,10 @@ class VulkanStateTracker void TrackRayTracingPipelineProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceRayTracingPipelinePropertiesKHR* ray_properties); + void TrackAccelerationStructureProperties( + VkPhysicalDevice physicalDevice, + VkPhysicalDeviceAccelerationStructurePropertiesKHR* acceleration_structure_properties); + void TrackRayTracingShaderGroupHandles(VkDevice device, VkPipeline pipeline, size_t data_size, const void* data); void TrackAcquireFullScreenExclusiveMode(VkDevice device, VkSwapchainKHR swapchain); diff --git a/framework/encode/vulkan_state_writer.cpp b/framework/encode/vulkan_state_writer.cpp index 4a52fa1738..70368ee64e 100644 --- a/framework/encode/vulkan_state_writer.cpp +++ b/framework/encode/vulkan_state_writer.cpp @@ -137,7 +137,7 @@ uint64_t VulkanStateWriter::WriteState(const VulkanStateTable& state_table, uint StandardCreateWrite(state_table); // physical-device / raytracing properties - WriteRayTracingPipelinePropertiesState(state_table); + WriteRayTracingPropertiesState(state_table); // Utility object creation. StandardCreateWrite(state_table); @@ -1932,7 +1932,7 @@ void VulkanStateWriter::WriteGetAccelerationStructureDeviceAddressKHRCall( parameter_stream_.Clear(); } -void VulkanStateWriter::WriteRayTracingPipelinePropertiesState(const VulkanStateTable& state_table) +void VulkanStateWriter::WriteRayTracingPropertiesState(const VulkanStateTable& state_table) { state_table.VisitWrappers([&](const vulkan_wrappers::PhysicalDeviceWrapper* wrapper) { assert(wrapper != nullptr); @@ -1941,9 +1941,17 @@ void VulkanStateWriter::WriteRayTracingPipelinePropertiesState(const VulkanState { parameter_stream_.Clear(); encoder_.EncodeHandleIdValue(wrapper->handle_id); + + // pNext-chaining + auto pipeline_props = *wrapper->ray_tracing_pipeline_properties; + pipeline_props.pNext = wrapper->acceleration_structure_properties + ? (void*)&wrapper->acceleration_structure_properties.value() + : nullptr; + VkPhysicalDeviceProperties2 properties2 = {}; properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; - properties2.pNext = (void*)&wrapper->ray_tracing_pipeline_properties.value(); + properties2.pNext = &pipeline_props; + EncodeStructPtr(&encoder_, &properties2); WriteFunctionCall(format::ApiCall_vkGetPhysicalDeviceProperties2, ¶meter_stream_); parameter_stream_.Clear(); diff --git a/framework/encode/vulkan_state_writer.h b/framework/encode/vulkan_state_writer.h index 1070d12621..5122ee4eee 100644 --- a/framework/encode/vulkan_state_writer.h +++ b/framework/encode/vulkan_state_writer.h @@ -156,7 +156,7 @@ class VulkanStateWriter void WriteDeviceMemoryState(const VulkanStateTable& state_table); - void WriteRayTracingPipelinePropertiesState(const VulkanStateTable& state_table); + void WriteRayTracingPropertiesState(const VulkanStateTable& state_table); void WriteRayTracingShaderGroupHandlesState(const VulkanStateTable& state_table); From 97f20833a5343e8968a43611195f00d7b1766ddd Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Thu, 16 Jan 2025 09:17:34 +0100 Subject: [PATCH 20/23] Fetch VkPhysicalDeviceAccelerationStructurePropertiesKHR, in case nobody else did --- framework/decode/vulkan_address_replacer.cpp | 43 +++++++++++++------ framework/decode/vulkan_address_replacer.h | 2 + framework/decode/vulkan_decoder_base.cpp | 2 +- .../decode/vulkan_replay_consumer_base.cpp | 9 ++-- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 4e6acfa94e..099375c00f 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -160,20 +160,22 @@ decode::VulkanAddressReplacer::acceleration_structure_asset_t::~acceleration_str VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* device_info, const encode::VulkanDeviceTable* device_table, + const encode::VulkanInstanceTable* instance_table, const decode::CommonObjectInfoTable& object_table) : device_table_(device_table) { - GFXRECON_ASSERT(device_info != nullptr && device_table != nullptr); + GFXRECON_ASSERT(device_info != nullptr && device_table != nullptr && instance_table != nullptr); const VulkanPhysicalDeviceInfo* physical_device_info = object_table.GetVkPhysicalDeviceInfo(device_info->parent_id); GFXRECON_ASSERT(physical_device_info != nullptr); + physical_device_ = physical_device_info->handle; device_ = device_info->handle; resource_allocator_ = device_info->allocator.get(); get_device_address_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_2 ? device_table->GetBufferDeviceAddress : device_table->GetBufferDeviceAddressKHR; - // get_physical_device_properties_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_1 ? + get_physical_device_properties_fn_ = instance_table->GetPhysicalDeviceProperties2; if (physical_device_info != nullptr) { @@ -191,6 +193,9 @@ void VulkanAddressReplacer::SetRaytracingProperties( const std::optional& replay_properties, const std::optional& replay_as_properties) { + GFXRECON_ASSERT(capture_properties); + GFXRECON_ASSERT(replay_properties); + if (capture_properties) { capture_ray_properties_ = *capture_properties; @@ -210,11 +215,6 @@ void VulkanAddressReplacer::SetRaytracingProperties( { valid_sbt_alignment_ = false; } - - if (!capture_properties || !replay_properties || !replay_as_properties) - { - GFXRECON_LOG_WARNING("VulkanAddressReplacer::SetRaytracingProperties: missing device-information") - } } VulkanAddressReplacer::~VulkanAddressReplacer() @@ -591,9 +591,10 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( if (!as_buffer_usable || !scratch_buffer_usable) { + MarkInjectedCommandsHelper mark_injected_commands_helper; GFXRECON_LOG_INFO_ONCE( - "VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: Replay adjusted mismatching " - "acceleration-structures using shadow-structures and -buffers") + "VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: Replay is adjusting mismatching " + "acceleration-structures using shadow-structures and -buffers"); // now definitely requiring address-replacement force_replace = true; @@ -630,13 +631,27 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // hot swap acceleration-structure handle build_geometry_info.dstAccelerationStructure = replacement_as.handle; - uint32_t min_scratch_offset_alignment = - replay_acceleration_structure_properties_ - ? replay_acceleration_structure_properties_->minAccelerationStructureScratchOffsetAlignment - : 128; + // we did not populate the acceleration-structure yet (capture might not even contain that call) + if (!replay_acceleration_structure_properties_) + { + VkPhysicalDeviceAccelerationStructurePropertiesKHR as_properties = {}; + as_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_PROPERTIES_KHR; + as_properties.pNext = nullptr; + + VkPhysicalDeviceProperties2 physical_device_properties = {}; + physical_device_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + physical_device_properties.pNext = &as_properties; + get_physical_device_properties_fn_(physical_device_, &physical_device_properties); + replay_acceleration_structure_properties_ = as_properties; + } // create a replacement scratch-buffer - if (!create_buffer(replacement_as.scratch, scratch_size, 0, min_scratch_offset_alignment, false)) + if (!create_buffer( + replacement_as.scratch, + scratch_size, + 0, + replay_acceleration_structure_properties_->minAccelerationStructureScratchOffsetAlignment, + false)) { GFXRECON_LOG_ERROR("ProcessCmdBuildAccelerationStructuresKHR: scratch-buffer creation failed"); return; diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 673fb09e14..71c65673d5 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -45,6 +45,7 @@ class VulkanAddressReplacer VulkanAddressReplacer(const VulkanDeviceInfo* device_info, const encode::VulkanDeviceTable* device_table, + const encode::VulkanInstanceTable* instance_table, const decode::CommonObjectInfoTable& object_table); //! prevent copying @@ -289,6 +290,7 @@ class VulkanAddressReplacer std::optional replay_acceleration_structure_properties_{}; bool valid_sbt_alignment_ = true; + VkPhysicalDevice physical_device_ = VK_NULL_HANDLE; VkDevice device_ = VK_NULL_HANDLE; decode::VulkanResourceAllocator* resource_allocator_ = nullptr; diff --git a/framework/decode/vulkan_decoder_base.cpp b/framework/decode/vulkan_decoder_base.cpp index 4c34afd288..7040f8f90d 100644 --- a/framework/decode/vulkan_decoder_base.cpp +++ b/framework/decode/vulkan_decoder_base.cpp @@ -611,7 +611,7 @@ void VulkanDecoderBase::DispatchVulkanAccelerationStructuresWritePropertiesMetaC std::size_t bytes_read = ValueDecoder::DecodeHandleIdValue(parameter_buffer, sizeof(format::HandleId), &device_id); bytes_read += ValueDecoder::DecodeEnumValue(parameter_buffer + bytes_read, sizeof(VkQueryType), &query_type); - ValueDecoder::DecodeHandleIdValue( + bytes_read += ValueDecoder::DecodeHandleIdValue( parameter_buffer + bytes_read, sizeof(format::HandleId), &acceleration_structure_id); for (auto consumer : consumers_) diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index 14721c3ecd..2a3c1bac46 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -9455,9 +9455,12 @@ VulkanAddressReplacer& VulkanReplayConsumerBase::GetDeviceAddressReplacer(const auto it = _device_address_replacers.find(device_info); if (it == _device_address_replacers.end()) { - auto [new_it, success] = _device_address_replacers.insert( - { device_info, - VulkanAddressReplacer(device_info, GetDeviceTable(device_info->handle), *object_info_table_) }); + auto [new_it, success] = + _device_address_replacers.insert({ device_info, + VulkanAddressReplacer(device_info, + GetDeviceTable(device_info->handle), + GetInstanceTable(device_info->parent), + *object_info_table_) }); GFXRECON_ASSERT(success); return new_it->second; } From ec2788e3d33de1ac035ae1846d4bb692df33708a Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Thu, 16 Jan 2025 10:50:24 +0100 Subject: [PATCH 21/23] Consolidate alignment utils in dedicated header --- android/framework/util/CMakeLists.txt | 1 + framework/decode/vulkan_address_replacer.cpp | 85 +++++++---------- framework/decode/vulkan_address_replacer.h | 6 +- framework/decode/vulkan_rebind_allocator.cpp | 8 +- framework/util/CMakeLists.txt | 1 + framework/util/alignment_utils.h | 96 ++++++++++++++++++++ framework/util/linear_hashmap.h | 25 +---- 7 files changed, 139 insertions(+), 83 deletions(-) create mode 100644 framework/util/alignment_utils.h diff --git a/android/framework/util/CMakeLists.txt b/android/framework/util/CMakeLists.txt index b840d13f8a..83d0d27a85 100644 --- a/android/framework/util/CMakeLists.txt +++ b/android/framework/util/CMakeLists.txt @@ -2,6 +2,7 @@ add_library(gfxrecon_util STATIC "") target_sources(gfxrecon_util PRIVATE + ${GFXRECON_SOURCE_DIR}/framework/util/alignment_utils.h ${GFXRECON_SOURCE_DIR}/framework/util/argument_parser.h ${GFXRECON_SOURCE_DIR}/framework/util/argument_parser.cpp ${GFXRECON_SOURCE_DIR}/framework/util/buffer_writer.h diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 099375c00f..ff4f1e70ae 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -24,6 +24,7 @@ #include "decode/vulkan_address_replacer.h" #include "decode/vulkan_address_replacer_shaders.h" #include "decode/mark_injected_commands.h" +#include "util/alignment_utils.h" #include "util/logging.h" GFXRECON_BEGIN_NAMESPACE(gfxrecon) @@ -97,22 +98,6 @@ struct QueueSubmitHelper } }; -inline VkDeviceAddress aligned_address(VkDeviceAddress address, uint64_t alignment) -{ - return alignment ? (address + alignment - 1) & ~(alignment - 1) : address; -} - -inline uint32_t aligned_size(uint32_t size, uint32_t alignment) -{ - return alignment ? (size + alignment - 1) & ~(alignment - 1) : size; -} - -inline uint32_t div_up(uint32_t nom, uint32_t denom) -{ - GFXRECON_ASSERT(denom > 0); - return (nom + denom - 1) / denom; -} - struct hashmap_t { VkDeviceAddress storage; @@ -193,9 +178,6 @@ void VulkanAddressReplacer::SetRaytracingProperties( const std::optional& replay_properties, const std::optional& replay_as_properties) { - GFXRECON_ASSERT(capture_properties); - GFXRECON_ASSERT(replay_properties); - if (capture_properties) { capture_ray_properties_ = *capture_properties; @@ -209,11 +191,14 @@ void VulkanAddressReplacer::SetRaytracingProperties( replay_acceleration_structure_properties_ = *replay_as_properties; } - if (capture_ray_properties_.shaderGroupHandleSize != replay_ray_properties_.shaderGroupHandleSize || - capture_ray_properties_.shaderGroupHandleAlignment != replay_ray_properties_.shaderGroupHandleAlignment || - capture_ray_properties_.shaderGroupBaseAlignment != replay_ray_properties_.shaderGroupBaseAlignment) + if (capture_ray_properties_ && replay_ray_properties_) { - valid_sbt_alignment_ = false; + if (capture_ray_properties_->shaderGroupHandleSize != replay_ray_properties_->shaderGroupHandleSize || + capture_ray_properties_->shaderGroupHandleAlignment != replay_ray_properties_->shaderGroupHandleAlignment || + capture_ray_properties_->shaderGroupBaseAlignment != replay_ray_properties_->shaderGroupBaseAlignment) + { + valid_sbt_alignment_ = false; + } } } @@ -278,6 +263,13 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( return; } + if (capture_ray_properties_ == std::nullopt || replay_ray_properties_ == std::nullopt) + { + GFXRECON_LOG_ERROR_ONCE("VulkanAddressReplacer::ProcessCmdTraceRays: missing " + "VkPhysicalDeviceRayTracingPipelinePropertiesKHR for capture/replay, cannot proceed"); + return; + } + // figure out if the captured group-handles are valid for replay bool valid_group_handles = true; @@ -290,12 +282,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } } - // TODO: testing only -> remove when closing issue #1526 - // _valid_sbt_alignment = false; - // valid_group_handles = false; - std::unordered_set buffer_set; - auto address_remap = [&address_tracker, &buffer_set](VkStridedDeviceAddressRegionKHR* address_region) { if (address_region->size > 0) { @@ -311,13 +298,10 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // in-place address-remap address_region->deviceAddress = buffer_info->replay_address + offset; - } - else - { - GFXRECON_LOG_INFO_ONCE( - "VulkanAddressReplacer::ProcessCmdTraceRays: missing buffer_info->replay_address, remap failed") + return true; } } + return false; }; // in-place remap: capture-addresses -> replay-addresses @@ -419,16 +403,16 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( uint32_t sbt_offset = 0; auto& shadow_buf_context = shadow_sbt_map_[command_buffer_info->handle]; - const uint32_t handle_size_aligned = aligned_size(replay_ray_properties_.shaderGroupHandleSize, - replay_ray_properties_.shaderGroupHandleAlignment); + const auto handle_size_aligned = static_cast(util::aligned_value( + replay_ray_properties_->shaderGroupHandleSize, replay_ray_properties_->shaderGroupHandleAlignment)); for (auto& region : { raygen_sbt, miss_sbt, hit_sbt, callable_sbt }) { if (region != nullptr) { uint32_t num_handles_limit = region->size / region->stride; - uint32_t group_size = aligned_size(num_handles_limit * handle_size_aligned, - replay_ray_properties_.shaderGroupBaseAlignment); + auto group_size = static_cast(util::aligned_value( + num_handles_limit * handle_size_aligned, replay_ray_properties_->shaderGroupBaseAlignment)); sbt_offset += group_size; // adjust group-size @@ -437,12 +421,12 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } } // raygen: stride == size - raygen_sbt->size = raygen_sbt->stride = replay_ray_properties_.shaderGroupBaseAlignment; + raygen_sbt->size = raygen_sbt->stride = replay_ray_properties_->shaderGroupBaseAlignment; if (!create_buffer(shadow_buf_context, sbt_offset, VK_BUFFER_USAGE_SHADER_BINDING_TABLE_BIT_KHR, - replay_ray_properties_.shaderGroupBaseAlignment)) + replay_ray_properties_->shaderGroupBaseAlignment)) { GFXRECON_LOG_ERROR("VulkanAddressReplacer: shadow shader-binding-table creation failed"); return; @@ -475,7 +459,8 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( &replacer_params); // run a single workgroup constexpr uint32_t wg_size = 32; - device_table_->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); + device_table_->CmdDispatch( + command_buffer_info->handle, util::div_up(replacer_params.num_handles, wg_size), 1, 1); // post memory-barrier for (const auto& buf : buffer_set) @@ -533,13 +518,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( capture_address = buffer_info->replay_address + offset; return true; } - else - { - // GFXRECON_LOG_WARNING( - // "ProcessCmdBuildAccelerationStructuresKHR: missing buffer_info->replay_address, remap - // failed"); - return false; - } + return false; }; std::vector addresses_to_replace; @@ -788,7 +767,8 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( &replacer_params); // dispatch workgroups constexpr uint32_t wg_size = 32; - device_table_->CmdDispatch(command_buffer_info->handle, div_up(replacer_params.num_handles, wg_size), 1, 1); + device_table_->CmdDispatch( + command_buffer_info->handle, util::div_up(replacer_params.num_handles, wg_size), 1, 1); // post memory-barrier for (const auto& buf : buffer_set) @@ -1232,11 +1212,13 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ uint32_t memory_type_index = graphics::GetMemoryTypeIndex(memory_properties_, memory_requirements.memoryTypeBits, memory_property_flags); - if (memory_type_index == std::numeric_limits::max()) + if (memory_type_index == std::numeric_limits::max() && use_host_mem) { /* fallback to coherent */ memory_type_index = - graphics::GetMemoryTypeIndex(memory_properties_, memory_requirements.memoryTypeBits, memory_property_flags); + graphics::GetMemoryTypeIndex(memory_properties_, + memory_requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); } GFXRECON_ASSERT(memory_type_index != std::numeric_limits::max()); @@ -1279,7 +1261,8 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_ GFXRECON_ASSERT(buffer_context.device_address != 0); // ensure alignment for returned address - buffer_context.device_address = aligned_address(buffer_context.device_address, min_alignment); + buffer_context.device_address = util::aligned_value(buffer_context.device_address, min_alignment); + GFXRECON_ASSERT(!min_alignment || !(buffer_context.device_address % min_alignment)); if (use_host_mem) { diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 71c65673d5..79151b94c7 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -284,9 +284,9 @@ class VulkanAddressReplacer VkPipelineStageFlags dst_stage, VkAccessFlags dst_access); - const encode::VulkanDeviceTable* device_table_ = nullptr; - VkPhysicalDeviceMemoryProperties memory_properties_ = {}; - VkPhysicalDeviceRayTracingPipelinePropertiesKHR capture_ray_properties_{}, replay_ray_properties_{}; + const encode::VulkanDeviceTable* device_table_ = nullptr; + VkPhysicalDeviceMemoryProperties memory_properties_ = {}; + std::optional capture_ray_properties_{}, replay_ray_properties_{}; std::optional replay_acceleration_structure_properties_{}; bool valid_sbt_alignment_ = true; diff --git a/framework/decode/vulkan_rebind_allocator.cpp b/framework/decode/vulkan_rebind_allocator.cpp index a3084829d1..4901c3f157 100644 --- a/framework/decode/vulkan_rebind_allocator.cpp +++ b/framework/decode/vulkan_rebind_allocator.cpp @@ -56,7 +56,7 @@ #include "decode/vulkan_enum_util.h" #include "format/format.h" #include "format/format_util.h" -#include "util/logging.h" +#include "util/alignment_utils.h" #include "util/platform.h" #include "graphics/vulkan_struct_get_pnext.h" @@ -249,12 +249,8 @@ VkResult VulkanRebindAllocator::CreateBuffer(const VkBufferCreateInfo* create if ((create_info != nullptr) && (buffer != nullptr) && (allocator_data != nullptr)) { - auto aligned_size = [](uint32_t size, uint32_t alignment) -> uint32_t { - return (size + alignment - 1) & ~(alignment - 1); - }; - auto modified_info = *create_info; - modified_info.size = aligned_size(create_info->size, min_buffer_alignment_); + modified_info.size = util::aligned_value(create_info->size, min_buffer_alignment_); result = functions_.create_buffer(device_, &modified_info, nullptr, buffer); if (result >= 0) diff --git a/framework/util/CMakeLists.txt b/framework/util/CMakeLists.txt index 5289d49de4..9c30d3491e 100644 --- a/framework/util/CMakeLists.txt +++ b/framework/util/CMakeLists.txt @@ -30,6 +30,7 @@ add_library(gfxrecon_util STATIC "") target_sources(gfxrecon_util PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/alignment_utils.h ${CMAKE_CURRENT_LIST_DIR}/argument_parser.h ${CMAKE_CURRENT_LIST_DIR}/argument_parser.cpp ${CMAKE_CURRENT_LIST_DIR}/buffer_writer.h diff --git a/framework/util/alignment_utils.h b/framework/util/alignment_utils.h new file mode 100644 index 0000000000..303b68945b --- /dev/null +++ b/framework/util/alignment_utils.h @@ -0,0 +1,96 @@ +/* +** Copyright (c) 2025 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#ifndef GFXRECON_UTIL_ALIGNED_VALUE_H +#define GFXRECON_UTIL_ALIGNED_VALUE_H + +#include "util/defines.h" +#include "util/logging.h" + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(util) + +/** + * @brief is_pow_2 can be used to determine if value is a power-of-two. + * + * @param v an integer-value + * @return true, if provided value is a power-of-two. + */ +inline constexpr bool is_pow_2(uint64_t value) +{ + return !(value & (value - 1)); +} + +/** + * @brief next_pow_2 will return the next power of two, following a provided value. + * it will return value, if that was already a power-of-two. + * + * @param v provided integer-value + * @return v, if that already was a power-of-two, otherwise the next power-of-two following it. + */ +inline constexpr uint64_t next_pow_2(uint64_t v) +{ + if (is_pow_2(v)) + { + return v; + } + v--; + v |= v >> 1U; + v |= v >> 2U; + v |= v >> 4U; + v |= v >> 8U; + v |= v >> 16U; + v |= v >> 32U; + v++; + return v; +} + +/** + * @brief Return a value, equal or larger to a provided value, which is aligned to a provided alignment. + * + * @param value provided integer-value + * @param alignment provided alignment, needs to be a power-of-two + * @return an aligned value. + */ +inline constexpr uint64_t aligned_value(uint64_t value, uint64_t alignment) +{ + GFXRECON_ASSERT(is_pow_2(alignment)); + return alignment ? (value + alignment - 1) & ~(alignment - 1) : value; +} + +/** + * @brief Perform an integer-division, round up to the next value. + * + * @param nom provided nominator + * @param denom provided denominator + * @return result of integer-division, rounded to next value. + */ +inline constexpr uint32_t div_up(uint32_t nom, uint32_t denom) +{ + GFXRECON_ASSERT(denom > 0); + return (nom + denom - 1) / denom; +} + +GFXRECON_END_NAMESPACE(util) +GFXRECON_END_NAMESPACE(gfxrecon) + +#endif // #define GFXRECON_UTIL_ALIGNED_VALUE_H diff --git a/framework/util/linear_hashmap.h b/framework/util/linear_hashmap.h index ccb319b2ff..0138a3e251 100644 --- a/framework/util/linear_hashmap.h +++ b/framework/util/linear_hashmap.h @@ -23,6 +23,7 @@ #ifndef GFXRECONSTRUCT_UTIL_LINEAR_HASHMAP_H #define GFXRECONSTRUCT_UTIL_LINEAR_HASHMAP_H +#include "util/alignment_utils.h" #include "util/defines.h" #include "util/hash.h" #include @@ -33,28 +34,6 @@ GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(util) -inline constexpr bool is_pow_2(uint64_t v) -{ - return !(v & (v - 1)); -} - -inline constexpr uint64_t next_pow_2(uint64_t v) -{ - if (is_pow_2(v)) - { - return v; - } - v--; - v |= v >> 1U; - v |= v >> 2U; - v |= v >> 4U; - v |= v >> 8U; - v |= v >> 16U; - v |= v >> 32U; - v++; - return v; -} - /** * @brief linear_hashmap is a hashmap using open addressing with linear probing. * it can be used for POD key/value pairs and operates on a single array without chaining. @@ -84,7 +63,7 @@ class linear_hashmap } explicit linear_hashmap(uint64_t min_capacity) : - m_capacity(next_pow_2(min_capacity)), m_storage(std::make_unique(m_capacity)) + m_capacity(util::next_pow_2(min_capacity)), m_storage(std::make_unique(m_capacity)) { clear(); } From 5e0f610625d869fa979fc537e9cbce200d1126b6 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Thu, 16 Jan 2025 15:04:43 +0100 Subject: [PATCH 22/23] Repeat call to SetRaytracingProperties, if necessary. we might have miss it during init --- framework/decode/vulkan_address_replacer.cpp | 61 +++++++++++++------- framework/decode/vulkan_address_replacer.h | 6 +- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index ff4f1e70ae..791fc9be65 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -150,27 +150,25 @@ VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* device_table_(device_table) { GFXRECON_ASSERT(device_info != nullptr && device_table != nullptr && instance_table != nullptr); - - const VulkanPhysicalDeviceInfo* physical_device_info = object_table.GetVkPhysicalDeviceInfo(device_info->parent_id); - GFXRECON_ASSERT(physical_device_info != nullptr); - physical_device_ = physical_device_info->handle; + physical_device_info_ = object_table.GetVkPhysicalDeviceInfo(device_info->parent_id); + GFXRECON_ASSERT(physical_device_info_ != nullptr); device_ = device_info->handle; resource_allocator_ = device_info->allocator.get(); - get_device_address_fn_ = physical_device_info->parent_api_version >= VK_API_VERSION_1_2 + get_device_address_fn_ = physical_device_info_->parent_api_version >= VK_API_VERSION_1_2 ? device_table->GetBufferDeviceAddress : device_table->GetBufferDeviceAddressKHR; get_physical_device_properties_fn_ = instance_table->GetPhysicalDeviceProperties2; - if (physical_device_info != nullptr) + if (physical_device_info_ != nullptr) { - GFXRECON_ASSERT(physical_device_info->replay_device_info != nullptr); - SetRaytracingProperties(physical_device_info->capture_raytracing_properties, - physical_device_info->replay_device_info->raytracing_properties, - physical_device_info->replay_device_info->acceleration_structure_properties); + GFXRECON_ASSERT(physical_device_info_->replay_device_info != nullptr); + SetRaytracingProperties(physical_device_info_->capture_raytracing_properties, + physical_device_info_->replay_device_info->raytracing_properties, + physical_device_info_->replay_device_info->acceleration_structure_properties); } - GFXRECON_ASSERT(physical_device_info->replay_device_info->memory_properties.has_value()); - memory_properties_ = *physical_device_info->replay_device_info->memory_properties; + GFXRECON_ASSERT(physical_device_info_->replay_device_info->memory_properties.has_value()); + memory_properties_ = *physical_device_info_->replay_device_info->memory_properties; } void VulkanAddressReplacer::SetRaytracingProperties( @@ -263,13 +261,6 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( return; } - if (capture_ray_properties_ == std::nullopt || replay_ray_properties_ == std::nullopt) - { - GFXRECON_LOG_ERROR_ONCE("VulkanAddressReplacer::ProcessCmdTraceRays: missing " - "VkPhysicalDeviceRayTracingPipelinePropertiesKHR for capture/replay, cannot proceed"); - return; - } - // figure out if the captured group-handles are valid for replay bool valid_group_handles = true; @@ -387,9 +378,26 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( } else { - GFXRECON_LOG_INFO_ONCE("VulkanAddressReplacer::ProcessCmdTraceRays: Replay adjusted a mismatching " + GFXRECON_LOG_INFO_ONCE("VulkanAddressReplacer::ProcessCmdTraceRays: Replay is adjusting a mismatching " "raytracing shader-binding-table using a shadow-buffer"); + // raytracing-pipeline properties not populated yet, check if we missed it + if (capture_ray_properties_ == std::nullopt || replay_ray_properties_ == std::nullopt) + { + SetRaytracingProperties(physical_device_info_->capture_raytracing_properties, + physical_device_info_->replay_device_info->raytracing_properties, + physical_device_info_->replay_device_info->acceleration_structure_properties); + } + + // capture does contain the call, bail out + if (capture_ray_properties_ == std::nullopt || replay_ray_properties_ == std::nullopt) + { + GFXRECON_LOG_ERROR_ONCE( + "VulkanAddressReplacer::ProcessCmdTraceRays: missing " + "VkPhysicalDeviceRayTracingPipelinePropertiesKHR for capture/replay, cannot proceed"); + return; + } + // output-handles if (!create_buffer(pipeline_context_sbt.output_handle_buffer, max_num_handles * sizeof(VkDeviceAddress))) { @@ -610,7 +618,16 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // hot swap acceleration-structure handle build_geometry_info.dstAccelerationStructure = replacement_as.handle; - // we did not populate the acceleration-structure yet (capture might not even contain that call) + // acceleration-structure properties not populated yet, check if we missed it + if (!replay_acceleration_structure_properties_) + { + SetRaytracingProperties( + physical_device_info_->capture_raytracing_properties, + physical_device_info_->replay_device_info->raytracing_properties, + physical_device_info_->replay_device_info->acceleration_structure_properties); + } + + // capture did not contain the call, inject if (!replay_acceleration_structure_properties_) { VkPhysicalDeviceAccelerationStructurePropertiesKHR as_properties = {}; @@ -620,7 +637,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( VkPhysicalDeviceProperties2 physical_device_properties = {}; physical_device_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; physical_device_properties.pNext = &as_properties; - get_physical_device_properties_fn_(physical_device_, &physical_device_properties); + get_physical_device_properties_fn_(physical_device_info_->handle, &physical_device_properties); replay_acceleration_structure_properties_ = as_properties; } diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index 79151b94c7..bb155f7a1f 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -290,9 +290,9 @@ class VulkanAddressReplacer std::optional replay_acceleration_structure_properties_{}; bool valid_sbt_alignment_ = true; - VkPhysicalDevice physical_device_ = VK_NULL_HANDLE; - VkDevice device_ = VK_NULL_HANDLE; - decode::VulkanResourceAllocator* resource_allocator_ = nullptr; + const decode::VulkanPhysicalDeviceInfo* physical_device_info_ = nullptr; + VkDevice device_ = VK_NULL_HANDLE; + decode::VulkanResourceAllocator* resource_allocator_ = nullptr; // common layout used for all pipelines VkPipelineLayout pipeline_layout_ = VK_NULL_HANDLE; From 4c1c7a7d47f1798b7b812f89551653490a1ed093 Mon Sep 17 00:00:00 2001 From: Fabian Schmidt Date: Thu, 16 Jan 2025 16:59:46 +0100 Subject: [PATCH 23/23] simpler API for SetRaytracingProperties --- framework/decode/vulkan_address_replacer.cpp | 56 +++++++++---------- framework/decode/vulkan_address_replacer.h | 10 +--- .../decode/vulkan_device_address_tracker.cpp | 9 +-- 3 files changed, 30 insertions(+), 45 deletions(-) diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp index 791fc9be65..50ce76fcc1 100644 --- a/framework/decode/vulkan_address_replacer.cpp +++ b/framework/decode/vulkan_address_replacer.cpp @@ -159,34 +159,33 @@ VulkanAddressReplacer::VulkanAddressReplacer(const VulkanDeviceInfo* : device_table->GetBufferDeviceAddressKHR; get_physical_device_properties_fn_ = instance_table->GetPhysicalDeviceProperties2; - - if (physical_device_info_ != nullptr) - { - GFXRECON_ASSERT(physical_device_info_->replay_device_info != nullptr); - SetRaytracingProperties(physical_device_info_->capture_raytracing_properties, - physical_device_info_->replay_device_info->raytracing_properties, - physical_device_info_->replay_device_info->acceleration_structure_properties); - } - GFXRECON_ASSERT(physical_device_info_->replay_device_info->memory_properties.has_value()); - memory_properties_ = *physical_device_info_->replay_device_info->memory_properties; + SetRaytracingProperties(physical_device_info_); } -void VulkanAddressReplacer::SetRaytracingProperties( - const std::optional& capture_properties, - const std::optional& replay_properties, - const std::optional& replay_as_properties) +void VulkanAddressReplacer::SetRaytracingProperties(const decode::VulkanPhysicalDeviceInfo* physical_device_info) { - if (capture_properties) - { - capture_ray_properties_ = *capture_properties; - } - if (replay_properties) - { - replay_ray_properties_ = *replay_properties; - } - if (replay_as_properties) + if (physical_device_info != nullptr) { - replay_acceleration_structure_properties_ = *replay_as_properties; + physical_device_info_ = physical_device_info; + if (physical_device_info->capture_raytracing_properties) + { + capture_ray_properties_ = *physical_device_info->capture_raytracing_properties; + } + + if (physical_device_info->replay_device_info != nullptr) + { + if (physical_device_info->replay_device_info->raytracing_properties) + { + replay_ray_properties_ = *physical_device_info->replay_device_info->raytracing_properties; + } + if (physical_device_info->replay_device_info->acceleration_structure_properties) + { + replay_acceleration_structure_properties_ = + *physical_device_info->replay_device_info->acceleration_structure_properties; + } + GFXRECON_ASSERT(physical_device_info_->replay_device_info->memory_properties.has_value()); + memory_properties_ = *physical_device_info_->replay_device_info->memory_properties; + } } if (capture_ray_properties_ && replay_ray_properties_) @@ -384,9 +383,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays( // raytracing-pipeline properties not populated yet, check if we missed it if (capture_ray_properties_ == std::nullopt || replay_ray_properties_ == std::nullopt) { - SetRaytracingProperties(physical_device_info_->capture_raytracing_properties, - physical_device_info_->replay_device_info->raytracing_properties, - physical_device_info_->replay_device_info->acceleration_structure_properties); + SetRaytracingProperties(physical_device_info_); } // capture does contain the call, bail out @@ -621,10 +618,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR( // acceleration-structure properties not populated yet, check if we missed it if (!replay_acceleration_structure_properties_) { - SetRaytracingProperties( - physical_device_info_->capture_raytracing_properties, - physical_device_info_->replay_device_info->raytracing_properties, - physical_device_info_->replay_device_info->acceleration_structure_properties); + SetRaytracingProperties(physical_device_info_); } // capture did not contain the call, inject diff --git a/framework/decode/vulkan_address_replacer.h b/framework/decode/vulkan_address_replacer.h index bb155f7a1f..bcbc54aa08 100644 --- a/framework/decode/vulkan_address_replacer.h +++ b/framework/decode/vulkan_address_replacer.h @@ -59,14 +59,10 @@ class VulkanAddressReplacer /** * @brief Set raytracing-related properties * - * @param capture_properties optional capture-time raytracing pipeline properties - * @param replay_properties optional replay-time raytracing pipeline properties - * @param replay_as_properties optional replay-time acceleration-structure properties + * @param physical_device_info a physical-device info struct. */ - void SetRaytracingProperties( - const std::optional& capture_properties, - const std::optional& replay_properties, - const std::optional& replay_as_properties); + void SetRaytracingProperties(const decode::VulkanPhysicalDeviceInfo* physical_device_info); + /** * @brief ProcessCmdTraceRays will check and potentially correct input-parameters to 'vkCmdTraceRays', * like buffer-device-addresses and shader-group-handles. diff --git a/framework/decode/vulkan_device_address_tracker.cpp b/framework/decode/vulkan_device_address_tracker.cpp index 56d9935640..bd257944a9 100644 --- a/framework/decode/vulkan_device_address_tracker.cpp +++ b/framework/decode/vulkan_device_address_tracker.cpp @@ -81,13 +81,8 @@ const VulkanBufferInfo* VulkanDeviceAddressTracker::GetBufferByHandle(VkBuffer h auto handle_it = buffer_handles_.find(handle); if (handle_it != buffer_handles_.end()) { - const auto& [h, handle_id] = *handle_it; - const VulkanBufferInfo* found_buffer_info = object_info_table_.GetVkBufferInfo(handle_id); - - if (found_buffer_info != nullptr) - { - return found_buffer_info; - } + const auto& [h, handle_id] = *handle_it; + return object_info_table_.GetVkBufferInfo(handle_id); } return nullptr; }