Improvements for feature extraction: depth, camera, and scaled mesh t…

…ransform behaviour

-Support R16F pixel format used by the 16 bits depth feature extractor.
-Add the name of the camera viewpoint to the feature extractor's component, ensuring multi-viewpoint camera now captures from correct viewpoints.
-In the DataExport, get the transform matrix without scale before converting it to quaternion to prevent UE4's check inside that function.
-Only use SRGB and engine gamma for the captured texture with the RGBA8 format, not for other pixel format.  Reason:  when we encode exact number values into the texture, the gamma and SRGB settings will cause the read-back values to be different.

Source/Plugins/NVSceneCapturer/Source/NVSceneCapturer/Private/NVImageExporter.cpp

@@ -51,6 +51,8 @@ bool CanPixelFormatBeExported(EPixelFormat CheckPixelFormat)
 	case EPixelFormat::PF_A16B16G16R16:
 	case EPixelFormat::PF_G32R32F:
 	case EPixelFormat::PF_G8:
+    case EPixelFormat::PF_R8_UINT:
+	case EPixelFormat::PF_R16F:
 	case EPixelFormat::PF_ShadowDepth:
 		return true;
 	default:
@@ -228,7 +230,7 @@ TArray<uint8> FNVImageExporter::CompressImagePNG(const FNVTexturePixelData& Sour
             }
 			else
 			{
-				UE_LOG(LogNVSceneCapturer, Error, TEXT("Unsupported pixel format."));
+				UE_LOG(LogNVSceneCapturer, Error, TEXT("Invalid export image settings."));
 			}
         }
     }

Source/Plugins/NVSceneCapturer/Source/NVSceneCapturer/Private/NVSceneCaptureComponent2D.cpp

@@ -81,15 +81,28 @@ void UNVSceneCaptureComponent2D::InitTextureRenderTarget()
         // Automatically create a new texture if there's no valid TextureTarget specified
         const FName TextureTargetName = MakeUniqueObjectName(this, UTextureRenderTarget2D::StaticClass(), TEXT("SceneCaptureTextureTarget"));
         TextureTarget = NewObject<UTextureRenderTarget2D>(this, TextureTargetName);
-        ensure(TextureTarget);
-        if (TextureTarget)
+        if (ensure(TextureTarget))
         {
-            TextureTarget->TargetGamma = GEngine ? GEngine->GetDisplayGamma() : 0.f;
+            // Only use SRGB and engine gamma for the RGBA8 format, not for other pixel format.
+            // Check FTextureRenderTarget2DResource::InitDynamicRHI to see how they set flag bUseSRGB base on gamma
+            // NOTE: This code assume any other format than the RGBA8 encode exact number values into the texture and want to readback exact encoded value
+            // TODO: We may need to have a separated flag indicate whether we want to use gamma and/or SRGB on the render target or not
+            if (TextureTargetFormat == RTF_RGBA8)
+            {
+                TextureTarget->TargetGamma = GEngine ? GEngine->GetDisplayGamma() : 0.f;
+                TextureTarget->SRGB = true;
+            }
+            else
+            {
+                TextureTarget->TargetGamma = 1.f;
+                TextureTarget->SRGB = false;
+            }
+
             TextureTarget->bForceLinearGamma = false;
-            TextureTarget->SRGB = false;
             TextureTarget->bAutoGenerateMips = false;
             TextureTarget->bNeedsTwoCopies = true;
             TextureTarget->bGPUSharedFlag = true;
+
             if (OverrideTexturePixelFormat == EPixelFormat::PF_Unknown)
             {
                 TextureTarget->RenderTargetFormat = TextureTargetFormat;

Source/Plugins/NVSceneCapturer/Source/NVSceneCapturer/Private/NVSceneCapturerViewpointComponent.cpp

@@ -43,7 +43,8 @@ void UNVSceneCapturerViewpointComponent::SetupFeatureExtractors()
         if (FeatureExtractor && FeatureExtractor->IsEnabled())
         {
             UClass* ExtractorClass = FeatureExtractor->GetClass();
-            FName NewExtractorName = MakeUniqueObjectName(this, ExtractorClass, *FeatureExtractor->GetDisplayName());
+            FName NewExtractorName = FName(*FString::Printf(TEXT("%s.%s"), *this->GetName(), *FeatureExtractor->GetDisplayName()));
+            NewExtractorName = MakeUniqueObjectName(this, ExtractorClass, NewExtractorName);
             UNVSceneFeatureExtractor* NewSubFeatureExtractor = NewObject<UNVSceneFeatureExtractor>(GetOwner(),
                     ExtractorClass, NewExtractorName, EObjectFlags::RF_Transient, FeatureExtractor);
             if (NewSubFeatureExtractor)

Source/Plugins/NVSceneCapturer/Source/NVSceneCapturer/Private/NVSceneFeatureExtractor_DataExport.cpp

@@ -193,12 +193,14 @@ bool UNVSceneFeatureExtractor_AnnotationData::GatherActorData(const AActor* Chec
         const FRotator& ActorRotator_UE4 = ActorToWorldTransform.Rotator();
 
         // OpenCV coordinate system
-        ActorData.quaternion_worldspace = NVSceneCapturerUtils::ConvertQuaternionToOpenCVCoordinateSystem(ActorToWorldMatrix_UE4.ToQuat());
+        // NOTE: The actor transform matrix may included scaling, if we just use ToQuat(), UE4 will just return [0, 0, 0, 1] when the rotation matrix is not an identity matrix
+        // => we need to convert it first using GetMatrixWithoutScale
+        ActorData.quaternion_worldspace = NVSceneCapturerUtils::ConvertQuaternionToOpenCVCoordinateSystem(ActorToWorldMatrix_UE4.GetMatrixWithoutScale().ToQuat());
         ActorData.rotation_worldspace = ActorData.quaternion_worldspace.Rotator();
 
-        const FQuat& ActorCamQuat_OpenCV = ActorToCameraMatrix_OpenCV.ToQuat();
+        const FQuat& ActorCamQuat_OpenCV = ActorToCameraMatrix_OpenCV.GetMatrixWithoutScale().ToQuat();
         const FVector& ActorCamLocation_OpenCV = ActorToCameraMatrix_OpenCV.GetOrigin();
-        ActorData.quaternion_xyzw = NVSceneCapturerUtils::ConvertQuaternionToOpenCVCoordinateSystem(ActorToCameraMatrix_UE4.ToQuat());
+        ActorData.quaternion_xyzw = NVSceneCapturerUtils::ConvertQuaternionToOpenCVCoordinateSystem(ActorToCameraMatrix_UE4.GetMatrixWithoutScale().ToQuat());
         ActorData.rotation = ActorToCameraMatrix_UE4.Rotator();
 
         ActorData.actor_to_camera_matrix = ActorToCameraMatrix_OpenCV;

Source/Plugins/NVSceneCapturer/Source/NVSceneCapturer/Private/NVSceneFeatureExtractor_ImageExport.cpp

@@ -51,25 +51,22 @@ UNVSceneCaptureComponent2D* UNVSceneFeatureExtractor_PixelData::CreateSceneCaptu
     UNVSceneCaptureComponent2D* NewSceneCaptureComp2D = nullptr;
     if (OwnerViewpoint)
     {
-        const FName NewSceneCaptureComponentName = MakeUniqueObjectName(OwnerViewpoint, GetClass(),
-            *FString::Printf(TEXT("%s_%s"), *GetDisplayName(), *ComponentName));
-
         AActor* OwnerActor = OwnerCapturer ? OwnerCapturer : OwnerViewpoint->GetOwner();
+        const FName NewSceneCaptureComponentName = MakeUniqueObjectName(OwnerActor, GetClass(),
+            *FString::Printf(TEXT("%s_%s"), *GetName(), *ComponentName));
         NewSceneCaptureComp2D = NewObject<UNVSceneCaptureComponent2D>(OwnerActor,
             UNVSceneCaptureComponent2D::StaticClass(), NewSceneCaptureComponentName, EObjectFlags::RF_Transient);
 
         if (NewSceneCaptureComp2D)
         {
+            NewSceneCaptureComp2D->SetupAttachment(OwnerViewpoint);
             if (bOverrideShowFlagSettings)
             {
                 NewSceneCaptureComp2D->ShowFlagSettings = OverrideShowFlagSettings;
             }
 
             const auto& CapturerSettings = OwnerViewpoint->GetCapturerSettings();
 
-            FAttachmentTransformRules AttachmentTransformRule = FAttachmentTransformRules::SnapToTargetNotIncludingScale;
-            NewSceneCaptureComp2D->AttachToComponent(OwnerViewpoint, AttachmentTransformRule);
-
             NewSceneCaptureComp2D->TextureTargetSize = CapturerSettings.CapturedImageSize;
             NewSceneCaptureComp2D->FOVAngle = CapturerSettings.GetFOVAngle();
             if (CapturerSettings.bUseExplicitCameraIntrinsic)

Source/Plugins/NVSceneCapturer/Source/NVSceneCapturer/Public/NVSceneCaptureComponent2D.h

@@ -98,6 +98,8 @@ class NVSCENECAPTURER_API UNVSceneCaptureComponent2D : public USceneCaptureCompo
     UPROPERTY(EditAnywhere, Category = "SceneCapture")
     TEnumAsByte<ETextureRenderTargetFormat> TextureTargetFormat;
 
+    // Pixel format of the texture used to capture the scene
+    // NOTE: This can be different from the TextureTargetFormat
     UPROPERTY(EditAnywhere, Category = "SceneCapture")
     TEnumAsByte<EPixelFormat> OverrideTexturePixelFormat;