// Copyright 2023 PICO Inc. All Rights Reserved. #include "PICO_FaceTrackingComponent.h" #include "AnimationRuntime.h" #include "Engine/SkeletalMesh.h" #include "PICO_MovementUtility.h" int UFaceTrackingComponentPICO::FTComponentCount = 0; // Sets default values for this component's properties UFaceTrackingComponentPICO::UFaceTrackingComponentPICO() : FTTargetMeshComponentName(NAME_None) , InvalidFaceDataResetTime(2.0f) , bUpdateFaceTracking(true) , Mode(EFaceTrackingModePICO::Default) , FTTargetMeshComponent(nullptr) { // Set this component to be initialized when the game starts, and to be ticked every frame. You can turn these features // off to improve performance if you don't need them. PrimaryComponentTick.bCanEverTick = true; PrimaryComponentTick.bStartWithTickEnabled = true; BlendShapeNameMapping.Add(EFaceBlendShapePICO::JawOpen, "jawOpen"); // ... } // Called when the game starts void UFaceTrackingComponentPICO::BeginPlay() { Super::BeginPlay(); bool Supported; TArray Modes; UMovementFunctionLibraryPICO::GetFaceTrackingSupportedPICO(Supported, Modes); if (!Supported) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Face tracking is not supported. (%s:%s)"), *GetOwner()->GetName(), *GetName()); SetComponentTickEnabled(false); return; } if (FTTargetMeshComponentName == NAME_None) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Invalid mesh component name. (%s:%s)"), *GetOwner()->GetName(), *GetName()); SetComponentTickEnabled(false); return; } if (!InitializeFaceTracking()) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Failed to initialize face tracking. (%s:%s)"), *GetOwner()->GetName(), *GetName()); SetComponentTickEnabled(false); return; } if (!UMovementFunctionLibraryPICO::StartFaceTrackingPICO(Mode)) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Failed to start face tracking. (%s: %s)"), *GetOwner()->GetName(), *GetName()); SetComponentTickEnabled(false); return; } ++FTComponentCount; } // Called every frame void UFaceTrackingComponentPICO::TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) { Super::TickComponent(DeltaTime, TickType, ThisTickFunction); if (!IsValid(FTTargetMeshComponent)) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("No target mesh specified. (%s:%s)"), *GetOwner()->GetName(), *GetName()); return; } FFaceStatePICO FaceState; if (bUpdateFaceTracking && UMovementFunctionLibraryPICO::GetFaceTrackingDataPICO(0, FaceState) && (FaceState.IsUpperFaceDataValid || FaceState.IsLowerFaceDataValid)) { InvalidFaceDataTimer = 0.0f; MorphTargetsManager.ResetMeshMorphTargetCurves(FTTargetMeshComponent); for (int32 FaceBlendShapeIndex = 0; FaceBlendShapeIndex < static_cast(XR_FACE_EXPRESSION_COUNT_BD); ++FaceBlendShapeIndex) { if (ValidBlendShape[FaceBlendShapeIndex]) { FName BlendShapeName = BlendShapeNameMapping[static_cast(FaceBlendShapeIndex)]; MorphTargetsManager.SetMeshMorphTargetValue(BlendShapeName, FaceState.FaceExpressionWeights[FaceBlendShapeIndex]); } } for (int32 FaceLipsyncBlendShapeIndex = XR_FACE_EXPRESSION_COUNT_BD; FaceLipsyncBlendShapeIndex < static_cast(XR_FACE_EXPRESSION_COUNT_BD + XR_LIP_EXPRESSION_COUNT_BD); ++FaceLipsyncBlendShapeIndex) { if (ValidBlendShape[FaceLipsyncBlendShapeIndex]) { FName BlendShapeName = BlendShapeNameMapping[static_cast(FaceLipsyncBlendShapeIndex)]; MorphTargetsManager.SetMeshMorphTargetValue(BlendShapeName, FaceState.LipsyncExpressionWeights[FaceLipsyncBlendShapeIndex - XR_FACE_EXPRESSION_COUNT_BD]); } } } else { InvalidFaceDataTimer += DeltaTime; if (InvalidFaceDataTimer >= InvalidFaceDataResetTime) { MorphTargetsManager.ResetMeshMorphTargetCurves(FTTargetMeshComponent); } } MorphTargetsManager.UpdateMeshMorphTargets(FTTargetMeshComponent); } void UFaceTrackingComponentPICO::EndPlay(const EEndPlayReason::Type EndPlayReason) { if (IsComponentTickEnabled()) { if (--FTComponentCount == 0) { if (!UMovementFunctionLibraryPICO::StopFaceTrackingPICO()) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Failed to stop face tracking. (%s: %s)"), *GetOwner()->GetName(), *GetName()); } } } Super::EndPlay(EndPlayReason); } void UFaceTrackingComponentPICO::SetBlendShapeValue(EFaceBlendShapePICO BlendShape, float Value) { if (BlendShape >= EFaceBlendShapePICO::COUNT) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Cannot set BlendShape value with invalid BlendShape index.")); return; } if (!ValidBlendShape[static_cast(BlendShape)]) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Cannot set BlendShape value for an BlendShape with an invalid associated morph target name. BlendShape name: %s"), *StaticEnum()->GetValueAsString(BlendShape)); return; } FName BlendShapeName = BlendShapeNameMapping[BlendShape]; MorphTargetsManager.SetMeshMorphTargetValue(BlendShapeName, Value); } float UFaceTrackingComponentPICO::GetBlendShapeValue(EFaceBlendShapePICO BlendShape) const { if (BlendShape >= EFaceBlendShapePICO::COUNT) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Cannot request BlendShape value using an invalid BlendShape index.")); return 0.0f; } FName BlendShapeName = BlendShapeNameMapping[BlendShape]; if (BlendShapeName == NAME_None) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Cannot request BlendShape value for an BlendShape with an invalid associated morph target name. BlendShape name: %s"), *StaticEnum()->GetValueAsString(BlendShape)); return 0.0f; } return MorphTargetsManager.GetMeshMorphTargetValue(BlendShapeName); } void UFaceTrackingComponentPICO::ClearBlendShapeValues() { MorphTargetsManager.EmptyMorphTargets(); } bool UFaceTrackingComponentPICO::InitializeFaceTracking() { FTTargetMeshComponent = FindComponentByNamePICO(GetOwner(), FTTargetMeshComponentName); if (!IsValid(FTTargetMeshComponent)) { UE_LOG(PICOOpenXRMovement, Warning, TEXT("Could not find skeletal mesh component with name: (%s). (%s:%s)"), *FTTargetMeshComponentName.ToString(), *GetOwner()->GetName(), *GetName()); return false; } if (FTTargetMeshComponent && FTTargetMeshComponent->GetSkinnedAsset()) { const TMap& MorphTargetIndexMap = Cast(FTTargetMeshComponent->GetSkinnedAsset())->GetMorphTargetIndexMap(); for (const auto& it : BlendShapeNameMapping) { ValidBlendShape[static_cast(it.Key)] = MorphTargetIndexMap.Contains(it.Value); } return true; } return false; } void FMorphTargetsManagerPICO::ResetMeshMorphTargetCurves(USkinnedMeshComponent* TargetMeshComponent) { if (TargetMeshComponent) { TargetMeshComponent->ActiveMorphTargets.Reset(); if (TargetMeshComponent->GetSkinnedAsset()) { TargetMeshComponent->MorphTargetWeights.SetNum(TargetMeshComponent->GetSkinnedAsset()->GetMorphTargets().Num()); if (TargetMeshComponent->MorphTargetWeights.Num() > 0) { FMemory::Memzero(TargetMeshComponent->MorphTargetWeights.GetData(), TargetMeshComponent->MorphTargetWeights.GetAllocatedSize()); } } else { TargetMeshComponent->MorphTargetWeights.Reset(); } } } void FMorphTargetsManagerPICO::UpdateMeshMorphTargets(USkinnedMeshComponent* TargetMeshComponent) { if (TargetMeshComponent && TargetMeshComponent->GetSkinnedAsset()) { if (MeshMorphTargetCurves.Num() > 0) { FAnimationRuntime::AppendActiveMorphTargets(Cast(TargetMeshComponent->GetSkinnedAsset()), MeshMorphTargetCurves, TargetMeshComponent->ActiveMorphTargets, TargetMeshComponent->MorphTargetWeights); } } } void FMorphTargetsManagerPICO::SetMeshMorphTargetValue(FName MorphTargetName, float Value) { float* CurveValPtr = MeshMorphTargetCurves.Find(MorphTargetName); bool bShouldAddToList = FPlatformMath::Abs(Value) > ZERO_ANIMWEIGHT_THRESH; if (bShouldAddToList) { if (CurveValPtr) { *CurveValPtr = Value; } else { MeshMorphTargetCurves.Add(MorphTargetName, Value); } } else { MeshMorphTargetCurves.Remove(MorphTargetName); } } float FMorphTargetsManagerPICO::GetMeshMorphTargetValue(FName MorphTargetName) const { const float* CurveValPtr = MeshMorphTargetCurves.Find(MorphTargetName); if (CurveValPtr) { return *CurveValPtr; } else { return 0.0f; } } void FMorphTargetsManagerPICO::EmptyMorphTargets() { MeshMorphTargetCurves.Empty(); }