/****************************************************************************** * Copyright (C) Ultraleap, Inc. 2011-2024. * * * * Use subject to the terms of the Apache License 2.0 available at * * http://www.apache.org/licenses/LICENSE-2.0, or another agreement * * between Ultraleap and you, your company or other organization. * ******************************************************************************/ #include "LeapWidgetInteractionComponent.h" #include "DrawDebugHelpers.h" #include "Engine/StaticMesh.h" #include "Kismet/KismetMathLibrary.h" #include "LeapUtility.h" #include "UObject/ConstructorHelpers.h" #include "Engine/Engine.h" ULeapWidgetInteractionComponent::ULeapWidgetInteractionComponent() : LeapHandType(EHandType::LEAP_HAND_LEFT) , WidgetInteraction(EUIInteractionType::FAR) , CursorStaticMesh(nullptr) , MaterialBase(nullptr) , CursorSize(0.03) , bAutoMode(true) , IndexDistanceFromUI(0.0f) , HandVisibility(false) , LeapSubsystem(nullptr) , WristRotationFactor(0.0f) , InterpolationSpeed(10) , YAxisCalibOffset(4.0f) , ZAxisCalibOffset(4.0f) , ModeChangeThreshold(30.0f) , LeapPawn(nullptr) , PointerActor(nullptr) , World(nullptr) , LeapDynMaterial(nullptr) , PlayerCameraManager(nullptr) , bIsPinched(false) , bHandTouchWidget(false) , bAutoModeTrigger(false) , AxisRotOffset(45.0f) , TriggerFarOffset(20.0f) , FingerJointEstimatedLen(3.5f) , ShoulderWidth(15.0f) , PinchOffsetX(6.0f) , PinchOffsetY(2.0f) , bHidden(false) { CreatStaticMeshForCursor(); } void ULeapWidgetInteractionComponent::InitCalibrationArrays() { CalibratedHeadRot.Add({0, 0, 0}); // Look straight CalibratedHeadPos.Add({0, 0, 0}); CalibratedHeadRot.Add({-AxisRotOffset, 0, 0}); // Look down CalibratedHeadPos.Add({0, 0, -ZAxisCalibOffset}); CalibratedHeadRot.Add({AxisRotOffset, 0, 0}); // Look Up CalibratedHeadPos.Add({0, 0, ZAxisCalibOffset}); CalibratedHeadRot.Add({0, 0, -AxisRotOffset}); // Roll left CalibratedHeadPos.Add({0, -YAxisCalibOffset, -ZAxisCalibOffset / 2}); CalibratedHeadRot.Add({0, 0, AxisRotOffset}); // Roll right CalibratedHeadPos.Add({0, YAxisCalibOffset, -ZAxisCalibOffset / 2}); } ULeapWidgetInteractionComponent::~ULeapWidgetInteractionComponent() { OnRayComponentVisible.Clear(); } void ULeapWidgetInteractionComponent::CreatStaticMeshForCursor() { static ConstructorHelpers::FObjectFinder DefaultMesh(TEXT("StaticMesh'/Engine/BasicShapes/Sphere.Sphere'")); CursorStaticMesh = CreateDefaultSubobject(TEXT("CursorMesh")); if (CursorStaticMesh == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("CursorStaticMesh is nullptr in CreatStaticMeshForCursor()")); return; } CursorStaticMesh->SetCollisionEnabled(ECollisionEnabled::NoCollision); MaterialBase = LoadObject( nullptr, TEXT("Material'/UltraleapTracking/InteractionEngine/Materials/IE2_Materials/M_LaserPointer-Outer.M_LaserPointer-Outer'")); if (DefaultMesh.Succeeded()) { CursorStaticMesh->SetStaticMesh(DefaultMesh.Object); CursorStaticMesh->SetWorldScale3D(CursorSize * FVector::OneVector); } } void ULeapWidgetInteractionComponent::HandleDistanceChange(float Dist, float MinDistance) { // Adding 20 cm for the distance before we exit near field interactions float ExistDistance = MinDistance + TriggerFarOffset; if (Dist < MinDistance && WidgetInteraction == EUIInteractionType::FAR && !bAutoModeTrigger) { WidgetInteraction = EUIInteractionType::NEAR; // Broadcast event when the rays visbility change if (OnRayComponentVisible.IsBound()) { OnRayComponentVisible.Broadcast(false); } bAutoModeTrigger = true; if (bAutoMode) { CursorStaticMesh->SetHiddenInGame(true); SetHiddenInGame(true); } } else if (Dist > ExistDistance && WidgetInteraction == EUIInteractionType::NEAR && bAutoModeTrigger) { WidgetInteraction = EUIInteractionType::FAR; // Broadcast event when the rays visbility change if (OnRayComponentVisible.IsBound()) { OnRayComponentVisible.Broadcast(true); } bAutoModeTrigger = false; if (bAutoMode) { CursorStaticMesh->SetHiddenInGame(false); SetHiddenInGame(false); } } } void ULeapWidgetInteractionComponent::DrawLeapCursor(FLeapHandData& Hand) { FLeapHandData TmpHand = Hand; if (TmpHand.HandType != LeapHandType) { return; } if (CursorStaticMesh != nullptr && LeapPawn != nullptr && PlayerCameraManager != nullptr) { // The cursor position is the addition of the Pawn pose and the hand pose FVector Position = FVector::ZeroVector; FVector Direction = FVector(); FVector IndexDistalNext = TmpHand.Index.Distal.NextJoint; FVector IndexIntermNext = TmpHand.Index.Intermediate.NextJoint; FVector IndexMetaNext = TmpHand.Index.Metacarpal.NextJoint; FRotator ForwardRot = PlayerCameraManager->GetActorForwardVector().Rotation(); ForwardRot = FRotator(0, ForwardRot.Yaw, 0); FVector ForwardDirection = ForwardRot.Vector(); bool bNear = (WidgetInteraction == EUIInteractionType::NEAR); Position = bNear ? IndexIntermNext : IndexMetaNext; FVector FilteredPosition = Position; Direction = bNear ? (ForwardDirection) : GetHandRayDirection(TmpHand, FilteredPosition); FVector FilteredDirection = FVector::ZeroVector; FTransform TargetTrans = FTransform(); TargetTrans.SetLocation(FilteredPosition); TargetTrans.SetRotation(Direction.Rotation().Quaternion()); FTransform NewTransform = UKismetMathLibrary::TInterpTo(GetComponentTransform(), TargetTrans, World->GetDeltaSeconds(), InterpolationSpeed); FHitResult SweepHitResult; K2_SetWorldTransform(NewTransform, true, SweepHitResult, true); CursorStaticMesh->SetWorldLocation(LastHitResult.ImpactPoint); float Dist = FVector::Dist(Position, LastHitResult.ImpactPoint); if (WidgetInteraction == EUIInteractionType::NEAR) { FingerJointEstimatedLen = FVector::Dist(TmpHand.Index.Intermediate.PrevJoint, IndexDistalNext); if (Dist < (IndexDistanceFromUI + FingerJointEstimatedLen)) { NearClickLeftMouse(TmpHand.HandType); } // added 2 cm, cause of the jitter can cause accidental release // Also makes better user experience when using sliders else if (Dist > (IndexDistanceFromUI + FingerJointEstimatedLen + 2.0f)) { NearReleaseLeftMouse(TmpHand.HandType); } } // In auto mode, automatically enable FAR or NEAR interactions depending on the distance // between the hand and the widget // Also trigger event when visibility changed if (bAutoMode) { HandleDistanceChange(Dist); } } else { UE_LOG(UltraleapTrackingLog, Error, TEXT("nullptr in DrawLeapCircles")); } } FVector ULeapWidgetInteractionComponent::GetHandRayDirection(FLeapHandData& TmpHand, FVector& Position) { if (World == nullptr && PlayerCameraManager == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("World or PlayerCameraManager nullptr in GetHandRayDirection")); return FVector::ZeroVector; } // Use neck offset to overcome camera roll and pitch rotations FVector NeckOffset = GetNeckOffset(); // Get the neck postion FVector CameraLocationWithNeckOffset = PlayerCameraManager->GetCameraLocation(); CameraLocationWithNeckOffset -= NeckOffset; CameraLocationWithNeckOffset -= 15 * FVector::UpVector; // Get the estimated right direction of the camera FRotator RightRot = PlayerCameraManager->GetActorRightVector().Rotation(); RightRot = FRotator(0, RightRot.Yaw, 0); FVector RightDirection = RightRot.Vector(); // Get shoulders positions, with respect to the neck FVector ShoulderPos = FVector::ZeroVector; // Use the camera location with offset to overcome head Roll and Pitch ShoulderPos = CameraLocationWithNeckOffset; ShoulderPos += WristRotationFactor * PlayerCameraManager->GetActorForwardVector(); ShoulderPos += RightDirection * (TmpHand.HandType == EHandType::LEAP_HAND_LEFT ? -ShoulderWidth : ShoulderWidth); // Get approximate pintch position if (WidgetInteraction == EUIInteractionType::FAR) { Position += FVector::UpVector; Position += PinchOffsetX * PlayerCameraManager->GetActorForwardVector(); Position += PlayerCameraManager->GetActorRightVector() * (TmpHand.HandType == EHandType::LEAP_HAND_LEFT ? PinchOffsetY : -PinchOffsetY); } // Get the direction from the shoulders to the pinch position FVector Direction = Position - ShoulderPos; return Direction; } FVector ULeapWidgetInteractionComponent::GetNeckOffset() { if (PlayerCameraManager == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("PlayerCameraManager in GetNeckOffset")); return FVector(); } FRotator HMDRotation = PlayerCameraManager->GetCameraRotation(); float AlphaPitch = 0; float AlphaRoll = 0; FVector PitchOffset, RollOffset; if (HMDRotation.Pitch < 0) { AlphaPitch = UKismetMathLibrary::NormalizeToRange(-HMDRotation.Pitch, 0, -CalibratedHeadRot[1].Pitch); PitchOffset = FMath::Lerp(FVector(0), CalibratedHeadPos[1], AlphaPitch); } else { AlphaPitch = UKismetMathLibrary::NormalizeToRange(HMDRotation.Pitch, 0, CalibratedHeadRot[2].Pitch); PitchOffset = FMath::Lerp(FVector(0), CalibratedHeadPos[2], AlphaPitch); } if (HMDRotation.Roll < 0) { AlphaRoll = UKismetMathLibrary::NormalizeToRange(-HMDRotation.Roll, 0, -CalibratedHeadRot[3].Roll); RollOffset = FMath::Lerp(FVector(0), CalibratedHeadPos[3], AlphaRoll); } else { AlphaRoll = UKismetMathLibrary::NormalizeToRange(HMDRotation.Roll, 0, CalibratedHeadRot[4].Roll); RollOffset = FMath::Lerp(FVector(0), CalibratedHeadPos[4], AlphaRoll); } FVector Offset = PitchOffset + RollOffset; FRotator Rot = FRotator(0, HMDRotation.Yaw, 0); return Rot.RotateVector(Offset); } void ULeapWidgetInteractionComponent::BeginPlay() { Super::BeginPlay(); World = GetWorld(); if (GEngine == nullptr || World == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("GEngine or World is nullptr in BeginPlay")); return; } LeapSubsystem = ULeapSubsystem::Get(); if (LeapSubsystem == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("LeapSubsystem is nullptr in BeginPlay")); return; } LeapPawn = UGameplayStatics::GetPlayerPawn(World, 0); if (LeapPawn == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("LeapPawn is nullptr in BeginPlay")); return; } PlayerCameraManager = UGameplayStatics::GetPlayerCameraManager(World, 0); if (PlayerCameraManager == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("PlayerCameraManager is nullptr in BeginPlay")); return; } // Subscribe events from leap, for pinch, unpinch and get the tracking data if (WidgetInteraction != EUIInteractionType::NEAR) { LeapSubsystem->OnLeapPinchMulti.AddUObject(this, &ULeapWidgetInteractionComponent::OnLeapPinch); LeapSubsystem->OnLeapUnPinchMulti.AddUObject(this, &ULeapWidgetInteractionComponent::OnLeapUnPinch); } // Will need to get tracking data regardless of the interaction type (near or far) LeapSubsystem->OnLeapFrameMulti.AddUObject(this, &ULeapWidgetInteractionComponent::OnLeapTrackingData); LeapSubsystem->SetUsePawnOrigin(true, LeapPawn); if (MaterialBase != nullptr) { LeapDynMaterial = UMaterialInstanceDynamic::Create(MaterialBase, NULL); } else { UE_LOG(UltraleapTrackingLog, Error, TEXT("MaterialBase is nullptr in BeginPlay")); return; } if (LeapDynMaterial != nullptr && CursorStaticMesh != nullptr) { CursorStaticMesh->SetMaterial(0, LeapDynMaterial); FVector Scale = CursorSize * FVector::OneVector; CursorStaticMesh->SetWorldScale3D(Scale); } else { UE_LOG(UltraleapTrackingLog, Error, TEXT("LeapDynMaterial or StaticMesh is nullptr in BeginPlay")); return; } if (LeapHandType == EHandType::LEAP_HAND_LEFT) { PointerIndex = 0; } else { PointerIndex = 1; } //Hide on begin play CursorStaticMesh->SetHiddenInGame(true); SetHiddenInGame(true); InitCalibrationArrays(); } void ULeapWidgetInteractionComponent::OnLeapPinch(const FLeapHandData& HandData) { if (HandData.HandType == LeapHandType && !bIsPinched && WidgetInteraction == EUIInteractionType::FAR) { ScaleUpCursorAndClickButton(); bIsPinched = true; } } void ULeapWidgetInteractionComponent::OnLeapUnPinch(const FLeapHandData& HandData) { if (HandData.HandType == LeapHandType && bIsPinched && WidgetInteraction == EUIInteractionType::FAR) { ScaleDownCursorAndUnclickButton(); bIsPinched = false; } } void ULeapWidgetInteractionComponent::NearClickLeftMouse(TEnumAsByte HandType) { if (!bHandTouchWidget && HandType == LeapHandType) { ScaleUpCursorAndClickButton(); bHandTouchWidget = true; } } void ULeapWidgetInteractionComponent::NearReleaseLeftMouse(TEnumAsByte HandType) { if (bHandTouchWidget && HandType == LeapHandType) { ScaleDownCursorAndUnclickButton(); bHandTouchWidget = false; } } void ULeapWidgetInteractionComponent::ScaleUpCursorAndClickButton(const FKey Button) { if (bHidden) { return; } if (CursorStaticMesh != nullptr) { // Scale the cursor by 1/2 FVector Scale = CursorSize * FVector::OneVector; Scale = Scale / 2; CursorStaticMesh->SetWorldScale3D(Scale); } // Press the LeftMouseButton PressPointerKey(Button); } void ULeapWidgetInteractionComponent::ScaleDownCursorAndUnclickButton(const FKey Button) { if (bHidden) { return; } ResetCursorScale(); // Release the LeftMouseButton ReleasePointerKey(Button); } void ULeapWidgetInteractionComponent::ResetCursorScale() { if (CursorStaticMesh != nullptr) { // Scale the cursor by 2 FVector Scale = CursorStaticMesh->GetComponentScale(); if (FMath::IsNearlyEqual(Scale.X, (CursorSize / 2), 1.0E-2F)) { Scale = Scale * 2; CursorStaticMesh->SetWorldScale3D(Scale); } } } #if WITH_EDITOR void ULeapWidgetInteractionComponent::PostEditChangeProperty(FPropertyChangedEvent& PropertyChangedEvent) { Super::PostEditChangeProperty(PropertyChangedEvent); if (WidgetInteraction == EUIInteractionType::NEAR) { // Set the WidgetInteraction type to far when the distance is more than 30 if (InteractionDistance > ModeChangeThreshold && !bAutoMode) { WidgetInteraction = EUIInteractionType::FAR; } } } #endif void ULeapWidgetInteractionComponent::SpawnStaticMeshActor(const FVector& InLocation) { if (World == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("World is nullptr in SpawnStaticMeshActor")); return; } PointerActor = GetWorld()->SpawnActor(AStaticMeshActor::StaticClass()); if (PointerActor == nullptr) { UE_LOG(UltraleapTrackingLog, Error, TEXT("PointerActor is nullptr in SpawnStaticMeshActor")); return; } PointerActor->SetMobility(EComponentMobility::Movable); PointerActor->SetActorLocation(InLocation); UStaticMeshComponent* MeshComponent = PointerActor->GetStaticMeshComponent(); if (MeshComponent && CursorStaticMesh) { MeshComponent->SetStaticMesh(CursorStaticMesh->GetStaticMesh()); } else { UE_LOG(UltraleapTrackingLog, Error, TEXT("MeshComponent or StaticMesh is nullptr in SpawnStaticMeshActor")); } } void ULeapWidgetInteractionComponent::CleanUpEvents() { // Clean up the subsystem events if (LeapSubsystem != nullptr) { LeapSubsystem->OnLeapFrameMulti.Clear(); LeapSubsystem->OnLeapPinchMulti.Clear(); LeapSubsystem->OnLeapUnPinchMulti.Clear(); LeapSubsystem->SetUsePawnOrigin(false, nullptr); } } void ULeapWidgetInteractionComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) { Super::EndPlay(EndPlayReason); CleanUpEvents(); } void ULeapWidgetInteractionComponent::InitializeComponent() { } void ULeapWidgetInteractionComponent::HandleWidgetChange() { TWeakObjectPtr HitActor = nullptr; #if (ENGINE_MAJOR_VERSION >= 5 && ENGINE_MINOR_VERSION >= 1) HitActor = LastHitResult.GetActor(); #else HitActor = LastHitResult.Actor; #endif if (HitActor == nullptr) { bHidden = true; CursorStaticMesh->SetHiddenInGame(bHidden); SetHiddenInGame(bHidden); return; } if (HitActor->Tags.Contains(FName("UltraleapUMG"))) { if (bHidden) { bHidden = false; CursorStaticMesh->SetHiddenInGame(bHidden); SetHiddenInGame(bHidden); } } else { if (!bHidden) { bHidden = true; CursorStaticMesh->SetHiddenInGame(bHidden); SetHiddenInGame(bHidden); } } } void ULeapWidgetInteractionComponent::OnLeapTrackingData(const FLeapFrameData& Frame) { HandleWidgetChange(); TArray Hands = Frame.Hands; HandleVisibilityChange(Frame); for (int32 i = 0; i < Hands.Num(); ++i) { DrawLeapCursor(Hands[i]); } } void ULeapWidgetInteractionComponent::HandleVisibilityChange(const FLeapFrameData& Frame) { bool LatestHandVis = LeapHandType == EHandType::LEAP_HAND_LEFT ? Frame.LeftHandVisible : Frame.RightHandVisible; if (LatestHandVis != HandVisibility) { HandVisibility = LatestHandVis; CursorStaticMesh->SetHiddenInGame(!HandVisibility); SetHiddenInGame(!HandVisibility); } }