AR Anchor drifts or follows camera movement in Kotlin/ARCore

[Mayandi-icanio]

Problem Statement

When placing an AR anchor in front of the camera to capture an image, the anchor does not remain fixed in the physical environment. Instead, it maintains its relative position to the device, effectively "following" the camera as the user moves or rotates the phone.

Technical Context
Platform: Android

Language: Kotlin

Library: ARCore / Sceneview (specify if using Sceneform or Filament)

Observed Behavior
The user triggers an anchor placement at a specific distance (e.g., 1 meter) in front of the camera.

The anchor is rendered correctly initially.

As the phone moves left, right, or rotates, the anchor stays locked to the screen's viewport rather than staying at the physical GPS/feature point location where it was placed.

Expected Behavior
The anchor should be pinned to a specific point in the 3D world coordinates. Once placed, the user should be able to walk around the anchor or move the camera away from it without the anchor changing its world position.

package com.mn.anchorar

import android.Manifest
import android.content.pm.PackageManager
import android.util.Log
import android.widget.Toast
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.clickable
import androidx.compose.foundation.gestures.detectDragGestures
import androidx.compose.foundation.gestures.detectTapGestures
import androidx.compose.foundation.gestures.detectTransformGestures
import androidx.compose.foundation.interaction.MutableInteractionSource
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.material3.FloatingActionButton
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateListOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import com.google.ar.core.Config
import com.google.ar.core.Frame
import com.google.ar.core.Pose
import com.google.ar.core.Session
import com.google.ar.core.TrackingFailureReason
import com.google.ar.core.TrackingState
import dev.romainguy.kotlin.math.Quaternion
import io.github.sceneview.ar.ARScene
import io.github.sceneview.ar.ARSceneView
import io.github.sceneview.ar.rememberARCameraNode
import io.github.sceneview.math.Position
import io.github.sceneview.math.Size
import io.github.sceneview.node.CubeNode
import io.github.sceneview.node.Node
import io.github.sceneview.rememberCollisionSystem
import io.github.sceneview.rememberEngine
import io.github.sceneview.rememberEnvironmentLoader
import io.github.sceneview.rememberMaterialLoader
import io.github.sceneview.rememberModelLoader
import io.github.sceneview.rememberRenderer
import io.github.sceneview.rememberScene
import io.github.sceneview.rememberView
import android.os.Handler
import android.os.Looper
import kotlin.math.atan2
import kotlin.math.tan

@composable
fun ARModelScreen() {
val context = LocalContext.current
var hasPermission by remember {
mutableStateOf(
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED
)
}

val engine = rememberEngine()
val modelLoader = rememberModelLoader(engine)
val materialLoader = rememberMaterialLoader(engine)
val environmentLoader = rememberEnvironmentLoader(engine)
val view = rememberView(engine)

val planeRenderer = remember { mutableStateOf(true) }
val childNodes = remember { mutableStateListOf<Node>() }
val cameraNode = rememberARCameraNode(engine)
val trackingFailureReason = remember { mutableStateOf<TrackingFailureReason?>(null) }
val arSceneViewRef = remember { mutableStateOf<ARSceneView?>(null) }
val isSessionReady = remember { mutableStateOf(false) }
val frame = remember { mutableStateOf<Frame?>(null) }
val arSession = remember { mutableStateOf<Session?>(null) }
val isTakingPhoto = remember { mutableStateOf(false) }
var isARSessionPaused by remember { mutableStateOf(false) }

// Map of Node -> locked world Position so we can re-lock every frame
val fixedNodePositions = remember { mutableMapOf<Node, Triple<Position, Quaternion, Unit>>() }

val permissionLauncher = rememberLauncherForActivityResult(
    contract = ActivityResultContracts.RequestPermission()
) { granted ->
    hasPermission = granted
    if (!granted) {
        Toast.makeText(context, "Camera permission required", Toast.LENGTH_SHORT).show()
    }
}

LaunchedEffect(Unit) {
    if (!hasPermission) {
        permissionLauncher.launch(Manifest.permission.CAMERA)
    }
}

if (!hasPermission) return

fun isTrackingStable(currentFrame: Frame): Boolean {
    return currentFrame.camera.trackingState == TrackingState.TRACKING
}

fun captureArFrame() {
    val currentFrame = frame.value
    val session = arSession.value
    val arSceneView = arSceneViewRef.value

    if (currentFrame == null || session == null || !isSessionReady.value
        || isTakingPhoto.value || arSceneView == null
    ) {
        Log.e("AR_CAPTURE", "Cannot capture: missing requirements")
        return
    }
    if (currentFrame.camera.trackingState != TrackingState.TRACKING) {
        Log.e("AR_CAPTURE", "Camera is not tracking. Cannot capture.")
        return
    }
    if (!isTrackingStable(currentFrame)) {
        Log.e("AR_CAPTURE", "Camera tracking unstable. Cannot capture.")
        return
    }
    if (isARSessionPaused) {
        Log.e("AR_CAPTURE", "Session is paused. Cannot capture.")
        return
    }

    isTakingPhoto.value = true

    try {
        val cameraPose = currentFrame.camera.pose
        val intrinsics = currentFrame.camera.textureIntrinsics
        val focalLength = intrinsics.focalLength
        val imageDimensions = intrinsics.imageDimensions

        val verticalFov = 2 * atan2(imageDimensions[1].toFloat(), 2 * focalLength[1])
        val horizontalFov = 2 * atan2(imageDimensions[0].toFloat(), 2 * focalLength[0])



        val distance = 0.5f
        val forward = floatArrayOf(0f, 0f, -distance)
        val anchorPose = cameraPose.compose(Pose.makeTranslation(forward))

        // Snapshot the position and rotation at capture time — never updated again
        val lockedPosition = Position(
            anchorPose.tx(),
            anchorPose.ty(),
            anchorPose.tz()
        )
        val lockedRotation = Quaternion(
            anchorPose.qx(),
            anchorPose.qy(),
            anchorPose.qz(),
            anchorPose.qw()
        )

        Handler(Looper.getMainLooper()).postDelayed({
            val baseHeight = 2 * distance * tan(verticalFov / 2)
            val baseWidth  = 2 * distance * tan(horizontalFov / 2)

            val edgeThickness = 0.01f
            val frameDepth   = 0.01f

            fun createFrameMaterial() = materialLoader.createColorInstance(
                color = android.graphics.Color.argb(80, 255, 0, 0),
                metallic = 0.0f,
                roughness = 1.0f,
                reflectance = 0.0f
            )

            // ✅ KEY FIX: Use plain Node instead of AnchorNode.
            // AnchorNode re-syncs its world transform to the ARCore anchor every frame,
            // which causes the visible drift. A plain Node has no such tracking logic —
            // its worldPosition stays exactly where we set it.
            val worldNode = Node(engine = engine)
            worldNode.worldPosition = lockedPosition
            worldNode.worldQuaternion = lockedRotation
            worldNode.isEditable = false
            worldNode.isVisible = true

            // Top edge
            val topEdge = CubeNode(
                engine = engine,
                size = Size(baseWidth, edgeThickness, frameDepth),
                materialInstance = createFrameMaterial()
            )
            topEdge.position = Position(0f, baseHeight / 2, 0f)
            topEdge.isVisible = true
            worldNode.addChildNode(topEdge)

            // Bottom edge
            val bottomEdge = CubeNode(
                engine = engine,
                size = Size(baseWidth, edgeThickness, frameDepth),
                materialInstance = createFrameMaterial()
            )
            bottomEdge.position = Position(0f, -baseHeight / 2, 0f)
            bottomEdge.isVisible = true
            worldNode.addChildNode(bottomEdge)

            // Left edge
            val leftEdge = CubeNode(
                engine = engine,
                size = Size(edgeThickness, baseHeight, frameDepth),
                materialInstance = createFrameMaterial()
            )
            leftEdge.position = Position(-baseWidth / 2, 0f, 0f)
            leftEdge.isVisible = true
            worldNode.addChildNode(leftEdge)

            // Right edge
            val rightEdge = CubeNode(
                engine = engine,
                size = Size(edgeThickness, baseHeight, frameDepth),
                materialInstance = createFrameMaterial()
            )
            rightEdge.position = Position(baseWidth / 2, 0f, 0f)
            rightEdge.isVisible = true
            worldNode.addChildNode(rightEdge)

            // Overlay plane
            val overlayPlane = CubeNode(
                engine = engine,
                size = Size(baseWidth, baseHeight, 0.001f),
                materialInstance = createFrameMaterial()
            )
            overlayPlane.position = Position(0f, 0f, 0f)
            overlayPlane.isVisible = true
            worldNode.addChildNode(overlayPlane)

            // Register this node so onSessionUpdated can re-lock it every frame
            fixedNodePositions[worldNode] = Triple(lockedPosition, lockedRotation, Unit)

            childNodes.add(worldNode)
            Toast.makeText(context, "Capture placed", Toast.LENGTH_SHORT).show()
            isTakingPhoto.value = false
        }, 100)

    } catch (e: Exception) {
        Log.e("AR_CAPTURE", "Capture failed", e)
        Toast.makeText(context, "Capture failed: ${e.message}", Toast.LENGTH_SHORT).show()
        isTakingPhoto.value = false
    }
}

Box(modifier = Modifier.fillMaxSize()) {
    ARScene(
        modifier = Modifier
            .fillMaxSize()
            .pointerInput(Unit) {
                detectTapGestures { }
                detectDragGestures { _, _ -> }
                detectTransformGestures { _, _, _, _ -> }
            }
            .clickable(
                indication = null,
                interactionSource = remember { MutableInteractionSource() }
            ) { },
        childNodes = childNodes,
        engine = engine,
        modelLoader = modelLoader,
        planeRenderer = planeRenderer.value,
        cameraNode = cameraNode,
        materialLoader = materialLoader,
        view = view,
        renderer = rememberRenderer(engine),
        scene = rememberScene(engine),
        environmentLoader = environmentLoader,
        collisionSystem = rememberCollisionSystem(view),
        onTrackingFailureChanged = {
            trackingFailureReason.value = it
        },
        onViewCreated = {
            arSceneViewRef.value = this
        },
        onTouchEvent = { _, _ -> true },
        onSessionUpdated = { session, updatedFrame ->
            frame.value = updatedFrame
            arSession.value = session

            // ✅ Re-lock every fixed node every frame.
            // Even though plain Nodes don't auto-drift, ARCore's coordinate space
            // can subtly shift during scene remapping. This guarantees stability.
            fixedNodePositions.forEach { (node, triple) ->
                val (pos, rot, _) = triple
                node.worldPosition = pos
                node.worldQuaternion = rot
            }

            if (updatedFrame.camera.trackingState == TrackingState.TRACKING) {
                if (!isSessionReady.value) {
                    isSessionReady.value = true
                }
            }
        },
        onSessionPaused = {
            isARSessionPaused = true
        },
        onSessionResumed = {
            isARSessionPaused = false
        },
        sessionConfiguration = { session, config ->
            config.depthMode =
                when (session.isDepthModeSupported(Config.DepthMode.AUTOMATIC)) {
                    true -> Config.DepthMode.AUTOMATIC
                    else -> Config.DepthMode.DISABLED
                }
            config.planeFindingMode = Config.PlaneFindingMode.VERTICAL
            config.instantPlacementMode = Config.InstantPlacementMode.LOCAL_Y_UP
            config.focusMode = Config.FocusMode.AUTO
            config.updateMode = Config.UpdateMode.LATEST_CAMERA_IMAGE
            config.lightEstimationMode = Config.LightEstimationMode.DISABLED
        },
        onSessionFailed = { exception ->
            Toast.makeText(context, "AR error: ${exception.message}", Toast.LENGTH_SHORT).show()
        }
    )

    FloatingActionButton(
        onClick = { captureArFrame() },
        modifier = Modifier
            .align(Alignment.BottomCenter)
            .padding(32.dp)
    ) {
        Text(
            if (isTakingPhoto.value) "..." else "Capture",
            modifier = Modifier.padding(12.dp)
        )
    }
}

}