diff --git a/example/viam_example_app/ios/Flutter/AppFrameworkInfo.plist b/example/viam_example_app/ios/Flutter/AppFrameworkInfo.plist
index 7c569640062..1dc6cf7652b 100644
--- a/example/viam_example_app/ios/Flutter/AppFrameworkInfo.plist
+++ b/example/viam_example_app/ios/Flutter/AppFrameworkInfo.plist
@@ -21,6 +21,6 @@
CFBundleVersion
1.0
MinimumOSVersion
- 12.0
+ 13.0
diff --git a/example/viam_example_app/ios/Podfile b/example/viam_example_app/ios/Podfile
index 79d16c83ed2..53732e912f3 100644
--- a/example/viam_example_app/ios/Podfile
+++ b/example/viam_example_app/ios/Podfile
@@ -45,7 +45,7 @@ post_install do |installer|
installer.generated_projects.each do |project|
project.targets.each do |target|
target.build_configurations.each do |config|
- config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '12.0'
+ config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '13.0'
end
end
end
diff --git a/example/viam_example_app/ios/Runner.xcodeproj/project.pbxproj b/example/viam_example_app/ios/Runner.xcodeproj/project.pbxproj
index ded87b019f0..67cfee2bedb 100644
--- a/example/viam_example_app/ios/Runner.xcodeproj/project.pbxproj
+++ b/example/viam_example_app/ios/Runner.xcodeproj/project.pbxproj
@@ -453,7 +453,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
- IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 13.0;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SUPPORTED_PLATFORMS = iphoneos;
@@ -580,7 +580,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
- IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 13.0;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
@@ -629,7 +629,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
- IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 13.0;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SUPPORTED_PLATFORMS = iphoneos;
diff --git a/example/viam_example_app/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/example/viam_example_app/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme
index 8e3ca5dfe19..e3773d42e24 100644
--- a/example/viam_example_app/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme
+++ b/example/viam_example_app/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme
@@ -26,6 +26,7 @@
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+ customLLDBInitFile = "$(SRCROOT)/Flutter/ephemeral/flutter_lldbinit"
shouldUseLaunchSchemeArgsEnv = "YES">
diff --git a/example/viam_example_app/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/example/viam_example_app/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist
deleted file mode 100644
index 18d981003d6..00000000000
--- a/example/viam_example_app/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
- IDEDidComputeMac32BitWarning
-
-
-
diff --git a/example/viam_example_app/ios/Runner.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings b/example/viam_example_app/ios/Runner.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings
deleted file mode 100644
index f9b0d7c5ea1..00000000000
--- a/example/viam_example_app/ios/Runner.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
- PreviewsEnabled
-
-
-
diff --git a/example/viam_example_app/ios/Runner/Info.plist b/example/viam_example_app/ios/Runner/Info.plist
index 1ad391ce58f..3e3c2419a8f 100644
--- a/example/viam_example_app/ios/Runner/Info.plist
+++ b/example/viam_example_app/ios/Runner/Info.plist
@@ -8,6 +8,12 @@
_rpc._tcp
+ NSMotionUsageDescription
+ This app requires access to the barometer to provide altitude information.
+ NSCameraUsageDescription
+ This app requires camera access for AR functionality.
+ NSARKitUsageDescription
+ This app uses ARKit to track your phone's position in space.
CADisableMinimumFrameDurationOnPhone
CFBundleDevelopmentRegion
diff --git a/example/viam_example_app/lib/resources/arm_screen.dart b/example/viam_example_app/lib/resources/arm_screen.dart
new file mode 100644
index 00000000000..65f15a25d3b
--- /dev/null
+++ b/example/viam_example_app/lib/resources/arm_screen.dart
@@ -0,0 +1,32 @@
+import 'package:flutter/material.dart';
+import 'package:viam_example_app/resources/arm_widgets/arkit_widget.dart';
+import 'package:viam_example_app/resources/arm_widgets/imu_widget.dart';
+import 'package:viam_sdk/viam_sdk.dart';
+
+/// A widget to control an [Arm].
+class ViamArmWidgetNew extends StatelessWidget {
+ /// The [Arm]Expand commentComment on line R9ResolvedCode has comments. Press enter to view.
+ final Arm arm;
+
+ const ViamArmWidgetNew({
+ super.key,
+ required this.arm,
+ });
+
+ @override
+ Widget build(BuildContext context) {
+ return Column(
+ children: [
+ // ImuWidget(arm: arm, updateNotifier: ArmNotifier()),
+ ARKitArmWidget(arm: arm, updateNotifier: ArmNotifier()),
+ ],
+ );
+ }
+}
+class ArmNotifier extends ChangeNotifier {
+ ArmNotifier();
+
+ void update() {
+ notifyListeners();
+ }
+}
diff --git a/example/viam_example_app/lib/resources/arm_widgets/arkit_widget.dart b/example/viam_example_app/lib/resources/arm_widgets/arkit_widget.dart
new file mode 100644
index 00000000000..6e8a857e373
--- /dev/null
+++ b/example/viam_example_app/lib/resources/arm_widgets/arkit_widget.dart
@@ -0,0 +1,423 @@
+import 'dart:async';
+import 'dart:collection';
+import 'dart:math' as math;
+
+import 'package:arkit_plugin/arkit_plugin.dart';
+import 'package:flutter/material.dart';
+import 'package:vector_math/vector_math_64.dart' as vector_math;
+import 'package:viam_sdk/viam_sdk.dart';
+
+import '../../spatialmath/spatial_math.dart';
+
+class ARKitArmWidget extends StatefulWidget {
+ final Arm arm;
+
+ const ARKitArmWidget({
+ super.key,
+ required this.arm,
+ });
+
+ @override
+ State createState() => _ARKitArmWidgetState();
+}
+
+class _ARKitArmWidgetState extends State {
+ ARKitController? arkitController;
+
+ static const double _positionScale = 800.0; // 1m phone movement to 800mm (0.8m) arm movement
+ static const double _rotationDeadbandRad = 0.25;
+
+ bool _isMovingArm = false;
+ bool _isARKitInitialized = false;
+ String? _lastError;
+
+ // Queue for batching arm movements to reduce lag
+ final Queue _poseQueue = Queue();
+ int _poseCounter = 0;
+ bool _isProcessingQueue = false;
+
+ vector_math.Vector3 _currentPhonePositionARKit = vector_math.Vector3.zero();
+ vector_math.Matrix3 _currentPhoneRotationARKit = vector_math.Matrix3.identity();
+
+ // Store the starting states from the phone and arm
+ Pose? _referenceArmPose;
+ vector_math.Vector3? _referencePhonePositionARKit;
+ vector_math.Matrix3? _referencePhoneRotationARKit;
+ bool _isReferenceSet = false;
+
+ Pose? targetArmPose; // where we are trying to go
+ Pose? currentArmPose; // where we actually are
+
+ // Frame transformation from ARKit to Viam
+ // Viam Frame: Right-handed Z is up (X+ Forward, Y+ Left, Z+ Up)
+ // ARKit Frame: Right-handed Y is up (X+ Right, Y+ Up, Z+ Back (towards user))
+ // Full mapping: ARKit(X,Y,Z) → Viam(-Y,X,Z) rotated
+ late final vector_math.Quaternion _arkitToViamFrameTransform = () {
+ // 1: Rotate +90° around Z-axis (swaps X and Y)
+ final rotZ = vector_math.Quaternion.axisAngle(
+ vector_math.Vector3(0.0, 0.0, 1.0),
+ math.pi / 2,
+ );
+ // 2: Rotate -90° around X-axis (makes Y → Z)
+ final rotX = vector_math.Quaternion.axisAngle(
+ vector_math.Vector3(1.0, 0.0, 0.0),
+ -math.pi / 2,
+ );
+ // Quaternion multiplication is performed right to left
+ // Apply rotX first, then rotZ
+ return rotZ * rotX;
+ }();
+
+ late final vector_math.Quaternion inverseARKitToViamFrameTransform = vector_math.Quaternion(
+ -_arkitToViamFrameTransform.x,
+ -_arkitToViamFrameTransform.y,
+ -_arkitToViamFrameTransform.z,
+ _arkitToViamFrameTransform.w,
+ );
+
+ @override
+ void dispose() {
+ try {
+ if (arkitController != null) {
+ arkitController!.updateAtTime = null;
+ arkitController!.dispose();
+ }
+ } catch (e) {
+ debugPrint('Error disposing ARKit controller: $e');
+ }
+ _poseQueue.clear();
+ super.dispose();
+ }
+
+ void onARKitViewCreated(ARKitController controller) {
+ try {
+ arkitController = controller;
+
+ setState(() {
+ _isARKitInitialized = true;
+ _lastError = null;
+ });
+
+ // Called every frame (~60fps) - get camera transform and update position
+ arkitController!.updateAtTime = (time) {
+ if (!_isReferenceSet || arkitController == null) return;
+
+ arkitController!.pointOfViewTransform().then((transform) {
+ if (transform != null && mounted) {
+ try {
+ // Extract position from transform matrix (4th column)
+ _currentPhonePositionARKit = vector_math.Vector3(
+ transform[12],
+ transform[13],
+ transform[14],
+ );
+
+ // Extract orientation from transform matrix (upper-left 3x3 rotation matrix)
+ _currentPhoneRotationARKit = vector_math.Matrix3(
+ transform[0],
+ transform[1],
+ transform[2],
+ transform[4],
+ transform[5],
+ transform[6],
+ transform[8],
+ transform[9],
+ transform[10],
+ );
+ // Create pose and send to arm
+ _createPoseFromARKit();
+ } catch (e) {
+ if (mounted) {
+ setState(() {
+ _lastError = 'Error processing camera transform: $e';
+ });
+ }
+ }
+ }
+ }).catchError((e) {
+ if (mounted) {
+ setState(() {
+ _lastError = 'Error getting camera transform: $e';
+ });
+ }
+ });
+ };
+ } catch (e) {
+ setState(() {
+ _lastError = 'Failed to initialize ARKit: $e';
+ _isARKitInitialized = false;
+ });
+ }
+ }
+
+ /// Create a pose based on ARKit camera position and orientation
+ void _createPoseFromARKit() {
+ if (_referenceArmPose == null || _referencePhonePositionARKit == null || _referencePhoneRotationARKit == null) {
+ return;
+ }
+ try {
+ // Calculate new position in Viam frame
+ final positionDelta = _currentPhonePositionARKit - _referencePhonePositionARKit!;
+ final newX = _referenceArmPose!.x + (-positionDelta.z * _positionScale);
+ final newY = _referenceArmPose!.y + ((-positionDelta.x) * _positionScale);
+ final newZ = _referenceArmPose!.z + (positionDelta.y * _positionScale);
+
+ // Step 1: Calculate rotation delta between current and reference phone rotation.
+ // To find the difference between two quaternions, we can multiply the current quaternion by the inverse of the other quaternion
+ // 1a: Convert current phone rotation to quaternion
+ // 1b: Convert reference phone rotation to quaternion
+ // 1c: Calculate inverse of reference phone rotation
+ // 1d: Delta = Current * Inverse(Reference)
+ final currentRotationQuaternionARKit = vector_math.Quaternion.fromRotation(_currentPhoneRotationARKit);
+ final referenceRotationQuaternionARKit = vector_math.Quaternion.fromRotation(_referencePhoneRotationARKit!);
+ final inverseReferencePhoneQuaternion = vector_math.Quaternion(
+ -referenceRotationQuaternionARKit.x,
+ -referenceRotationQuaternionARKit.y,
+ -referenceRotationQuaternionARKit.z,
+ referenceRotationQuaternionARKit.w,
+ );
+ final rotationDeltaARKit = currentRotationQuaternionARKit * inverseReferencePhoneQuaternion;
+
+ // Step 2: Convert the rotation delta quaternion to an angle in radians
+ double rotationAngle = 2 * math.acos(rotationDeltaARKit.w.clamp(-1.0, 1.0));
+
+ // Step 3: Apply deadband filter, skip small rotations to reduce jitter
+ final OrientationVector newRotationOV;
+ if (rotationAngle < _rotationDeadbandRad) {
+ newRotationOV = OrientationVector(
+ _referenceArmPose!.theta,
+ _referenceArmPose!.oX,
+ _referenceArmPose!.oY,
+ _referenceArmPose!.oZ,
+ );
+ } else {
+ // Step 4: Convert rotation delta from ARKit frame to Viam frame
+ final rotationDeltaViam = _arkitToViamFrameTransform * rotationDeltaARKit * inverseARKitToViamFrameTransform;
+
+ // Step 5: Apply delta to reference arm orientation.
+ // 5a: Convert orientation values from reference pose to orientation vector
+ // 5b: Convert orientation vector to quaternion
+ // 5c: Apply delta to rotation quaternion to get new rotation quaternion
+ final referenceRotationOrientationVector = OrientationVector(
+ _referenceArmPose!.theta,
+ _referenceArmPose!.oX,
+ _referenceArmPose!.oY,
+ _referenceArmPose!.oZ,
+ );
+ final referenceRotationQuaternion = referenceRotationOrientationVector.toQuaternion();
+ // Convert spatial_math Quaternion to vector_math Quaternion for multiplication
+ final referenceRotationQuaternionViam = vector_math.Quaternion(
+ referenceRotationQuaternion.imag,
+ referenceRotationQuaternion.jmag,
+ referenceRotationQuaternion.kmag,
+ referenceRotationQuaternion.real,
+ );
+ final newRotationQuaternionViam = referenceRotationQuaternionViam * rotationDeltaViam;
+
+ // Step 6: Convert quaternions back to orientation vector to be used for the new pose
+ // 6a: Convert the new rotation quaternion to a spatial math quaternion
+ // 6b: Convert the spatial math quaternion to an orientation vector
+ final newRotationQuaternionSpatialMath =
+ Quaternion(newRotationQuaternionViam.w, newRotationQuaternionViam.x, newRotationQuaternionViam.y, newRotationQuaternionViam.z);
+ newRotationOV = newRotationQuaternionSpatialMath.toOrientationVectorRadians();
+ }
+
+ // Step 7: Create the new pose with the new rotation and position values
+ final newPose = Pose(
+ x: newX,
+ y: newY,
+ z: newZ,
+ theta: newRotationOV.theta,
+ oX: newRotationOV.ox,
+ oY: newRotationOV.oy,
+ oZ: newRotationOV.oz,
+ );
+
+ // Skip if pose hasn't changed significantly
+ if (_poseQueue.isNotEmpty) {
+ final lastPose = _poseQueue.last;
+ if ((newPose.x - lastPose.x).abs() < 1.0 && (newPose.y - lastPose.y).abs() < 1.0 && (newPose.z - lastPose.z).abs() < 1.0) {
+ return;
+ }
+ }
+
+ setState(() {
+ targetArmPose = newPose;
+ });
+
+ _addPoseToQueue(newPose, 5);
+
+ if (!_isProcessingQueue) {
+ _executePoseFromQueue();
+ }
+ } catch (e) {
+ setState(() {
+ _lastError = e.toString();
+ });
+ }
+ }
+
+ void _addPoseToQueue(Pose pose, int n) {
+ if (n == 0 || _poseCounter % n == 0) {
+ _poseQueue.addLast(pose);
+ }
+ _poseCounter++;
+ }
+
+ void _executePoseFromQueue() async {
+ _isProcessingQueue = true;
+ while (_poseQueue.isNotEmpty) {
+ if (!_isMovingArm) {
+ final poseToExecute = _poseQueue.last;
+ _poseQueue.removeLast();
+ _isMovingArm = true;
+ try {
+ await widget.arm.moveToPosition(poseToExecute);
+ setState(() {
+ currentArmPose = poseToExecute;
+ _lastError = null;
+ });
+ } catch (e) {
+ setState(() {
+ _lastError = e.toString();
+ });
+ }
+ _isMovingArm = false;
+ }
+ }
+ _isProcessingQueue = false;
+ _poseCounter = 0;
+ }
+
+ /// Set reference point
+ Future _setReference() async {
+ if (arkitController == null || !_isARKitInitialized) {
+ setState(() {
+ _lastError = 'ARKit is not initialized yet';
+ });
+ return;
+ }
+
+ try {
+ // Get the current arm position and store it as the reference
+ final currentPoseSetRef = await widget.arm.endPosition();
+ // Get current ARKit camera position
+ final transform = await arkitController!.pointOfViewTransform();
+
+ if (transform == null) {
+ setState(() {
+ _lastError = 'Failed to get ARKit camera transform';
+ });
+ return;
+ }
+
+ // Extract position (4th column of transform matrix)
+ final positionVector = vector_math.Vector3(
+ transform[12], // X
+ transform[13], // Y
+ transform[14], // Z
+ );
+
+ // Extract orientation (upper-left 3x3 submatrix from transform matrix)
+ final rotationMatrix = vector_math.Matrix3(
+ transform[0],
+ transform[1],
+ transform[2],
+ transform[4],
+ transform[5],
+ transform[6],
+ transform[8],
+ transform[9],
+ transform[10],
+ );
+
+ setState(() {
+ _poseQueue.clear();
+ _poseCounter = 0;
+
+ _referenceArmPose = currentPoseSetRef;
+ _referencePhonePositionARKit = positionVector;
+ _referencePhoneRotationARKit = rotationMatrix;
+
+ targetArmPose = currentPoseSetRef;
+ currentArmPose = currentPoseSetRef;
+ _isReferenceSet = true;
+ _lastError = null;
+ });
+ } catch (e) {
+ setState(() {
+ _lastError = "Failed to set reference: ${e.toString()}";
+ });
+ }
+ }
+
+ @override
+ Widget build(BuildContext context) {
+ return Column(
+ children: [
+ SizedBox(
+ height: 100,
+ width: 100,
+ child: ARKitSceneView(
+ onARKitViewCreated: onARKitViewCreated,
+ enableTapRecognizer: false,
+ showStatistics: false,
+ ),
+ ),
+ const Text("Target Position"),
+ Text("X: ${(targetArmPose?.x ?? 0.0).toStringAsFixed(1)} mm"),
+ Text("Y: ${(targetArmPose?.y ?? 0.0).toStringAsFixed(1)} mm"),
+ Text("Z: ${(targetArmPose?.z ?? 0.0).toStringAsFixed(1)} mm"),
+ Text("oX: ${(targetArmPose?.oX ?? 0.0).toStringAsFixed(2)}"),
+ Text("oY: ${(targetArmPose?.oY ?? 0.0).toStringAsFixed(2)}"),
+ Text("oZ: ${(targetArmPose?.oZ ?? 0.0).toStringAsFixed(2)}"),
+ Text("Theta: ${(targetArmPose?.theta ?? 0.0).toStringAsFixed(2)}"),
+ const Text("Current Position"),
+ Text("X: ${(currentArmPose?.x ?? 0.0).toStringAsFixed(1)} mm"),
+ Text("Y: ${(currentArmPose?.y ?? 0.0).toStringAsFixed(1)} mm"),
+ Text("Z: ${(currentArmPose?.z ?? 0.0).toStringAsFixed(1)} mm"),
+ Text("oX: ${(currentArmPose?.oX ?? 0.0).toStringAsFixed(2)}"),
+ Text("oY: ${(currentArmPose?.oY ?? 0.0).toStringAsFixed(2)}"),
+ Text("oZ: ${(currentArmPose?.oZ ?? 0.0).toStringAsFixed(2)}"),
+ Text("Theta: ${(currentArmPose?.theta ?? 0.0).toStringAsFixed(2)}"),
+ const SizedBox(height: 10),
+ Text("Status: ${!_isReferenceSet ? 'Ready' : _isMovingArm ? 'Moving...' : 'Active'}"),
+ if (_lastError != null) Text("Error: $_lastError"),
+ const SizedBox(height: 10),
+ GestureDetector(
+ onLongPressDown: _isARKitInitialized ? (_) => _setReference() : null,
+ onLongPressUp: () async {
+ await widget.arm.stop();
+ setState(() {
+ _isReferenceSet = false;
+ _poseQueue.clear();
+ _poseCounter = 0;
+ });
+ },
+ onLongPressCancel: () async {
+ await widget.arm.stop();
+ setState(() {
+ _isReferenceSet = false;
+ _poseQueue.clear();
+ _poseCounter = 0;
+ });
+ },
+ child: Container(
+ padding: const EdgeInsets.all(12),
+ decoration: BoxDecoration(
+ border: Border.all(),
+ ),
+ child: Text(_isReferenceSet ? "CONTROLLING - RELEASE TO STOP" : "HOLD TO CONTROL"),
+ ),
+ ),
+ TextButton(
+ onPressed: () async {
+ await widget.arm.moveToJointPositions([0.0, 0.0, 0.0, 0.0, 0.0, 0.0]);
+ await widget.arm.moveToPosition(Pose(x: 300, y: 0, z: 100, oX: 0, oY: 0, oZ: -1, theta: 0));
+ },
+ child: const Text("Reset Position"),
+ ),
+ ],
+ );
+ }
+}
diff --git a/example/viam_example_app/lib/resources/arm_widgets/imu_widget.dart b/example/viam_example_app/lib/resources/arm_widgets/imu_widget.dart
new file mode 100644
index 00000000000..736f988a4c2
--- /dev/null
+++ b/example/viam_example_app/lib/resources/arm_widgets/imu_widget.dart
@@ -0,0 +1,607 @@
+import 'dart:async';
+import 'dart:collection';
+
+import 'package:flutter/material.dart';
+import 'package:sensors_plus/sensors_plus.dart';
+import 'package:viam_example_app/resources/arm_screen.dart';
+import 'package:viam_sdk/viam_sdk.dart';
+
+import '../../spatialmath/spatial_math.dart';
+
+class ImuWidget extends StatefulWidget {
+ final Arm arm;
+ final ArmNotifier updateNotifier;
+ const ImuWidget({
+ super.key,
+ required this.arm,
+ required this.updateNotifier,
+ });
+ @override
+ State createState() => _ImuWidgetState();
+}
+
+class _ImuWidgetState extends State {
+ @override
+ void initState() {
+ _initImu();
+ super.initState();
+ }
+
+ final _streamSubscriptions = >[];
+ Duration sensorInterval = SensorInterval.uiInterval;
+
+ static const double _positionScale = 800.0;
+
+ static const double _velocityDecay = 0.99;
+
+ static const double _deadZoneZ = 0.5;
+ // static const double _deadZoneXY = 0.1;
+ static const double _deadZoneX = 0.18;
+ static const double _deadZoneY = 0.22;
+ static const double _velocityThreshold = 0.01; // Threshold below which velocity is considered zero
+ bool _isMovingArm = false;
+ String? _lastError;
+
+ // Pose queue for batching arm movements to reduce lag
+ final Queue _poseQueue = Queue();
+ int _poseCounter = 0;
+ bool _isProcessingQueue = false;
+
+ // Velocity (meters per second)
+ double _velocityX = 0.0;
+ double _velocityY = 0.0;
+ double _velocityZ = 0.0;
+ // Position (meters)
+ double _positionX = 0.0;
+ double _positionY = 0.0;
+ double _positionZ = 0.0;
+ DateTime? _lastIntegrationTime;
+
+ // Orientation (radians)
+ double _orientationX = 0.0; // Roll
+ double _orientationY = 0.0; // Pitch
+ double _orientationZ = 0.0; // Yaw
+ DateTime? _lastGyroIntegrationTime;
+
+ Pose? _referenceArmPose; // arm position set once when you press "set reference"
+ bool _isReferenceSet = false;
+ Pose? _targetArmPose;
+ Pose? _currentArmPose;
+ bool stillPressed = false;
+
+ void _initImu() {
+ _streamSubscriptions.add(
+ userAccelerometerEventStream(samplingPeriod: sensorInterval).listen(
+ _createPoseFromImu,
+ onError: (_) => _showSensorError("User Accelerometer"),
+ cancelOnError: true,
+ ),
+ );
+
+ /// Note: Orientation logic is commented out because we cannot convert to orientation vector without the spatial math package.
+ _streamSubscriptions.add(
+ gyroscopeEventStream(samplingPeriod: sensorInterval).listen(
+ _updateOrientationFromGyroscope,
+ onError: (_) => _showSensorError("Gyroscope"),
+ cancelOnError: true,
+ ),
+ );
+ }
+
+ void _showSensorError(String sensorName) {
+ showDialog(
+ context: context,
+ builder: (context) => AlertDialog(
+ title: const Text("Sensor Not Found"),
+ content: Text("It seems that your device doesn't support $sensorName Sensor"),
+ ),
+ );
+ }
+
+ /// Update orientation by integrating gyroscope angular velocity
+ void _updateOrientationFromGyroscope(GyroscopeEvent event) {
+ final now = DateTime.now();
+
+ // Initialize integration time on first run
+ if (_lastGyroIntegrationTime == null) {
+ _lastGyroIntegrationTime = now;
+ return;
+ }
+
+ // Calculate time delta between now and last integration time (in seconds) so we know how long we've been rotating for.
+ // tldr: to get orientation, we need angular velocity * time.
+ final dt = now.difference(_lastGyroIntegrationTime!).inMilliseconds / 1000.0;
+ _lastGyroIntegrationTime = now;
+
+ // Skip if dt is too large, meaning the phone has been stationary for too long in between movements.
+ if (dt > 0.5) {
+ return;
+ }
+
+ // Don't update orientation if reference point hasn't been set
+ if (!_isReferenceSet) {
+ return;
+ }
+
+ // Calucluate orientation change: integrate angular velocity over time to get orientation.
+ // Gyroscope values are in radians/second
+ // 1. angular velocity to angular position (euler angles) w integration
+ _orientationX += event.x * dt; // roll
+ _orientationY += event.y * dt; // pitch
+ _orientationZ += event.z * dt; // yaw
+
+ // Apply small decay to prevent drift
+ _orientationX *= 0.999;
+ _orientationY *= 0.999;
+ _orientationZ *= 0.999;
+ }
+
+ /// Create a pose based on IMU accelerometer data using acceleration to get position
+ /// The pose is added to a queue for sequential processing.
+ Future _createPoseFromImu(UserAccelerometerEvent event) async {
+ if (!_isReferenceSet || _referenceArmPose == null) {
+ return;
+ }
+
+ final now = DateTime.now();
+
+ // Initialize integration time on first run
+ if (_lastIntegrationTime == null) {
+ _lastIntegrationTime = now;
+ return;
+ }
+
+ // Calculate time delta between now and last integration time (in seconds)
+ // tldr: to get velocity, we need acceleration * time.
+ final dt = now.difference(_lastIntegrationTime!).inMilliseconds / 1000.0;
+ _lastIntegrationTime = now;
+
+ // Skip if dt is too large, meaning the phone has been stationary for too long in between movements.
+ if (dt > 0.5) {
+ return;
+ }
+
+ // Apply dead zone to acceleration to filter out noise
+ final accelX = event.x.abs() > _deadZoneX ? event.x : 0.0;
+ final accelY = event.y.abs() > _deadZoneY ? event.y : 0.0;
+ final accelZ = event.z.abs() > _deadZoneZ ? event.z : 0.0;
+ print("accelX: ${event.x}, accelY: ${event.y}, accelZ: ${event.z}");
+
+ if (stillPressed) {
+ // if (accelX == 0.0 && accelY == 0.0 && accelZ == 0.0) {
+ print("still pressed and accel is 0");
+ _velocityX = 0.0;
+ _velocityY = 0.0;
+ _velocityZ = 0.0;
+ return;
+ // }
+ }
+
+ // Calculate velocity
+ _velocityX += accelX * dt;
+ _velocityY += accelY * dt;
+ _velocityZ += accelZ * dt;
+
+ // Apply decay to prevent drift when stationary
+ _velocityX *= _velocityDecay;
+ _velocityY *= _velocityDecay;
+ _velocityZ *= _velocityDecay;
+
+ // Filter out very small velocities to prevent drift
+ if (_velocityX.abs() < _velocityThreshold) _velocityX = 0.0;
+ if (_velocityY.abs() < _velocityThreshold) _velocityY = 0.0;
+ if (_velocityZ.abs() < _velocityThreshold) _velocityZ = 0.0;
+
+ // Calculate position
+ _positionX += _velocityX * dt;
+ _positionY += _velocityY * dt;
+ _positionZ += _velocityZ * dt;
+ // print("positionX: $_positionX, positionY: $_positionY, positionZ: $_positionZ");
+
+ try {
+ // Calculate new target position based on reference + phone displacement
+ // referenceArmPose is the arm's position in the real world when we set the reference
+ final newX = _referenceArmPose!.x + (_positionY * _positionScale);
+ final newY = _referenceArmPose!.y + ((-_positionX) * _positionScale);
+ final newZ = _referenceArmPose!.z + (_positionZ * _positionScale);
+
+ // Attempted to convert orientation values from angular velocities (yaw pitch roll) to orientation vectors
+ // Then convert orientation vectors to quaternions and multiply them to get the new arm orientation quaternion
+ // 2. convert euler angles to quaternion
+ final phoneEulerAngles = EulerAngles(
+ _orientationX, // roll
+ _orientationY, // pitch
+ _orientationZ // yaw
+ );
+ final phoneQuaternion = phoneEulerAngles.toQuaternion();
+
+ // 3: Convert latest pose to quaternion
+ final armOrientationVector =
+ OrientationVector(_referenceArmPose!.theta, _referenceArmPose!.oX, _referenceArmPose!.oY, _referenceArmPose!.oZ);
+ final armQuaternion = armOrientationVector.toQuaternion();
+
+ // 4: add quaternions (which is actually multiplying them which there is now a function for)
+ final newArmQuaternion = armQuaternion.mul(phoneQuaternion);
+
+ // 5. convert the new quaternion back to orientation vector
+ final newOrientationVector = newArmQuaternion.toOrientationVectorRadians();
+ final newOrientationX = newOrientationVector.ox;
+ final newOrientationY = newOrientationVector.oy;
+ final newOrientationZ = newOrientationVector.oz;
+ final newTheta = newOrientationVector.theta;
+ print("theta: $newTheta");
+ // print("newOrientationX: $newOrientationX, newOrientationY: $newOrientationY, newOrientationZ: $newOrientationZ, newTheta: $newTheta");
+ // 6. add pose to queue
+ final newPose = Pose(
+ // x: newX,
+ // y: newY,
+ // z: newZ,
+ x: _referenceArmPose!.x,
+ y: _referenceArmPose!.y,
+ z: _referenceArmPose!.z,
+ theta: newTheta,
+ oX: newOrientationX,
+ oY: newOrientationY,
+ oZ: newOrientationZ,
+ );
+
+ if (_poseQueue.isNotEmpty) {
+ if (newPose.x == _poseQueue.last.x &&
+ newPose.y == _poseQueue.last.y &&
+ newPose.z == _poseQueue.last.z &&
+ newPose.oX == _poseQueue.last.oX &&
+ newPose.oY == _poseQueue.last.oY &&
+ newPose.oZ == _poseQueue.last.oZ) {
+ return;
+ }
+ }
+ setState(() {
+ _targetArmPose = newPose;
+ });
+
+ // Add pose to queue
+ _addPoseToQueue(newPose, 5);
+
+ if (!_isProcessingQueue) {
+ _executePoseFromQueue();
+ }
+ } catch (e) {
+ setState(() {
+ _lastError = e.toString();
+ });
+ }
+ }
+
+ // Add every nth pose to the queue,
+ void _addPoseToQueue(Pose pose, int n) {
+ if (n == 0 || _poseCounter % n == 0) {
+ _poseQueue.addLast(pose);
+ // debugPrint("new pose added to queue: ${pose.x}, ${pose.y}, ${pose.z}");
+ }
+ _poseCounter++;
+ }
+
+ void _executePoseFromQueue() async {
+ _isProcessingQueue = true;
+ while (_poseQueue.isNotEmpty) {
+ if (!_isMovingArm) {
+ final poseToExecute = _poseQueue.first;
+ _poseQueue.removeFirst();
+ _isMovingArm = true;
+ try {
+ await widget.arm.moveToPosition(poseToExecute);
+ setState(() {
+ _currentArmPose = poseToExecute;
+ _lastError = null;
+ });
+ } catch (e) {
+ setState(() {
+ _lastError = e.toString();
+ });
+ }
+ _isMovingArm = false;
+ }
+ }
+ // debugPrint("queue is empty");
+ _isProcessingQueue = false;
+ _poseCounter = 0;
+ }
+
+ /// Set reference point
+ /// Gets the arm's current position in the real world and stores it as the reference.
+ /// Clears position and orientation tracking so the phone starts at (0,0,0) relative to this reference.
+ Future _setReference() async {
+ try {
+ stillPressed = false;
+ // Get the current arm position
+ final currentArmPose = await widget.arm.endPosition();
+
+ setState(() {
+ // Zero out the phone position tracking
+ _positionX = 0.0;
+ _positionY = 0.0;
+ _positionZ = 0.0;
+ _velocityX = 0.0;
+ _velocityY = 0.0;
+ _velocityZ = 0.0;
+ _lastIntegrationTime = null;
+
+ // Zero out orientation tracking
+ _orientationX = 0.0;
+ _orientationY = 0.0;
+ _orientationZ = 0.0;
+ _lastGyroIntegrationTime = null;
+
+ // Clear pose queue and reset counter
+ _poseQueue.clear();
+ _poseCounter = 0;
+
+ // Store the current arm position as the reference
+ _referenceArmPose = currentArmPose;
+ _targetArmPose = currentArmPose; // Initialize target pose
+ _currentArmPose = currentArmPose; // Initialize current pose
+ _isReferenceSet = true;
+ _lastError = null;
+ });
+ } catch (e) {
+ setState(() {
+ _lastError = "Failed to set reference: ${e.toString()}";
+ });
+ }
+ }
+
+ @override
+ Widget build(BuildContext context) {
+ return Column(
+ mainAxisAlignment: MainAxisAlignment.center,
+ children: [
+ const Text("Target Position", style: TextStyle(fontWeight: FontWeight.bold, fontSize: 18, color: Colors.blue)),
+ const SizedBox(height: 10),
+ if (_targetArmPose != null) ...[
+ Text("X: ${_targetArmPose!.x.toStringAsFixed(1)} mm", style: const TextStyle(fontSize: 14)),
+ Text("Y: ${_targetArmPose!.y.toStringAsFixed(1)} mm", style: const TextStyle(fontSize: 14)),
+ Text("Z: ${_targetArmPose!.z.toStringAsFixed(1)} mm", style: const TextStyle(fontSize: 14)),
+ Text("oX: ${_targetArmPose!.oX.toStringAsFixed(2)} rad", style: const TextStyle(fontSize: 14)),
+ Text("oY: ${_targetArmPose!.oY.toStringAsFixed(2)} rad", style: const TextStyle(fontSize: 14)),
+ Text("oZ: ${_targetArmPose!.oZ.toStringAsFixed(2)} rad", style: const TextStyle(fontSize: 14)),
+ Text("Theta: ${_targetArmPose!.theta.toStringAsFixed(1)} rad", style: const TextStyle(fontSize: 14)),
+ ] else
+ const Text("No target yet", style: TextStyle(fontSize: 12, color: Colors.grey)),
+ const SizedBox(height: 20),
+ const Text("Current Position", style: TextStyle(fontWeight: FontWeight.bold, fontSize: 18, color: Colors.green)),
+ const SizedBox(height: 10),
+ if (_currentArmPose != null) ...[
+ Text("X: ${_currentArmPose!.x.toStringAsFixed(1)} mm", style: const TextStyle(fontSize: 14)),
+ Text("Y: ${_currentArmPose!.y.toStringAsFixed(1)} mm", style: const TextStyle(fontSize: 14)),
+ Text("Z: ${_currentArmPose!.z.toStringAsFixed(1)} mm", style: const TextStyle(fontSize: 14)),
+ Text("oX: ${_currentArmPose!.oX.toStringAsFixed(2)} rad", style: const TextStyle(fontSize: 14)),
+ Text("oY: ${_currentArmPose!.oY.toStringAsFixed(2)} rad", style: const TextStyle(fontSize: 14)),
+ Text("oZ: ${_currentArmPose!.oZ.toStringAsFixed(2)} rad", style: const TextStyle(fontSize: 14)),
+ Text("Theta: ${_currentArmPose!.theta.toStringAsFixed(1)} rad", style: const TextStyle(fontSize: 14)),
+ ] else
+ const Text("No position data yet", style: TextStyle(fontSize: 12, color: Colors.grey)),
+ const SizedBox(height: 30),
+ ElevatedButton(
+ onPressed: _setReference,
+ style: ElevatedButton.styleFrom(
+ backgroundColor: _isReferenceSet ? Colors.green : Colors.blue,
+ ),
+ child: Text(_isReferenceSet ? "Reset Reference" : "Set Reference Point"),
+ ),
+ const SizedBox(height: 15),
+ Text("Status: ${!_isReferenceSet ? 'Waiting for reference...' : _isMovingArm ? 'Moving...' : 'Ready'}"),
+ if (_lastError != null)
+ Padding(
+ padding: const EdgeInsets.all(8.0),
+ child: Text(
+ "Error: $_lastError",
+ style: const TextStyle(color: Colors.red, fontSize: 10),
+ textAlign: TextAlign.center,
+ ),
+ ),
+ const SizedBox(height: 15),
+ if (!_isReferenceSet)
+ const Text(
+ "Press 'Set Reference Point' to begin",
+ style: TextStyle(fontSize: 12, fontStyle: FontStyle.italic, color: Colors.orange),
+ )
+ else
+ const Text(
+ "Move your phone through space!",
+ style: TextStyle(fontSize: 12, fontStyle: FontStyle.italic),
+ ),
+ TextButton(
+ onPressed: () async {
+ stillPressed = true;
+ },
+ child: Text("Still"),
+ ),
+ TextButton(
+ onPressed: () async {
+ await widget.arm.stop();
+ setState(() {
+ _isReferenceSet = false;
+ _poseQueue.clear();
+ _poseCounter = 0;
+ });
+ },
+ child: Text("Stop"),
+ ),
+ ElevatedButton(
+ onPressed: () async {
+ await widget.arm.moveToJointPositions([0.0, 0.0, 0.0, 0.0, 0.0, 0.0]);
+ await widget.arm.moveToPosition(Pose(x: 300, y: 0, z: 100, oX: 0, oY: 0, oZ: -1, theta: 0));
+ },
+ child: Text("reset position"),
+ ),
+ ],
+ );
+ }
+
+ /// Converts an OrientationVector to a quaternion.
+ ///
+ /// Translated from Go spatialmath code - OrientationVector.Quaternion()
+ /// OX, OY, OZ represent a point on the unit sphere where the end effector is pointing
+ /// Theta is rotation around that pointing axis
+ // vector_math.Quaternion ovToQuat(double oX, double oY, double oZ, double theta) {
+ // const double defaultAngleEpsilon = 1e-4;
+
+ // // Normalize the orientation vector
+ // final norm = math.sqrt(oX * oX + oY * oY + oZ * oZ);
+ // double normOX = oX;
+ // double normOY = oY;
+ // double normOZ = oZ;
+
+ // if (norm == 0.0) {
+ // // Default orientation: pointing up along Z axis
+ // normOX = 0.0;
+ // normOY = 0.0;
+ // normOZ = -1.0;
+ // } else {
+ // normOX /= norm;
+ // normOY /= norm;
+ // normOZ /= norm;
+ // }
+
+ // // acos(OZ) ranges from 0 (north pole) to pi (south pole)
+ // final lat = math.acos(normOZ.clamp(-1.0, 1.0));
+
+ // // If we're pointing at the Z axis then lon is 0, theta is the OV theta
+ // double lon = 0.0;
+
+ // if (1 - normOZ.abs() > defaultAngleEpsilon) {
+ // // If we are not at a pole, we need the longitude
+ // lon = math.atan2(normOY, normOX);
+ // }
+
+ // // Use ZYZ Euler angles to create quaternion
+ // // This matches: mgl64.AnglesToQuat(lon, lat, theta, mgl64.ZYZ)
+ // return _eulerZYZToQuat(lon, lat, theta);
+ // }
+
+ // /// Convert ZYZ Euler angles to quaternion
+ // /// Manually composing Q1(Z by z1) * Q2(Y by y) * Q3(Z by z2)
+ // vector_math.Quaternion _eulerZYZToQuat(double z1, double y, double z2) {
+ // // Create three rotation quaternions and compose them
+ // // Q1: Rotation around Z axis by z1
+ // final q1 = vector_math.Quaternion.axisAngle(vector_math.Vector3(0, 0, 1), z1);
+
+ // // Q2: Rotation around Y axis by y
+ // final q2 = vector_math.Quaternion.axisAngle(vector_math.Vector3(0, 1, 0), y);
+
+ // // Q3: Rotation around Z axis by z2
+ // final q3 = vector_math.Quaternion.axisAngle(vector_math.Vector3(0, 0, 1), z2);
+
+ // // Compose: Q = Q1 * Q2 * Q3 (intrinsic rotations)
+ // final result = q1 * q2 * q3;
+
+ // return result;
+ // }
+
+ // /// Converts a unit quaternion (q) to an OrientationVector.
+ // /// Converted from go code to flutter using Gemini
+ // Orientation_OrientationVectorRadians quatToOV(vector_math.Quaternion q) {
+ // double orientationVectorPoleRadius = 0.0001;
+ // double defaultAngleEpsilon = 1e-4;
+ // // Define initial axes as pure quaternions (Real/W=0)
+ // // xAxis: (0, -1, 0, 0) -> x=-1, y=0, z=0, w=0
+ // final vector_math.Quaternion xAxis = vector_math.Quaternion(0.0, -1.0, 0.0, 0.0);
+ // // zAxis: (0, 0, 0, 1) -> x=0, y=0, z=1, w=0
+ // final vector_math.Quaternion zAxis = vector_math.Quaternion(0.0, 0.0, 0.0, 1.0);
+
+ // final ov = Orientation_OrientationVectorRadians();
+
+ // // Get the transform of our +X and +Z points (Quaternion rotation formula: q * v * q_conj)
+ // final vector_math.Quaternion newX = q * xAxis * q.conjugated();
+ // final vector_math.Quaternion newZ = q * zAxis * q.conjugated();
+
+ // // Set the direction vector (OX, OY, OZ) from the rotated Z-axis (Imag, Jmag, Kmag components)
+ // ov.x = newZ.x;
+ // ov.y = newZ.y;
+ // ov.z = newZ.z;
+
+ // // Calculate the roll angle (Theta)
+
+ // // Check if we are near the poles (i.e., newZ.z/Kmag is close to 1 or -1)
+ // if (1 - (ov.z.abs()) > orientationVectorPoleRadius) {
+ // // General Case: Not Near the Pole
+
+ // // Vector3 versions of the rotated axes
+ // final vector_math.Vector3 v1 = vector_math.Vector3(newZ.x, newZ.y, newZ.z); // Local Z
+ // final vector_math.Vector3 v2 = vector_math.Vector3(newX.x, newX.y, newX.z); // Local X
+ // final vector_math.Vector3 globalZ = vector_math.Vector3(0.0, 0.0, 1.0); // Global Z
+
+ // // Normal to the local-x, local-z plane
+ // final vector_math.Vector3 norm1 = v1.cross(v2);
+
+ // // Normal to the global-z, local-z plane
+ // final vector_math.Vector3 norm2 = v1.cross(globalZ);
+
+ // // Find the angle (theta) between the two planes (using the angle between their normals)
+ // final double denominator = norm1.length * norm2.length;
+ // final double cosTheta = norm1.dot(norm2) / denominator; // Avoid division by zero, default to 1 (0 angle)
+
+ // // Clamp for float error
+ // double clampedCosTheta = cosTheta.clamp(-1.0, 1.0);
+
+ // final double theta = math.acos(clampedCosTheta);
+
+ // if (theta.abs() > orientationVectorPoleRadius) {
+ // // Determine directionality of the angle (sign of theta)
+
+ // // Axis is the new Z-axis (ov.OX, ov.OY, ov.OZ)
+ // final vector_math.Vector3 axis = vector_math.Vector3(ov.x, ov.y, ov.z);
+ // // Create a rotation quaternion for rotation by -theta around the new Z-axis
+ // final vector_math.Quaternion q2 = vector_math.Quaternion.axisAngle(axis, -theta);
+
+ // // Apply q2 rotation to the original Z-axis (0, 0, 0, 1)
+ // final vector_math.Quaternion testZQuat = q2 * zAxis * q2.conjugated();
+ // final vector_math.Vector3 testZVector = vector_math.Vector3(testZQuat.x, testZQuat.y, testZQuat.z);
+
+ // // Find the normal of the plane defined by v1 (local Z) and testZ
+ // final vector_math.Vector3 norm3 = v1.cross(testZVector);
+
+ // final double norm1Len = norm1.length;
+ // final double norm3Len = norm3.length;
+
+ // final double cosTest = norm1.dot(norm3) / (norm1Len * norm3Len);
+
+ // // Check if norm1 and norm3 are coplanar (angle close to 0)
+ // if (1.0 - cosTest < defaultAngleEpsilon * defaultAngleEpsilon) {
+ // ov.theta = -theta;
+ // } else {
+ // ov.theta = theta;
+ // }
+ // } else {
+ // ov.theta = 0.0;
+ // }
+ // } else {
+ // // Special Case: Near the Pole (Z-axis is up or down)
+
+ // // Use Atan2 on the rotated X-axis components (Jmag and Imag, or y and x in Dart)
+ // // -math.Atan2(newX.Jmag, -newX.Imag) -> Dart: -math.atan2(newX.y, -newX.x)
+ // ov.theta = -math.atan2(newX.y, -newX.x);
+
+ // if (newZ.z < 0) {
+ // // If pointing along the negative Z-axis (ov.OZ < 0)
+ // // -math.Atan2(newX.Jmag, newX.Imag) -> Dart: -math.atan2(newX.y, newX.x)
+ // ov.theta = -math.atan2(newX.y, newX.x);
+ // }
+ // }
+
+ // // Handle IEEE -0.0 for consistency
+ // if (ov.theta == -0.0) {
+ // ov.theta = 0.0;
+ // }
+ // return ov;
+ // }
+
+ @override
+ void dispose() {
+ super.dispose();
+ for (final subscription in _streamSubscriptions) {
+ subscription.cancel();
+ }
+ _poseQueue.clear();
+ }
+}
diff --git a/example/viam_example_app/lib/robot_screen.dart b/example/viam_example_app/lib/robot_screen.dart
index f533a565186..0f0230070e4 100644
--- a/example/viam_example_app/lib/robot_screen.dart
+++ b/example/viam_example_app/lib/robot_screen.dart
@@ -5,6 +5,8 @@
/// and send commands to them.
import 'package:flutter/material.dart';
+import 'package:flutter_dotenv/flutter_dotenv.dart';
+import 'package:viam_example_app/resources/arm_screen.dart';
import 'package:viam_sdk/protos/app/app.dart';
import 'package:viam_sdk/viam_sdk.dart';
@@ -34,7 +36,7 @@ class _RobotScreenState extends State {
///
/// This is initialized late because it requires an asynchronous
/// network call to establish the connection.
- late RobotClient client;
+ RobotClient? client;
@override
void initState() {
@@ -48,7 +50,7 @@ class _RobotScreenState extends State {
// You should always close the [RobotClient] to free up resources.
// Calling [RobotClient.close] will clean up any tasks and
// resources created by Viam.
- client.close();
+ client?.close();
super.dispose();
}
@@ -58,7 +60,9 @@ class _RobotScreenState extends State {
// Using the authenticated [Viam] the received as a parameter,
// we can obtain a connection to the Robot.
// There is a helpful convenience method on the [Viam] instance for this.
- final robotClient = await widget._viam.getRobotClient(widget.robot);
+ final options = RobotClientOptions.withApiKey(dotenv.env['API_KEY_ID']!, dotenv.env['API_KEY']!);
+ options.dialOptions.attemptMdns = false;
+ final robotClient = await RobotClient.atAddress(dotenv.env['ROBOT_LOCATION']!, options);
setState(() {
client = robotClient;
_isLoading = false;
@@ -68,7 +72,8 @@ class _RobotScreenState extends State {
/// A computed variable that returns the available [ResourceName]s of
/// this robot in an alphabetically sorted list.
List get _sortedResourceNames {
- return client.resourceNames..sort((a, b) => a.name.compareTo(b.name));
+ return client?.resourceNames ?? []
+ ..sort((a, b) => a.name.compareTo(b.name));
}
/// For this example, we have control screens for only these specific resource subtypes:
@@ -81,6 +86,7 @@ class _RobotScreenState extends State {
final availableResourceSubtypes = [
Camera.subtype.resourceSubtype,
Motor.subtype.resourceSubtype,
+ Arm.subtype.resourceSubtype,
];
return availableResourceSubtypes.contains(rn.subtype);
}
@@ -104,39 +110,66 @@ class _RobotScreenState extends State {
// [RobotClient.getResource(ResourceName)]
// to get a resource directly from a [RobotClient].
// e.g. client.getResource(rn)
- final camera = Camera.fromRobot(client, rn.name);
+ final camera = Camera.fromRobot(client!, rn.name);
// A [StreamClient] is a WebRTC stream that allows you to view
// a live stream from the camera. This requires that the connection
// to the smart machine be through WebRTC (the default option).
// If the connection is not using WebRTC, then this will error.
- final stream = client.getStream(rn.name);
+ final stream = client!.getStream(rn.name);
Navigator.of(context).push(MaterialPageRoute(builder: (_) => CameraScreen(camera, stream)));
} else if (rn.subtype == Motor.subtype.resourceSubtype) {
// Similar to camera above, get the motor from the robot client.
- final motor = Motor.fromRobot(client, rn.name);
+ final motor = Motor.fromRobot(client!, rn.name);
Navigator.of(context).push(MaterialPageRoute(builder: (_) => MotorScreen(motor)));
+ } else if (rn.subtype == Arm.subtype.resourceSubtype) {
+ final arm = Arm.fromRobot(client!, rn.name);
+ Navigator.of(context).push(MaterialPageRoute(builder: (_) => ViamArmWidgetNew(arm: arm)));
}
}
+ Widget getResourceWidget(ResourceName rName) {
+ if (rName.subtype == Arm.subtype.resourceSubtype) {
+ return Padding(padding: EdgeInsets.all(4), child: ViamArmWidgetNew(arm: Arm.fromRobot(client!, rName.name)));
+ }
+ return const Text(
+ 'No screen selected!',
+ );
+ }
+
@override
Widget build(BuildContext context) {
return Scaffold(
- appBar: AppBar(title: Text(widget.robot.name)),
- body: _isLoading
- ? const Center(child: CircularProgressIndicator.adaptive())
- : ListView.builder(
- itemCount: client.resourceNames.length,
- itemBuilder: (_, index) {
- final resourceName = _sortedResourceNames[index];
- return ListTile(
- title: Text(resourceName.name),
- subtitle: Text('${resourceName.namespace}:${resourceName.type}:${resourceName.subtype}'),
- // We only want to navigate to a resource if that resource is one that we implemented
- onTap: _isNavigable(resourceName) ? () => _navigateToResource(resourceName) : null,
- // Similarly, we only want to show the navigation icon if the resource is implemented
- trailing: _isNavigable(resourceName) ? const Icon(Icons.chevron_right) : null,
- );
- }));
+ appBar: AppBar(
+ title: Text(widget.robot.name),
+ ),
+ body: _isLoading
+ ? const Center(child: CircularProgressIndicator())
+ : SingleChildScrollView(
+ child: Column(
+ children: [
+ for (int i = 0; i < _sortedResourceNames.length; i++)
+ _isNavigable(_sortedResourceNames[i])
+ ? ExpansionTile(
+ title: Text(_sortedResourceNames[i].name),
+ subtitle: Text(
+ '${_sortedResourceNames[i].namespace}:${_sortedResourceNames[i].type}:${_sortedResourceNames[i].subtype}'),
+ children: [
+ Container(
+ color: Theme.of(context).colorScheme.surface,
+ child: getResourceWidget(_sortedResourceNames[i]),
+ )
+ ],
+ )
+ : ListTile(
+ title: Text(_sortedResourceNames[i].name),
+ subtitle: Text(
+ '${_sortedResourceNames[i].namespace}:${_sortedResourceNames[i].type}:${_sortedResourceNames[i].subtype}'),
+ enabled: false,
+ )
+ ],
+ ),
+ ),
+ );
}
}
diff --git a/example/viam_example_app/lib/spatialmath/common.dart b/example/viam_example_app/lib/spatialmath/common.dart
new file mode 100644
index 00000000000..9fa366edccb
--- /dev/null
+++ b/example/viam_example_app/lib/spatialmath/common.dart
@@ -0,0 +1,177 @@
+part of 'spatial_math.dart';
+
+// --- Constants from Go Files ---
+
+/// How close OZ must be to +/-1 in order to use pole math for computing theta.
+const double orientationVectorPoleRadius = 0.0001;
+
+/// A small epsilon value for float comparisons, assumed from context.
+const double defaultAngleEpsilon = 1e-9;
+
+// --- Utility Helpers (Stubs for go.viam.com/rdk/utils) ---
+
+/// Stub for utils.RadToDeg
+double radToDeg(double rad) {
+ return rad * (180.0 / math.pi);
+}
+
+/// Stub for utils.DegToRad
+double degToRad(double deg) {
+ return deg * (math.pi / 180.0);
+}
+
+/// Stub for utils.Float64AlmostEqual
+bool float64AlmostEqual(double a, double b, double tol) {
+ return (a - b).abs() <= tol;
+}
+
+// --- Stubbed Dependent Classes (from mgl64, r3, etc.) ---
+
+/// Stub for r3.Vector
+class R3Vector {
+ double x, y, z;
+ R3Vector(this.x, this.y, this.z);
+}
+
+/// Stub for mgl64.Vec3
+class Vec3 {
+ double x, y, z;
+ Vec3(this.x, this.y, this.z);
+
+ double dot(Vec3 other) {
+ return x * other.x + y * other.y + z * other.z;
+ }
+
+ Vec3 cross(Vec3 other) {
+ return Vec3(
+ y * other.z - z * other.y,
+ z * other.x - x * other.z,
+ x * other.y - y * other.x,
+ );
+ }
+
+ double len() {
+ return math.sqrt(x * x + y * y + z * z);
+ }
+}
+
+/// Stub for mgl64.Quat
+class MGLQuat {
+ double w;
+ Vec3 v;
+
+ MGLQuat(this.w, this.v);
+
+ MGLQuat normalize() {
+ final l = math.sqrt(w * w + v.x * v.x + v.y * v.y + v.z * v.z);
+ if (l == 0) return MGLQuat(1, Vec3(0, 0, 0));
+ return MGLQuat(w / l, Vec3(v.x / l, v.y / l, v.z / l));
+ }
+
+ MGLQuat scale(double s) {
+ return MGLQuat(w * s, Vec3(v.x * s, v.y * s, v.z * s));
+ }
+
+ double x() => v.x;
+ double y() => v.y;
+ double z() => v.z;
+
+ /// Stub for mgl64.QuatSlerp
+ static MGLQuat slerp(MGLQuat q1, MGLQuat q2, double t) {
+ // A simplified slerp implementation
+ double cosHalfTheta = q1.w * q2.w + q1.v.dot(q2.v);
+
+ if (cosHalfTheta.abs() >= 1.0) {
+ return q1;
+ }
+
+ double halfTheta = math.acos(cosHalfTheta);
+ double sinHalfTheta = math.sqrt(1.0 - cosHalfTheta * cosHalfTheta);
+
+ if (sinHalfTheta.abs() < 0.001) {
+ return MGLQuat(
+ q1.w * 0.5 + q2.w * 0.5,
+ Vec3(
+ q1.v.x * 0.5 + q2.v.x * 0.5,
+ q1.v.y * 0.5 + q2.v.y * 0.5,
+ q1.v.z * 0.5 + q2.v.z * 0.5,
+ ),
+ );
+ }
+
+ double ratioA = math.sin((1 - t) * halfTheta) / sinHalfTheta;
+ double ratioB = math.sin(t * halfTheta) / sinHalfTheta;
+
+ return MGLQuat(
+ (q1.w * ratioA + q2.w * ratioB),
+ Vec3(
+ (q1.v.x * ratioA + q2.v.x * ratioB),
+ (q1.v.y * ratioA + q2.v.y * ratioB),
+ (q1.v.z * ratioA + q2.v.z * ratioB),
+ ),
+ );
+ }
+
+ /// Stub for mgl64.QuatNlerp
+ static MGLQuat nlerp(MGLQuat q1, MGLQuat q2, double t) {
+ // Simplified nlerp
+ final q = MGLQuat(
+ (1 - t) * q1.w + t * q2.w,
+ Vec3(
+ (1 - t) * q1.v.x + t * q2.v.x,
+ (1 - t) * q1.v.y + t * q2.v.y,
+ (1 - t) * q1.v.z + t * q2.v.z,
+ ),
+ );
+ return q.normalize();
+ }
+
+ /// Stub for mgl64.AnglesToQuat(lon, lat, theta, mgl64.ZYZ)
+ static MGLQuat anglesToQuat(double z1, double y, double z2) {
+ // ZYZ Euler to Quaternion
+ final c1 = math.cos(z1 / 2);
+ final s1 = math.sin(z1 / 2);
+ final c2 = math.cos(y / 2);
+ final s2 = math.sin(y / 2);
+ final c3 = math.cos(z2 / 2);
+ final s3 = math.sin(z2 / 2);
+
+ return MGLQuat(
+ c1 * c2 * c3 - s1 * c2 * s3, // w
+ Vec3(
+ c1 * s2 * c3 - s1 * s2 * s3, // x
+ c1 * s2 * s3 + s1 * s2 * c3, // y
+ s1 * c2 * c3 + c1 * c2 * s3, // z
+ ),
+ );
+ }
+}
+
+/// Stub for R4AA (Axis-Angle)
+class R4AA {
+ double theta, rx, ry, rz;
+ R4AA(this.theta, this.rx, this.ry, this.rz);
+
+ /// Stub for R4AA.ToQuat()
+ Quaternion toQuat() {
+ final halfAngle = theta / 2.0;
+ final s = math.sin(halfAngle);
+ return Quaternion(
+ math.cos(halfAngle),
+ rx * s,
+ ry * s,
+ rz * s,
+ );
+ }
+}
+
+/// Stub for RotationMatrix
+class RotationMatrix {
+ List mat; // Expects a list of 9 doubles
+ RotationMatrix(this.mat);
+}
+
+/// Stub for NewZeroOrientation()
+Quaternion newZeroOrientation() {
+ return Quaternion(1, 0, 0, 0); // Identity quaternion
+}
diff --git a/example/viam_example_app/lib/spatialmath/euler_angles.dart b/example/viam_example_app/lib/spatialmath/euler_angles.dart
new file mode 100644
index 00000000000..5f03a10f6a9
--- /dev/null
+++ b/example/viam_example_app/lib/spatialmath/euler_angles.dart
@@ -0,0 +1,31 @@
+part of 'spatial_math.dart';
+
+class EulerAngles {
+ double roll;
+ double pitch;
+ double yaw;
+
+ EulerAngles(this.roll, this.pitch, this.yaw);
+
+ EulerAngles.zero()
+ : roll = 0.0,
+ pitch = 0.0,
+ yaw = 0.0;
+
+ Quaternion toQuaternion() {
+ final cy = math.cos(yaw * 0.5);
+ final sy = math.sin(yaw * 0.5);
+ final cp = math.cos(pitch * 0.5);
+ final sp = math.sin(pitch * 0.5);
+ final cr = math.cos(roll * 0.5);
+ final sr = math.sin(roll * 0.5);
+
+ final q = Quaternion.zero();
+ q.real = cr * cp * cy + sr * sp * sy;
+ q.imag = sr * cp * cy - cr * sp * sy;
+ q.jmag = cr * sp * cy + sr * cp * sy;
+ q.kmag = cr * cp * sy - sr * sp * cy;
+
+ return q;
+ }
+}
diff --git a/example/viam_example_app/lib/spatialmath/orientation_vector.dart b/example/viam_example_app/lib/spatialmath/orientation_vector.dart
new file mode 100644
index 00000000000..fbd37a6b55a
--- /dev/null
+++ b/example/viam_example_app/lib/spatialmath/orientation_vector.dart
@@ -0,0 +1,54 @@
+part of 'spatial_math.dart';
+
+class OrientationVector {
+ double theta;
+ double ox;
+ double oy;
+ double oz;
+
+ OrientationVector(this.theta, this.ox, this.oy, this.oz);
+
+ OrientationVector.zero()
+ : theta = 0.0,
+ ox = 0.0,
+ oy = 0.0,
+ oz = 1.0;
+
+ double _computeNormal() {
+ return math.sqrt(ox * ox + oy * oy + oz * oz);
+ }
+
+ String? isValid() {
+ if (_computeNormal() == 0.0) {
+ return "OrientationVector has a normal of 0, probably X, Y, and Z are all 0";
+ }
+ return null;
+ }
+
+ void normalize() {
+ final norm = _computeNormal();
+ if (norm == 0.0) {
+ oz = 1;
+ return;
+ }
+ ox /= norm;
+ oy /= norm;
+ oz /= norm;
+ }
+
+ Quaternion toQuaternion() {
+ normalize();
+
+ final lat = math.acos(oz);
+
+ double lon = 0.0;
+ final th = theta;
+
+ if (1 - oz.abs() > defaultAngleEpsilon) {
+ lon = math.atan2(oy, ox);
+ }
+
+ final q1 = MGLQuat.anglesToQuat(lon, lat, th); // not confident this is doing a ZYZ rotation correctly
+ return Quaternion(q1.w, q1.x(), q1.y(), q1.z());
+ }
+}
diff --git a/example/viam_example_app/lib/spatialmath/quaternion.dart b/example/viam_example_app/lib/spatialmath/quaternion.dart
new file mode 100644
index 00000000000..2ef8ef4d49c
--- /dev/null
+++ b/example/viam_example_app/lib/spatialmath/quaternion.dart
@@ -0,0 +1,100 @@
+part of 'spatial_math.dart';
+
+class Quaternion {
+ double real; // W
+ double imag; // X
+ double jmag; // Y
+ double kmag; // Z
+
+ Quaternion(this.real, this.imag, this.jmag, this.kmag);
+
+ Quaternion.identity()
+ : real = 1.0,
+ imag = 0.0,
+ jmag = 0.0,
+ kmag = 0.0;
+
+ Quaternion.zero()
+ : real = 0.0,
+ imag = 0.0,
+ jmag = 0.0,
+ kmag = 0.0;
+
+ OrientationVector toOrientationVectorRadians() {
+ return quatToOV(this);
+ }
+
+ Quaternion conj() {
+ return Quaternion(real, -imag, -jmag, -kmag);
+ }
+
+ // add quaternions by using this multiplication function
+ Quaternion mul(Quaternion other) {
+ return Quaternion(
+ real * other.real - imag * other.imag - jmag * other.jmag - kmag * other.kmag,
+ real * other.imag + imag * other.real + jmag * other.kmag - kmag * other.jmag,
+ real * other.jmag - imag * other.kmag + jmag * other.real + kmag * other.imag,
+ real * other.kmag + imag * other.jmag - jmag * other.imag + kmag * other.real,
+ );
+ }
+}
+
+OrientationVector quatToOV(Quaternion q) {
+ final xAxis = Quaternion(0, -1, 0, 0);
+ final zAxis = Quaternion(0, 0, 0, 1);
+ final ov = OrientationVector.zero();
+ // Get the transform of our +X and +Z points
+ final newX = q.mul(xAxis).mul(q.conj());
+ final newZ = q.mul(zAxis).mul(q.conj());
+ ov.ox = newZ.imag;
+ ov.oy = newZ.jmag;
+ ov.oz = newZ.kmag;
+
+ if (1 - newZ.kmag.abs() > orientationVectorPoleRadius) {
+ final v1 = Vec3(newZ.imag, newZ.jmag,
+ newZ.kmag); // might have to do something weird here to get a vector that doesn't distinguish between row and column
+ final v2 = Vec3(newX.imag, newX.jmag, newX.kmag);
+
+ final norm1 = v1.cross(v2);
+ final norm2 = v1.cross(Vec3(zAxis.imag, zAxis.jmag, zAxis.kmag));
+
+ double cosTheta = norm1.dot(norm2) / (norm1.len() * norm2.len());
+ if (cosTheta > 1) cosTheta = 1;
+ if (cosTheta < -1) cosTheta = -1;
+
+ final theta = math.acos(cosTheta);
+ if (theta > orientationVectorPoleRadius) {
+ final aa = R4AA(-theta, ov.ox, ov.oy, ov.oz);
+ final q2 = aa.toQuat();
+ final testZ = q2.mul(zAxis).mul(q2.conj());
+ final norm3 = v1.cross(Vec3(testZ.imag, testZ.jmag, testZ.kmag));
+ final cosTest = norm1.dot(norm3) / (norm1.len() * norm3.len());
+ if (1 - cosTest < defaultAngleEpsilon * defaultAngleEpsilon) {
+ ov.theta = -theta;
+ } else {
+ ov.theta = theta;
+ }
+ } else {
+ ov.theta = 0;
+ }
+ } else {
+ // Special case for when we point directly along the Z axis
+ ov.theta = -math.atan2(newX.jmag, -newX.imag);
+ if (newZ.kmag < 0) {
+ ov.theta = -math.atan2(newX.jmag, newX.imag);
+ }
+ }
+
+ if (ov.theta == -0.0) {
+ ov.theta = 0.0;
+ }
+
+ return ov;
+}
+
+bool quaternionAlmostEqual(Quaternion a, Quaternion b, double tol) {
+ return float64AlmostEqual(a.imag, b.imag, tol) &&
+ float64AlmostEqual(a.jmag, b.jmag, tol) &&
+ float64AlmostEqual(a.kmag, b.kmag, tol) &&
+ float64AlmostEqual(a.real, b.real, tol);
+}
diff --git a/example/viam_example_app/lib/spatialmath/spatial_math.dart b/example/viam_example_app/lib/spatialmath/spatial_math.dart
new file mode 100644
index 00000000000..58c94cc3321
--- /dev/null
+++ b/example/viam_example_app/lib/spatialmath/spatial_math.dart
@@ -0,0 +1,15 @@
+library spatial_math;
+
+import 'dart:math' as math;
+
+// These 'part' files will be combined into this single library.
+// This allows them to access each other's classes and functions
+// without circular import errors.
+part 'common.dart';
+part 'euler_angles.dart';
+part 'orientation_vector.dart';
+part 'quaternion.dart';
+
+// All public classes in the 'part' files are automatically
+// exported as part of the 'spatial_math' library.
+// The previous 'export' lines were incorrect and have been removed.
diff --git a/example/viam_example_app/pubspec.yaml b/example/viam_example_app/pubspec.yaml
index 5f36cb72807..5a72770a9aa 100644
--- a/example/viam_example_app/pubspec.yaml
+++ b/example/viam_example_app/pubspec.yaml
@@ -10,6 +10,9 @@ dependencies:
flutter:
sdk: flutter
flutter_dotenv: ^5.1.0
+ sensors_plus: ^7.0.0
+ arkit_plugin: ^1.2.1
+ vector_math: ^2.2.0
viam_sdk:
path: ../../
diff --git a/example/viam_robot_example_app/pubspec.yaml b/example/viam_robot_example_app/pubspec.yaml
index e7ceff821f5..402228f2f99 100644
--- a/example/viam_robot_example_app/pubspec.yaml
+++ b/example/viam_robot_example_app/pubspec.yaml
@@ -14,6 +14,8 @@ dependencies:
path: ../../
image: ^4.0.17
flutter_dotenv: ^5.1.0
+ sensors_plus: ^7.0.0
+
dev_dependencies:
flutter_test:
diff --git a/pubspec.yaml b/pubspec.yaml
index 6316cebb68c..fee09df03af 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -11,7 +11,7 @@ dependencies:
flutter:
sdk: flutter
flutter_webrtc: ^0.12.1+hotfix.1
- grpc: ^4.0.1
+ grpc: ^4.1.0
protobuf: ^3.0.0
image: ^4.0.16
logger: ^2.0.1