From 16cf6f0eeed2f8df90e205cd6df5e57859b56154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Fern=C3=A1ndez=20Garc=C3=ADa-Boente?= Date: Wed, 14 May 2025 15:22:04 +0200 Subject: [PATCH] Eraly Prototype --- .../com/igalia/wolvic/PlatformActivity.java | 8 + .../com/igalia/wolvic/VRBrowserActivity.java | 98 ++++++++++ .../wolvic/ui/widgets/MousePointerWidget.java | 185 ++++++++++++++++++ .../com/igalia/wolvic/PlatformActivity.java | 9 + .../com/igalia/wolvic/PlatformActivity.java | 7 + app/src/main/AndroidManifest.xml | 3 + .../main/res/drawable/ic_mouse_pointer.xml | 12 ++ .../com/igalia/wolvic/PlatformActivity.java | 12 ++ .../com/igalia/wolvic/PlatformActivity.java | 9 + .../com/igalia/wolvic/PlatformActivity.java | 10 + .../com/igalia/wolvic/PlatformActivity.java | 12 +- .../com/igalia/wolvic/PlatformActivity.java | 8 + .../cpp/DeviceDelegateVisionGlass.cpp | 43 +++- .../cpp/DeviceDelegateVisionGlass.h | 5 + app/src/visionglass/cpp/native-lib.cpp | 14 ++ .../com/igalia/wolvic/PlatformActivity.java | 183 ++++++++++++++++- 16 files changed, 607 insertions(+), 11 deletions(-) create mode 100644 app/src/common/shared/com/igalia/wolvic/ui/widgets/MousePointerWidget.java create mode 100644 app/src/main/res/drawable/ic_mouse_pointer.xml diff --git a/app/src/aosp/java/com/igalia/wolvic/PlatformActivity.java b/app/src/aosp/java/com/igalia/wolvic/PlatformActivity.java index 52ebdc4ddb..eba59f6812 100644 --- a/app/src/aosp/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/aosp/java/com/igalia/wolvic/PlatformActivity.java @@ -8,6 +8,8 @@ import android.content.Intent; import android.view.KeyEvent; +import android.bluetooth.BluetoothDevice; + import com.google.androidgamesdk.GameActivity; import com.igalia.wolvic.ui.widgets.WidgetManagerDelegate; @@ -49,6 +51,12 @@ public boolean onKeyUp(int keyCode, KeyEvent event) { return super.onKeyUp(keyCode, event); } + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + } + protected native void queueRunnable(Runnable aRunnable); protected native boolean platformExit(); } diff --git a/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java b/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java index ad4d46d77b..b094528837 100644 --- a/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java +++ b/app/src/common/shared/com/igalia/wolvic/VRBrowserActivity.java @@ -7,12 +7,16 @@ import static com.igalia.wolvic.ui.widgets.UIWidget.REMOVE_WIDGET; +import android.Manifest; +import android.bluetooth.BluetoothClass; +import android.bluetooth.BluetoothDevice; import android.content.BroadcastReceiver; import android.content.ComponentCallbacks2; import android.content.Context; import android.content.Intent; import android.content.IntentFilter; import android.content.SharedPreferences; +import android.content.pm.PackageManager; import android.content.res.Configuration; import android.graphics.Canvas; import android.graphics.Color; @@ -30,15 +34,21 @@ import android.os.Process; import android.util.Log; import android.util.Pair; +import android.view.InputDevice; import android.view.KeyEvent; +import android.view.MotionEvent; import android.view.Surface; import android.view.View; import android.view.ViewTreeObserver; import android.widget.FrameLayout; +import androidx.activity.result.ActivityResultLauncher; +import androidx.activity.result.contract.ActivityResultContracts; import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; import androidx.fragment.app.FragmentController; import androidx.lifecycle.Lifecycle; import androidx.lifecycle.LifecycleOwner; @@ -76,6 +86,7 @@ import com.igalia.wolvic.ui.widgets.AppServicesProvider; import com.igalia.wolvic.ui.widgets.HorizontalTabsBar; import com.igalia.wolvic.ui.widgets.KeyboardWidget; +import com.igalia.wolvic.ui.widgets.MousePointerWidget; import com.igalia.wolvic.ui.widgets.NavigationBarWidget; import com.igalia.wolvic.ui.widgets.OverlayContentWidget; import com.igalia.wolvic.ui.widgets.RootWidget; @@ -159,6 +170,48 @@ public void onReceive(Context context, Intent intent) { } }; + private BroadcastReceiver mBluetoothReceiver = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + Log.d(LOGTAG, "Bluetooth receiver triggered: " + intent.getAction()); + BluetoothDevice device = intent.getParcelableExtra(BluetoothDevice.EXTRA_DEVICE); + if (device == null) { + Log.e(LOGTAG, "Bluetooth device is null"); + return; + } + if (!isHidDevice(device)) { + Log.d(LOGTAG, "Bluetooth device is not a HID device: " + device.getName()); + return; + } + if (BluetoothDevice.ACTION_ACL_DISCONNECTED.equals(intent.getAction())) { + handleBluetoothHidDeviceDisconnected(device); + } else if (BluetoothDevice.ACTION_ACL_CONNECTED.equals(intent.getAction())) { + handleBluetoothHidDeviceConnected(device); + } else { + Log.d(LOGTAG, "Bluetooth device action not recognized: " + intent.getAction()); + } + } + }; + + // Helper method to check if a device is likely a HID device + private boolean isHidDevice(BluetoothDevice device) { + Log.d(LOGTAG, "Checking if device is HID: " + device.getName()); + + // This is a basic check. We might need more sophisticated logic + // based on the device's BluetoothClass or supported UUIDs. + if (device.getBluetoothClass() != null) { + BluetoothClass bluetoothClass = device.getBluetoothClass(); + Log.d(LOGTAG, "Bluetooth class: " + bluetoothClass.toString()); + int deviceClass = bluetoothClass.getDeviceClass(); + Log.d(LOGTAG, "Bluetooth device class: " + deviceClass); + return bluetoothClass.doesClassMatch(BluetoothClass.PROFILE_HID) + && deviceClass == BluetoothClass.Device.PERIPHERAL_POINTING; + } else { + Log.d(LOGTAG, "Bluetooth device class is null"); + } + return false; + } + private LifecycleRegistry mLifeCycle; @NonNull @@ -289,6 +342,47 @@ protected void attachBaseContext(Context base) { super.attachBaseContext(newContext); } + private void registerBluetoothReceiver() { + Log.d(LOGTAG, "Registering Bluetooth receiver"); + IntentFilter intentFilter = new IntentFilter(); + intentFilter.addAction(BluetoothDevice.ACTION_ACL_CONNECTED); + intentFilter.addAction(BluetoothDevice.ACTION_ACL_DISCONNECTED); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + registerReceiver(mBluetoothReceiver, intentFilter, null, null, Context.RECEIVER_NOT_EXPORTED); + } else { + registerReceiver(mBluetoothReceiver, intentFilter, null, null); + } + } + + private void checkAndRequestBluetoothConnectPermission() { + Log.d(LOGTAG, "Checking for BLUETOOTH_CONNECT permission"); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { // Check for Android 12+ + if (ContextCompat.checkSelfPermission(this, Manifest.permission.BLUETOOTH_CONNECT) == PackageManager.PERMISSION_GRANTED) { + // Permission is already granted + Log.d(LOGTAG, "BLUETOOTH_CONNECT permission already granted."); + registerBluetoothReceiver(); + } else { + // Permission is not granted, request it + Log.d(LOGTAG, "Requesting BLUETOOTH_CONNECT permission."); + requestPermission(null, Manifest.permission.BLUETOOTH_CONNECT, OriginatorType.APPLICATION, new WSession.PermissionDelegate.Callback() { + @Override + public void grant() { + registerBluetoothReceiver(); + } + + @Override + public void reject() { + Log.w(LOGTAG, "BLUETOOTH_CONNECT permission denied."); + } + }); + } + } else { + // On older Android versions, BLUETOOTH and BLUETOOTH_ADMIN are sufficient + Log.d(LOGTAG, "Running on Android version < 12. No runtime BLUETOOTH_CONNECT permission needed."); + registerBluetoothReceiver(); // Register the receiver directly + } + } + @Override protected void onCreate(Bundle savedInstanceState) { mFragmentController = FragmentController.createController(new FragmentControllerCallbacks(this, new Handler(Looper.getMainLooper()), 0)); @@ -405,6 +499,8 @@ protected void onCreate(Bundle savedInstanceState) { else setPointerMode(mSettings.getPointerMode()); + checkAndRequestBluetoothConnectPermission(); + // Show the launch dialogs, if needed. if (!showTermsServiceDialogIfNeeded()) { if (!showPrivacyDialogIfNeeded()) { @@ -500,6 +596,8 @@ public void onIsWindowFullscreenChanged(boolean isFullscreen) { addWidgets(Arrays.asList(mRootWidget, mNavigationBar, mKeyboard, mTray, mTabsBar, mWebXRInterstitial)); + + // Create the platform plugin after widgets are created to be extra safe. mPlatformPlugin = createPlatformPlugin(this); if (mPlatformPlugin != null) diff --git a/app/src/common/shared/com/igalia/wolvic/ui/widgets/MousePointerWidget.java b/app/src/common/shared/com/igalia/wolvic/ui/widgets/MousePointerWidget.java new file mode 100644 index 0000000000..109eb478fc --- /dev/null +++ b/app/src/common/shared/com/igalia/wolvic/ui/widgets/MousePointerWidget.java @@ -0,0 +1,185 @@ +package com.igalia.wolvic.ui.widgets; + +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.Path; +import android.util.AttributeSet; +import android.util.Log; +import android.view.LayoutInflater; + +import androidx.annotation.NonNull; +import androidx.databinding.DataBindingUtil; +import androidx.databinding.ObservableBoolean; +import androidx.lifecycle.Observer; +import androidx.lifecycle.ViewModelProvider; + +import com.igalia.wolvic.R; +import com.igalia.wolvic.VRBrowserActivity; +import com.igalia.wolvic.browser.engine.SessionStore; +import com.igalia.wolvic.ui.viewmodel.WindowViewModel; +import com.igalia.wolvic.utils.SystemUtils; + +public class MousePointerWidget extends UIWidget { + + private static final String LOGTAG = SystemUtils.createLogtag(MousePointerWidget.class); + + private WindowWidget mAttachedWindow; + private boolean mIsWindowAttached; + + private Paint mPaint; + private Path mPointerPath; + + private float mCurrentX = 0; + private float mCurrentY = 0; + + public MousePointerWidget(Context aContext) { + super(aContext); + initialize(aContext); + } + + public MousePointerWidget(Context aContext, AttributeSet aAttrs) { + super(aContext, aAttrs); + initialize(aContext); + } + + public MousePointerWidget(Context aContext, AttributeSet aAttrs, int aDefStyle) { + super(aContext, aAttrs, aDefStyle); + initialize(aContext); + } + + private void initialize(@NonNull Context aContext) { + updateUI(); + } + + private void updateUI() { + removeAllViews(); + + // Initialize paint for drawing the pointer + mPaint = new Paint(); + mPaint.setColor(Color.BLACK); // Or any color you prefer + mPaint.setStyle(Paint.Style.FILL); + mPaint.setAntiAlias(true); + + // Initialize path for the pointer shape (e.g., a triangle) + mPointerPath = new Path(); + // Define the shape of your mouse pointer here + // Example: a simple triangle pointing right-down + mPointerPath.moveTo(0, 0); + mPointerPath.lineTo(20, 10); + mPointerPath.lineTo(10, 20); + mPointerPath.close(); + + // Set initial visibility (you might want to hide it initially) + mWidgetPlacement.visible = false; + } + + @Override + protected void initializeWidgetPlacement(WidgetPlacement aPlacement) { + aPlacement.width = 8; + aPlacement.height = 8; + aPlacement.cylinder = false; + } + + @Override + protected void onDraw(Canvas canvas) { + Log.d(LOGTAG, "onDraw: " + mCurrentX + ", " + mCurrentY); + super.onDraw(canvas); + + // Draw the mouse pointer shape on the canvas + canvas.drawPath(mPointerPath, mPaint); + } + + // Method to update the pointer's position based on mouse events + // Method to update the pointer's position based on mouse events + public void updatePointerPosition(float x, float y) { + Log.d(LOGTAG, "updatePointerPosition: " + x + ", " + y); + mCurrentX = x; + mCurrentY = y; + + // Update the widget's placement directly + // You'll need to translate screen coordinates (from MotionEvent) + // to your VR scene coordinates. This will depend on your VR rendering setup. + // For a simple overlay, you might just set the translation directly. + + // Example (conceptual): + mWidgetPlacement.translationX = mCurrentX; + mWidgetPlacement.translationY = mCurrentY; + // mWidgetPlacement.translationZ = ... // Adjust Z if needed + + // Notify the WidgetManager that the placement has changed + if (mWidgetManager != null) { + mWidgetManager.updateWidget(this); + } + } + + // You might need methods to show and hide the pointer + public void showPointer() { + Log.d(LOGTAG, "showPointer"); + mWidgetPlacement.visible = true; + mWidgetManager.updateWidget(this); + } + + public void hidePointer() { + Log.d(LOGTAG, "hidePointer"); + mWidgetPlacement.visible = false; + mWidgetManager.updateWidget(this); + } + + @Override + public void show(@ShowFlags int aShowFlags) { + Log.d(LOGTAG, "show: " + aShowFlags); + if (!mWidgetPlacement.visible) { + mWidgetPlacement.visible = true; + mWidgetManager.addWidget(this); + } + } + + @Override + public void hide(@HideFlags int aHideFlags) { + Log.d(LOGTAG, "hide: " + aHideFlags); + if (mWidgetPlacement.visible) { + mWidgetPlacement.visible = false; + if (aHideFlags == REMOVE_WIDGET) { + mWidgetManager.removeWidget(this); + } else { + mWidgetManager.updateWidget(this); + } + } + } + + @Override + public void detachFromWindow() { + Log.d(LOGTAG, "detachFromWindow"); + mWidgetPlacement.parentHandle = -1; + + mIsWindowAttached = false; + } + + @Override + public void attachToWindow(@NonNull WindowWidget aWindow) { + Log.d(LOGTAG, "attachToWindow: " + aWindow); + if (mAttachedWindow == aWindow) { + return; + } + detachFromWindow(); + + mAttachedWindow = aWindow; + mWidgetPlacement.parentHandle = aWindow.getHandle(); + + + mIsWindowAttached = true; + } + + Observer mIsVisibleObserver = aVisible -> { + if (aVisible.get()) { + this.show(REQUEST_FOCUS); + + } else { + this.hide(UIWidget.KEEP_WIDGET); + } + + mWidgetManager.updateWidget(MousePointerWidget.this); + }; +} diff --git a/app/src/hvr/java/com/igalia/wolvic/PlatformActivity.java b/app/src/hvr/java/com/igalia/wolvic/PlatformActivity.java index 57109a6968..053e53ebc9 100644 --- a/app/src/hvr/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/hvr/java/com/igalia/wolvic/PlatformActivity.java @@ -5,6 +5,7 @@ package com.igalia.wolvic; +import android.bluetooth.BluetoothDevice; import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; @@ -382,6 +383,14 @@ public void surfaceDestroyed(SurfaceHolder holder) queueRunnable(this::nativeOnSurfaceDestroyed); } + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + Log.d(TAG, "Bluetooth HID device connected: " + device.getName()); + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + Log.d(TAG, "Bluetooth HID device disconnected: " + device.getName()); + } + public final PlatformActivityPlugin createPlatformPlugin(WidgetManagerDelegate delegate) { return null; } protected boolean platformExit() { diff --git a/app/src/lynx/java/com/igalia/wolvic/PlatformActivity.java b/app/src/lynx/java/com/igalia/wolvic/PlatformActivity.java index 15098a602e..c18ba4f8f4 100644 --- a/app/src/lynx/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/lynx/java/com/igalia/wolvic/PlatformActivity.java @@ -6,6 +6,7 @@ package com.igalia.wolvic; import android.app.NativeActivity; +import android.bluetooth.BluetoothDevice; import android.content.Intent; import android.view.KeyEvent; import android.view.View; @@ -50,6 +51,12 @@ public void run() { public final PlatformActivityPlugin createPlatformPlugin(WidgetManagerDelegate delegate) { return null; } + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + } + protected native void queueRunnable(Runnable aRunnable); protected native boolean platformExit(); } diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 328dd3157e..1e6b9367f9 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -13,6 +13,9 @@ + + + diff --git a/app/src/main/res/drawable/ic_mouse_pointer.xml b/app/src/main/res/drawable/ic_mouse_pointer.xml new file mode 100644 index 0000000000..cb7d53b3c3 --- /dev/null +++ b/app/src/main/res/drawable/ic_mouse_pointer.xml @@ -0,0 +1,12 @@ + + + + + + + + + \ No newline at end of file diff --git a/app/src/noapi/java/com/igalia/wolvic/PlatformActivity.java b/app/src/noapi/java/com/igalia/wolvic/PlatformActivity.java index 2b232c3493..cafb49c802 100644 --- a/app/src/noapi/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/noapi/java/com/igalia/wolvic/PlatformActivity.java @@ -6,6 +6,8 @@ package com.igalia.wolvic; import androidx.activity.ComponentActivity; + +import android.bluetooth.BluetoothDevice; import android.content.Intent; import android.opengl.GLSurfaceView; import android.os.Build; @@ -206,6 +208,16 @@ protected void onDestroy() { } } + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + Log.d(LOGTAG, "Bluetooth HID device connected: " + device.getName()); + + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + Log.d(LOGTAG, "Bluetooth HID device disconnected: " + device.getName()); + } + + void setImmersiveSticky() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { getWindow().setDecorFitsSystemWindows(false); diff --git a/app/src/oculusvr/java/com/igalia/wolvic/PlatformActivity.java b/app/src/oculusvr/java/com/igalia/wolvic/PlatformActivity.java index 76eeab74f3..1daaad6feb 100644 --- a/app/src/oculusvr/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/oculusvr/java/com/igalia/wolvic/PlatformActivity.java @@ -7,6 +7,7 @@ import android.Manifest; import android.app.NativeActivity; +import android.bluetooth.BluetoothDevice; import android.content.Intent; import android.net.Uri; import android.os.Build; @@ -127,6 +128,14 @@ public WindowInsetsCompat onApplyWindowInsets( public final PlatformActivityPlugin createPlatformPlugin(WidgetManagerDelegate delegate) { return null; } + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + Log.d(LOGTAG, "Bluetooth HID device connected: " + device.getName()); + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + Log.d(LOGTAG, "Bluetooth HID device disconnected: " + device.getName()); + } + protected native void queueRunnable(Runnable aRunnable); protected native boolean platformExit(); } diff --git a/app/src/pfdmxr/java/com/igalia/wolvic/PlatformActivity.java b/app/src/pfdmxr/java/com/igalia/wolvic/PlatformActivity.java index 5afccd35d9..3d51c1f4af 100644 --- a/app/src/pfdmxr/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/pfdmxr/java/com/igalia/wolvic/PlatformActivity.java @@ -7,6 +7,7 @@ import android.Manifest; import android.app.NativeActivity; +import android.bluetooth.BluetoothDevice; import android.content.Intent; import android.net.Uri; import android.os.Build; @@ -124,6 +125,15 @@ public WindowInsetsCompat onApplyWindowInsets( public final PlatformActivityPlugin createPlatformPlugin(WidgetManagerDelegate delegate) { return null; } + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + Log.d(LOGTAG, "Bluetooth HID device connected: " + device.getName()); + + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + Log.d(LOGTAG, "Bluetooth HID device disconnected: " + device.getName()); + } + protected native void queueRunnable(Runnable aRunnable); protected native boolean platformExit(); } diff --git a/app/src/picoxr/java/com/igalia/wolvic/PlatformActivity.java b/app/src/picoxr/java/com/igalia/wolvic/PlatformActivity.java index 1b8053cca3..7dd3af2cd1 100644 --- a/app/src/picoxr/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/picoxr/java/com/igalia/wolvic/PlatformActivity.java @@ -6,7 +6,9 @@ package com.igalia.wolvic; import android.app.NativeActivity; +import android.bluetooth.BluetoothDevice; import android.content.Intent; +import android.util.Log; import android.view.KeyEvent; import com.igalia.wolvic.ui.widgets.WidgetManagerDelegate; @@ -45,7 +47,15 @@ public void run() { platformExit(); } }); - } + } + + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + } + protected native void queueRunnable(Runnable aRunnable); protected native boolean platformExit(); } diff --git a/app/src/spaces/java/com/igalia/wolvic/PlatformActivity.java b/app/src/spaces/java/com/igalia/wolvic/PlatformActivity.java index c84e650674..8662413019 100644 --- a/app/src/spaces/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/spaces/java/com/igalia/wolvic/PlatformActivity.java @@ -6,6 +6,7 @@ package com.igalia.wolvic; import android.app.NativeActivity; +import android.bluetooth.BluetoothDevice; import android.content.Intent; import android.view.KeyEvent; import android.view.View; @@ -49,6 +50,13 @@ public void run() { } }); } + + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + } + protected native void queueRunnable(Runnable aRunnable); protected native boolean platformExit(); } diff --git a/app/src/visionglass/cpp/DeviceDelegateVisionGlass.cpp b/app/src/visionglass/cpp/DeviceDelegateVisionGlass.cpp index 4cf6c95a06..45feef928d 100644 --- a/app/src/visionglass/cpp/DeviceDelegateVisionGlass.cpp +++ b/app/src/visionglass/cpp/DeviceDelegateVisionGlass.cpp @@ -39,6 +39,7 @@ namespace crow { static const float kDiagonalFOV = 41.0f; const vrb::Vector kAverageHeight(0.0f, 1.6f, 0.0f); static const int32_t kControllerIndex = 0; +static const int32_t kControllerMouseIndex = 1; struct DeviceDelegateVisionGlass::State { vrb::RenderContextWeak context; @@ -123,6 +124,7 @@ DeviceDelegateVisionGlass::Create(vrb::RenderContextPtr& aContext) { DeviceDelegateVRGlassPtr result = std::make_shared > (); result->m.context = aContext; result->m.Initialize(); + result->activeControllerIndex = kControllerIndex; return result; } @@ -221,6 +223,21 @@ DeviceDelegateVisionGlass::SetControllerDelegate(ControllerDelegatePtr& aControl m.controller->SetButtonCount(kControllerIndex, 1); m.controller->SetButtonState(kControllerIndex, ControllerDelegate::BUTTON_TRIGGER, device::kImmersiveButtonTrigger, false, false); + + // Create the External Mouse controller + m.controller->CreateController(kControllerMouseIndex, kControllerMouseIndex, "Vision Glass Controller"); + m.controller->SetEnabled(kControllerMouseIndex, false); + m.controller->SetTargetRayMode(kControllerMouseIndex, device::TargetRayMode::TrackedPointer); + m.controller->SetControllerType(kControllerMouseIndex, device::VisionGlass); + m.controller->SetMode(kControllerMouseIndex, ControllerMode::Device); + m.controller->SetAimEnabled(kControllerMouseIndex, true); + m.controller->SetCapabilityFlags(kControllerMouseIndex, + device::Orientation | device::PositionEmulated); + + m.controller->SetButtonCount(kControllerMouseIndex, 1); + m.controller->SetButtonState(kControllerMouseIndex, ControllerDelegate::BUTTON_TRIGGER, + device::kImmersiveButtonTrigger, false, false); + } void @@ -247,7 +264,7 @@ DeviceDelegateVisionGlass::SetHitDistance(const float distance) { // Scale the beam so it reaches all the way to the pointer. auto beamTransform = vrb::Matrix::Identity(); beamTransform.ScaleInPlace(vrb::Vector(1.0, 1.0, distance)); - m.controller->SetBeamTransform(kControllerIndex, beamTransform); + m.controller->SetBeamTransform(activeControllerIndex, beamTransform); } void @@ -284,7 +301,7 @@ DeviceDelegateVisionGlass::StartFrame(const FramePrediction aPrediction) { auto calibratedControllerOrientation = m.headToControllerRelativeRotation * vrb::Quaternion(filteredOrientation); vrb::Matrix transformMatrix = vrb::Matrix::Rotation(calibratedControllerOrientation); auto pointerTransform = m.elbow->GetTransform(ElbowModel::HandEnum::None, headTransform, transformMatrix); - m.controller->SetTransform(kControllerIndex, pointerTransform); + m.controller->SetTransform(activeControllerIndex, pointerTransform); } void @@ -363,15 +380,31 @@ DeviceDelegateVisionGlass::ControllerButtonPressed(const bool aDown) { return; } - m.controller->SetButtonState(kControllerIndex, ControllerDelegate::BUTTON_TRIGGER, device::kImmersiveButtonTrigger, aDown, aDown); + m.controller->SetButtonState(activeControllerIndex, ControllerDelegate::BUTTON_TRIGGER, device::kImmersiveButtonTrigger, aDown, aDown); if (aDown && m.renderMode == device::RenderMode::Immersive) { - m.controller->SetSelectActionStart(kControllerIndex); + m.controller->SetSelectActionStart(activeControllerIndex); } else { - m.controller->SetSelectActionStop(kControllerIndex); + m.controller->SetSelectActionStop(activeControllerIndex); } } +void +DeviceDelegateVisionGlass::EnableExternalMouse() { + VRB_LOG("DeviceDelegateVisionGlass::EnableExternalMouse"); + activeControllerIndex = kControllerMouseIndex; + m.controller->SetEnabled(kControllerIndex, false); + m.controller->SetEnabled(kControllerMouseIndex, true); +} + +void +DeviceDelegateVisionGlass::DisableExternalMouse() { + VRB_LOG("DeviceDelegateVisionGlass::DisableExternalMouse"); + activeControllerIndex = kControllerIndex; + m.controller->SetEnabled(kControllerMouseIndex, false); + m.controller->SetEnabled(kControllerIndex, true); +} + void DeviceDelegateVisionGlass::setHead(const double aX, const double aY, const double aZ, const double aW) { m.headOrientation = vrb::Quaternion(aX, aY, aZ, aW); diff --git a/app/src/visionglass/cpp/DeviceDelegateVisionGlass.h b/app/src/visionglass/cpp/DeviceDelegateVisionGlass.h index 8967cf2298..4628e69b0a 100644 --- a/app/src/visionglass/cpp/DeviceDelegateVisionGlass.h +++ b/app/src/visionglass/cpp/DeviceDelegateVisionGlass.h @@ -54,12 +54,17 @@ class DeviceDelegateVisionGlass : public DeviceDelegate { void setHead(const double aX, const double aY, const double aZ, const double aW); void setControllerOrientation(const double aX, const double aY, const double aZ, const double aW); void CalibrateController(); + void EnableExternalMouse(); + void DisableExternalMouse(); + protected: struct State; DeviceDelegateVisionGlass(State& aState); virtual ~DeviceDelegateVisionGlass(); private: State& m; + uint32_t activeControllerIndex; + vrb::Quaternion CorrectedHeadOrientation() const; VRB_NO_DEFAULTS(DeviceDelegateVisionGlass) }; diff --git a/app/src/visionglass/cpp/native-lib.cpp b/app/src/visionglass/cpp/native-lib.cpp index dca0695d1e..f8009af6c2 100644 --- a/app/src/visionglass/cpp/native-lib.cpp +++ b/app/src/visionglass/cpp/native-lib.cpp @@ -111,6 +111,20 @@ JNI_METHOD(void, calibrateController) sAppContext->mDevice->CalibrateController(); } +JNI_METHOD(void, enableExternalMouse) +(JNIEnv*, jobject) { + if (!sAppContext || !sAppContext->mDevice) + return; + sAppContext->mDevice->EnableExternalMouse(); +} + +JNI_METHOD(void, disableExternalMouse) +(JNIEnv*, jobject) { + if (!sAppContext || !sAppContext->mDevice) + return; + sAppContext->mDevice->DisableExternalMouse(); +} + jint JNI_OnLoad(JavaVM* aVm, void*) { if (sAppContext) { return JNI_VERSION_1_6; diff --git a/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java b/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java index bcc59d4947..26dad85eef 100644 --- a/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java +++ b/app/src/visionglass/java/com/igalia/wolvic/PlatformActivity.java @@ -8,6 +8,7 @@ import android.annotation.SuppressLint; import android.app.AlertDialog; import android.app.Presentation; +import android.bluetooth.BluetoothDevice; import android.content.BroadcastReceiver; import android.content.Context; import android.content.DialogInterface; @@ -28,6 +29,7 @@ import android.view.ContextThemeWrapper; import android.view.Display; import android.view.GestureDetector; +import android.view.InputDevice; import android.view.KeyEvent; import android.view.LayoutInflater; import android.view.MotionEvent; @@ -40,6 +42,7 @@ import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.constraintlayout.widget.ConstraintSet; import androidx.databinding.DataBindingUtil; import androidx.fragment.app.FragmentActivity; import androidx.lifecycle.Lifecycle; @@ -59,6 +62,7 @@ import com.igalia.wolvic.databinding.VisionglassLayoutBinding; import com.igalia.wolvic.speech.SpeechRecognizer; import com.igalia.wolvic.speech.SpeechServices; +import com.igalia.wolvic.ui.widgets.MousePointerWidget; import com.igalia.wolvic.ui.widgets.UIWidget; import com.igalia.wolvic.ui.widgets.WidgetManagerDelegate; import com.igalia.wolvic.utils.StringUtils; @@ -88,6 +92,13 @@ public class PlatformActivity extends FragmentActivity implements SensorEventLis private boolean mSwitchedTo3DMode = false; private AlignPhoneDialogFragment mAlignDialogFragment; private AlignNotificationUIDialog mAlignNotificationUIDialog; + private SensorEvent mSensorEvent; + private float mLastMouseX; + private float mLastMouseY; + private float mMouseSensitivity = 0.1f; + private static final float[] WORLD_UP_AXIS = {0, 1, 0}; + private static final float[] WORLD_RIGHT_AXIS = {1, 0, 0}; + private float[] mHeadOrientation = new float[4]; @SuppressWarnings("unused") public static boolean filterPermission(final String aPermission) { @@ -389,6 +400,94 @@ private float[] fromSensorManagerToWorld(float[] eventValues) { return q; } + private float[] fromMouseMovementToWorld(MotionEvent ev) { + assert(ev.getActionMasked() == MotionEvent.ACTION_HOVER_MOVE); + + float currentMouseX = ev.getX(); + float currentMouseY = ev.getY(); + if (mLastMouseX == -1 || mLastMouseY == -1) { + // Initialize last position on the first event + mLastMouseX = currentMouseX; + mLastMouseY = currentMouseY; + // Return the current sensor quaternion if this is the first event + return mHeadOrientation; + } + + float deltaX = currentMouseX - mLastMouseX; + float deltaY = currentMouseY - mLastMouseY; + mLastMouseX = currentMouseX; + mLastMouseY = currentMouseY; + + // Calculate rotation angles from mouse deltas + float yawAngle = deltaX * mMouseSensitivity; + float pitchAngle = deltaY * mMouseSensitivity; + + // Assuming your Quaternion class takes angle in degrees or radians + // Adjust if your library uses a different unit + float[] mouseYawQuaternion = quaternionFromAxisAngle(WORLD_UP_AXIS[0], WORLD_UP_AXIS[1], WORLD_UP_AXIS[2], yawAngle); + float[] mousePitchQuaternion = quaternionFromAxisAngle(WORLD_RIGHT_AXIS[0], WORLD_RIGHT_AXIS[1], WORLD_RIGHT_AXIS[2], pitchAngle); + + // Combine quaternions: sensor * mouseYaw * mousePitch + // The order might need adjustment based on your desired rotation behavior + float[] combinedQuat = multiplyQuaternion(multiplyQuaternion(mHeadOrientation, mouseYawQuaternion), mousePitchQuaternion); + + return combinedQuat; + } + + private static float[] quaternionFromAxisAngle(float x, float y, float z, float angle) { + float halfAngle = (float) Math.toRadians(angle) / 2.0f; + float sinHalfAngle = (float) Math.sin(halfAngle); + return new float[]{(float) Math.cos(halfAngle), x * sinHalfAngle, y * sinHalfAngle, z * sinHalfAngle}; + } + + /** + * Multiplies two quaternions represented as float arrays [x, y, z, w]. + * + * @param q1 The first quaternion [x1, y1, z1, w1]. + * @param q2 The second quaternion [x2, y2, z2, w2]. + * @return A new float array representing the result of q1 * q2 [x_res, y_res, z_res, w_res]. + */ + private static float[] multiplyQuaternion(float[] q1, float[] q2) { + if (q1 == null || q1.length != 4 || q2 == null || q2.length != 4) { + throw new IllegalArgumentException("Quaternion arrays must be non-null and have a length of 4."); + } + + // Extract components (remembering the [x, y, z, w] format) + float x1 = q1[0]; + float y1 = q1[1]; + float z1 = q1[2]; + float w1 = q1[3]; + + float x2 = q2[0]; + float y2 = q2[1]; + float z2 = q2[2]; + float w2 = q2[3]; + + // Apply the quaternion multiplication formula + // w_res = w1*w2 - x1*x2 - y1*y2 - z1*z2 + // x_res = w1*x2 + x1*w2 + y1*z2 - z1*y2 + // y_res = w1*y2 - x1*z2 + y1*w2 + z1*x2 + // z_res = w1*z2 + x1*y2 - y1*x2 + z1*w2 + + float w_res = w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2; + float x_res = w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2; + float y_res = w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2; + float z_res = w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2; + + // Return the result in the [x, y, z, w] format + return new float[]{x_res, y_res, z_res, w_res}; + } + + private void setControllerOrientation(float[] quaternion) { + queueRunnable(() -> setControllerOrientation(quaternion[0], quaternion[1], quaternion[2], quaternion[3])); + + mBinding.realignButton.updatePosition(-quaternion[1], -quaternion[0]); + + if (mAlignDialogFragment != null && mAlignDialogFragment.isVisible()) { + mAlignDialogFragment.updatePosition(-quaternion[1], -quaternion[0]); + } + } + // SensorEventListener overrides @Override public void onSensorChanged(SensorEvent event) { @@ -396,15 +495,80 @@ public void onSensorChanged(SensorEvent event) { if (event.sensor.getType() != Sensor.TYPE_ROTATION_VECTOR) return; + mSensorEvent = event; + final float[] quaternion = fromSensorManagerToWorld(event.values); + setControllerOrientation(quaternion); + } - queueRunnable(() -> setControllerOrientation(quaternion[0], quaternion[1], quaternion[2], quaternion[3])); - mBinding.realignButton.updatePosition(-quaternion[1], -quaternion[0]); + private void updateMousePointerPosition(MotionEvent ev) { + // Update the mouse pointer position in the UI + float x = ev.getX(); + float y = ev.getY(); - if (mAlignDialogFragment != null && mAlignDialogFragment.isVisible()) { - mAlignDialogFragment.updatePosition(-quaternion[1], -quaternion[0]); + Log.d(LOGTAG, "Updating mouse pointer position: x=" + x + ", y=" + y); + + final float[] quaternion = fromMouseMovementToWorld(ev); + setControllerOrientation(quaternion); + } + + @Override + public boolean dispatchGenericMotionEvent(MotionEvent ev) { + if (ev.isFromSource(InputDevice.SOURCE_CLASS_POINTER)) { + switch (ev.getActionMasked()) { + case MotionEvent.ACTION_HOVER_MOVE: + // Mouse movement + // Update the pointer widget's position + updateMousePointerPosition(ev); + break; + case MotionEvent.ACTION_SCROLL: + // Mouse scroll wheel + float vscroll = ev.getAxisValue(MotionEvent.AXIS_VSCROLL); + float hscroll = ev.getAxisValue(MotionEvent.AXIS_HSCROLL); + Log.d(LOGTAG, "Mouse Scroll: vscroll=" + vscroll + ", hscroll=" + hscroll); + break; + } + } + + return super.dispatchGenericMotionEvent(ev); + } + + protected void handleBluetoothHidDeviceConnected(BluetoothDevice device) { + Log.d(LOGTAG, "Bluetooth HID device connected: " + device.getName()); + mSensorManager.unregisterListener(this); + queueRunnable(() -> enableExternalMouse()); + } + + protected void handleBluetoothHidDeviceDisconnected(BluetoothDevice device) { + Log.d(LOGTAG, "Bluetooth HID device disconnected: " + device.getName()); + registerPhoneIMUListener(); + queueRunnable(() -> disableExternalMouse()); + } + + @Override + public boolean dispatchTouchEvent(MotionEvent ev) { + if (ev.isFromSource(InputDevice.SOURCE_CLASS_POINTER)) { + switch (ev.getActionMasked()) { + case MotionEvent.ACTION_DOWN: + // Mouse button down + Log.d(LOGTAG, "Mouse Button Down"); + // You can check which button was pressed using ev.getButtonState() + break; + case MotionEvent.ACTION_UP: + // Mouse button up + Log.d(LOGTAG, "Mouse Button Up"); + break; + case MotionEvent.ACTION_MOVE: + // Mouse movement while a button is pressed (dragging) + float x = ev.getX(); + float y = ev.getY(); + Log.d(LOGTAG, "Mouse Drag Move: x=" + x + ", y=" + y); + // Update pointer position during dragging + break; + } } + return super.dispatchTouchEvent(ev); } @Override @@ -792,7 +956,13 @@ boolean onBackPressed() { private void startIMUService() { Log.d(LOGTAG, "Starting IMU service"); - VisionGlass.getInstance().startImu((w, x, y, z) -> queueRunnable(() -> setHead(x, y, z, w))); + VisionGlass.getInstance().startImu((w, x, y, z) -> { + mHeadOrientation[0] = (float) x; + mHeadOrientation[1] = (float) y; + mHeadOrientation[2] = (float) z; + mHeadOrientation[3] = (float) w; + queueRunnable(() -> setHead(x, y, z, w)); + }); } private final DisplayManager.DisplayListener mDisplayListener = @@ -921,6 +1091,8 @@ public void onDrawFrame(GL10 gl) { @SuppressWarnings("unused") private void setRenderMode(final int aMode) {} + public native void enableExternalMouse(); + public native void disableExternalMouse(); private native void activityCreated(Object aAssetManager); private native void updateViewport(int width, int height); private native void activityPaused(); @@ -931,4 +1103,5 @@ private void setRenderMode(final int aMode) {} private native void setHead(double x, double y, double z, double w); private native void setControllerOrientation(double x, double y, double z, double w); private native void calibrateController(); + }