Add iOS VoiceOver accessibility bridge

CMakeLists.txt

@@ -879,6 +879,8 @@ add_library(Common STATIC
 	Common/Thread/ThreadUtil.h
 	Common/Thread/ThreadManager.cpp
 	Common/Thread/ThreadManager.h
+	Common/UI/Accessibility.cpp
+	Common/UI/Accessibility.h
 	Common/UI/AsyncImageFileView.cpp
 	Common/UI/AsyncImageFileView.h
 	Common/UI/Root.cpp
@@ -1403,6 +1405,8 @@ elseif(IOS AND NOT LIBRETRO)
 		ios/SceneDelegate.h
 		ios/Controls.h
 		ios/Controls.mm
+		ios/AccessibilityBridge.h
+		ios/AccessibilityBridge.mm
 		ios/ViewControllerCommon.h
 		ios/ViewControllerCommon.mm
 		ios/ViewController.mm
@@ -1450,6 +1454,7 @@ elseif(IOS AND NOT LIBRETRO)
 		ios/CameraHelper.mm
 		ios/AudioEngine.mm
 		ios/LocationHelper.mm
+		ios/AccessibilityBridge.mm
 		ios/Controls.mm
 		Core/Util/DarwinFileSystemServices.mm
 		Common/Battery/AppleBatteryClient.m

Common/System/NativeApp.h

@@ -51,6 +51,8 @@ bool NativeIsRestarting();
 void NativeTouch(const TouchInput &touch);
 bool NativeKey(const KeyInput &key);
 void NativeAxis(const AxisInput *axis, size_t count);
+bool NativeAccessibilityFocus(int id);
+bool NativeAccessibilityClick(int id);
 void NativeAccelerometer(float tiltX, float tiltY, float tiltZ);
 void NativeMouseDelta(float dx, float dy);
 

Common/UI/Accessibility.cpp

@@ -0,0 +1,196 @@
+#include "Common/UI/Accessibility.h"
+
+#include <algorithm>
+#include <exception>
+#include <map>
+#include <mutex>
+
+#include "Common/Input/InputState.h"
+#include "Common/Log.h"
+#include "Common/System/NativeApp.h"
+#include "Common/UI/Screen.h"
+#include "Common/UI/UIScreen.h"
+
+namespace UI {
+
+static std::mutex g_accessibilitySnapshotLock;
+static std::mutex g_accessibilityInputLock;
+static std::vector<AccessibilityElementInfo> g_accessibilitySnapshot;
+static uint64_t g_accessibilitySnapshotVersion;
+static bool g_accessibilityEnabled;
+static std::map<int, int> g_heldAccessibilityPointers;
+
+static constexpr int ACCESSIBILITY_TAP_POINTER = 7;
+static constexpr int ACCESSIBILITY_FIRST_HOLD_POINTER = 8;
+static constexpr int ACCESSIBILITY_LAST_HOLD_POINTER = TOUCH_MAX_POINTERS - 1;
+
+static bool BoundsEqual(const Bounds &a, const Bounds &b) {
+	return a.x == b.x && a.y == b.y && a.w == b.w && a.h == b.h;
+}
+
+static bool AccessibilitySnapshotsEqual(const std::vector<AccessibilityElementInfo> &a, const std::vector<AccessibilityElementInfo> &b) {
+	if (a.size() != b.size()) {
+		return false;
+	}
+	for (size_t i = 0; i < a.size(); ++i) {
+		if (a[i].id != b[i].id || a[i].label != b[i].label || !BoundsEqual(a[i].bounds, b[i].bounds) ||
+			a[i].role != b[i].role || a[i].enabled != b[i].enabled || a[i].checked != b[i].checked || a[i].selected != b[i].selected ||
+			a[i].clickable != b[i].clickable || a[i].longClickable != b[i].longClickable ||
+			a[i].touchX != b[i].touchX || a[i].touchY != b[i].touchY) {
+			return false;
+		}
+	}
+	return true;
+}
+
+std::vector<AccessibilityElementInfo> BuildAccessibilitySnapshot(ScreenManager *screenManager) {
+	std::vector<AccessibilityElementInfo> elements;
+	if (!screenManager) {
+		return elements;
+	}
+
+	Screen *screen = screenManager->topScreen();
+	UIScreen *uiScreen = dynamic_cast<UIScreen *>(screen);
+	if (!uiScreen) {
+		return elements;
+	}
+
+	uiScreen->GetAccessibilityElements(elements);
+	return elements;
+}
+
+void UpdateCachedAccessibilitySnapshot(ScreenManager *screenManager) {
+	std::vector<AccessibilityElementInfo> snapshot;
+	try {
+		snapshot = BuildAccessibilitySnapshot(screenManager);
+	} catch (const std::exception &e) {
+		WARN_LOG(Log::UI, "Accessibility snapshot failed: %s", e.what());
+	} catch (...) {
+		WARN_LOG(Log::UI, "Accessibility snapshot failed with unknown exception");
+	}
+	bool releaseInputs = false;
+	{
+		std::lock_guard<std::mutex> guard(g_accessibilitySnapshotLock);
+		if (!AccessibilitySnapshotsEqual(g_accessibilitySnapshot, snapshot)) {
+			const bool hadGamepadControls = std::any_of(g_accessibilitySnapshot.begin(), g_accessibilitySnapshot.end(), [](const AccessibilityElementInfo &info) {
+				return info.role == AccessibilityRole::GamepadControl;
+			});
+			const bool hasGamepadControls = std::any_of(snapshot.begin(), snapshot.end(), [](const AccessibilityElementInfo &info) {
+				return info.role == AccessibilityRole::GamepadControl;
+			});
+			releaseInputs = hadGamepadControls && !hasGamepadControls;
+			g_accessibilitySnapshot = std::move(snapshot);
+			++g_accessibilitySnapshotVersion;
+			NOTICE_LOG(Log::UI, "Accessibility snapshot updated: version=%llu elements=%zu",
+				(unsigned long long)g_accessibilitySnapshotVersion, g_accessibilitySnapshot.size());
+		}
+	}
+	if (releaseInputs) {
+		ReleaseAccessibilityInputs();
+	}
+}
+
+std::vector<AccessibilityElementInfo> GetCachedAccessibilitySnapshot() {
+	std::lock_guard<std::mutex> guard(g_accessibilitySnapshotLock);
+	return g_accessibilitySnapshot;
+}
+
+uint64_t GetCachedAccessibilitySnapshotVersion() {
+	std::lock_guard<std::mutex> guard(g_accessibilitySnapshotLock);
+	return g_accessibilitySnapshotVersion;
+}
+
+void ClearCachedAccessibilitySnapshot() {
+	ReleaseAccessibilityInputs();
+	std::lock_guard<std::mutex> guard(g_accessibilitySnapshotLock);
+	if (!g_accessibilitySnapshot.empty()) {
+		g_accessibilitySnapshot.clear();
+		++g_accessibilitySnapshotVersion;
+	}
+}
+
+void SetAccessibilityEnabled(bool enabled) {
+	if (!enabled) {
+		ClearCachedAccessibilitySnapshot();
+	}
+	std::lock_guard<std::mutex> guard(g_accessibilitySnapshotLock);
+	g_accessibilityEnabled = enabled;
+}
+
+bool IsAccessibilityEnabled() {
+	std::lock_guard<std::mutex> guard(g_accessibilitySnapshotLock);
+	return g_accessibilityEnabled;
+}
+
+bool FocusAccessibilityElement(ScreenManager *screenManager, int id) {
+	if (!screenManager) {
+		return false;
+	}
+	UIScreen *screen = dynamic_cast<UIScreen *>(screenManager->topScreen());
+	return screen && screen->FocusAccessibilityElement(id);
+}
+
+static void SendAccessibilityTouch(const AccessibilityElementInfo &info, int pointerId, TouchInputFlags flags) {
+	TouchInput touch{};
+	touch.id = pointerId;
+	touch.x = info.touchX;
+	touch.y = info.touchY;
+	touch.flags = flags;
+	NativeTouch(touch);
+}
+
+bool PerformAccessibilityClick(int id, bool longClick) {
+	AccessibilityElementInfo info;
+	{
+		std::lock_guard<std::mutex> guard(g_accessibilitySnapshotLock);
+		auto iter = std::find_if(g_accessibilitySnapshot.begin(), g_accessibilitySnapshot.end(), [id](const AccessibilityElementInfo &candidate) {
+			return candidate.id == id;
+		});
+		if (iter == g_accessibilitySnapshot.end() || !iter->enabled || !iter->clickable) {
+			return false;
+		}
+		info = *iter;
+	}
+
+	{
+		std::lock_guard<std::mutex> guard(g_accessibilityInputLock);
+		auto held = g_heldAccessibilityPointers.find(id);
+		if (held != g_heldAccessibilityPointers.end()) {
+			SendAccessibilityTouch(info, held->second, TouchInputFlags::UP);
+			g_heldAccessibilityPointers.erase(held);
+			return true;
+		}
+		if (!longClick || !info.longClickable) {
+			if (info.role == AccessibilityRole::Tab) {
+				return NativeAccessibilityClick(id);
+			}
+			SendAccessibilityTouch(info, ACCESSIBILITY_TAP_POINTER, TouchInputFlags::DOWN);
+			SendAccessibilityTouch(info, ACCESSIBILITY_TAP_POINTER, TouchInputFlags::UP);
+			return true;
+		}
+
+		for (int pointerId = ACCESSIBILITY_FIRST_HOLD_POINTER; pointerId <= ACCESSIBILITY_LAST_HOLD_POINTER; ++pointerId) {
+			const bool used = std::any_of(g_heldAccessibilityPointers.begin(), g_heldAccessibilityPointers.end(), [pointerId](const auto &entry) {
+				return entry.second == pointerId;
+			});
+			if (!used) {
+				SendAccessibilityTouch(info, pointerId, TouchInputFlags::DOWN);
+				g_heldAccessibilityPointers[id] = pointerId;
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+void ReleaseAccessibilityInputs() {
+	{
+		std::lock_guard<std::mutex> guard(g_accessibilityInputLock);
+		g_heldAccessibilityPointers.clear();
+	}
+	TouchInput touch{};
+	touch.flags = TouchInputFlags::RELEASE_ALL;
+	NativeTouch(touch);
+}
+
+}  // namespace UI

Common/UI/Accessibility.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "Common/Math/geom2d.h"
+
+class ScreenManager;
+
+namespace UI {
+
+enum class AccessibilityRole {
+	StaticText,
+	Button,
+	Choice,
+	Checkbox,
+	Radio,
+	Tab,
+	Slider,
+	TextField,
+	Progress,
+	Heading,
+	GamepadControl,
+	Image,
+};
+
+struct AccessibilityElementInfo {
+	int id = -1;
+	std::string label;
+	Bounds bounds;
+	AccessibilityRole role = AccessibilityRole::StaticText;
+	bool enabled = true;
+	bool checked = false;
+	bool selected = false;
+	bool clickable = false;
+	bool longClickable = false;
+	float touchX = 0.0f;
+	float touchY = 0.0f;
+};
+
+std::vector<AccessibilityElementInfo> BuildAccessibilitySnapshot(ScreenManager *screenManager);
+void UpdateCachedAccessibilitySnapshot(ScreenManager *screenManager);
+std::vector<AccessibilityElementInfo> GetCachedAccessibilitySnapshot();
+uint64_t GetCachedAccessibilitySnapshotVersion();
+void ClearCachedAccessibilitySnapshot();
+void SetAccessibilityEnabled(bool enabled);
+bool IsAccessibilityEnabled();
+bool FocusAccessibilityElement(ScreenManager *screenManager, int id);
+bool PerformAccessibilityClick(int id, bool longClick);
+void ReleaseAccessibilityInputs();
+
+}  // namespace UI

Common/UI/TabHolder.cpp

@@ -231,7 +231,7 @@ void TabHolder::OnTabClick(EventParams &e) {
 	// In that case, we make the view gone and then visible - this scrolls scrollviews to the top.
 	if (e.b != 0) {
 		// SetCurrentTab calls EnsureTab if needed.
-		SetCurrentTab((int)e.a);
+		SetCurrentTab((int)e.a, e.x != 0);
 	}
 }
 
@@ -275,6 +275,7 @@ void ChoiceStrip::AddChoice(std::string_view title, ImageID imageId) {
 		orientation_ == ORIENT_HORIZONTAL ?
 		nullptr :
 		new LinearLayoutParams(FILL_PARENT, ITEM_HEIGHT));
+	c->SetAccessibilityTab(topTabs_);
 	c->OnClick.Handle(this, &ChoiceStrip::OnChoiceClick);
 	Add(c);
 	choices_.push_back(c);
@@ -287,13 +288,21 @@ void ChoiceStrip::AddChoice(ImageID buttonImage) {
 		orientation_ == ORIENT_HORIZONTAL ?
 		nullptr :
 		new LinearLayoutParams(FILL_PARENT, ITEM_HEIGHT));
+	c->SetAccessibilityTab(topTabs_);
 	c->OnClick.Handle(this, &ChoiceStrip::OnChoiceClick);
 	Add(c);
 	choices_.push_back(c);
 	if (selected_ == (int)choices_.size() - 1)
 		c->Press();
 }
 
+void ChoiceStrip::SetTopTabs(bool tabs) {
+	topTabs_ = tabs;
+	for (StickyChoice *choice : choices_) {
+		choice->SetAccessibilityTab(tabs);
+	}
+}
+
 void ChoiceStrip::OnChoiceClick(EventParams &e) {
 	// Unstick the other choices that weren't clicked.
 	for (int i = 0; i < (int)choices_.size(); i++) {
@@ -309,6 +318,7 @@ void ChoiceStrip::OnChoiceClick(EventParams &e) {
 	e2.a = selected_;
 	// Set to 1 to indicate an explicit click.
 	e2.b = 1;
+	e2.x = e2.v && static_cast<StickyChoice *>(e2.v)->ConsumeAccessibilityActivation() ? 1 : 0;
 	// Dispatch immediately (we're already on the UI thread as we're in an event handler).
 	OnChoice.Dispatch(e2);
 }

Common/UI/TabHolder.h

@@ -86,7 +86,7 @@ class ChoiceStrip : public LinearLayout {
 
 	bool Key(const KeyInput &input) override;
 
-	void SetTopTabs(bool tabs) { topTabs_ = tabs; }
+	void SetTopTabs(bool tabs);
 
 	std::string DescribeLog() const override { return "ChoiceStrip: " + View::DescribeLog(); }
 	std::string DescribeText() const override;