diff --git a/CMakeLists.txt b/CMakeLists.txt index ec22f64164f4..1f5993857634 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -879,6 +879,8 @@ add_library(Common STATIC Common/Thread/ThreadUtil.h Common/Thread/ThreadManager.cpp Common/Thread/ThreadManager.h + Common/UI/Accessibility.cpp + Common/UI/Accessibility.h Common/UI/AsyncImageFileView.cpp Common/UI/AsyncImageFileView.h Common/UI/Root.cpp @@ -1403,6 +1405,8 @@ elseif(IOS AND NOT LIBRETRO) ios/SceneDelegate.h ios/Controls.h ios/Controls.mm + ios/AccessibilityBridge.h + ios/AccessibilityBridge.mm ios/ViewControllerCommon.h ios/ViewControllerCommon.mm ios/ViewController.mm @@ -1450,6 +1454,7 @@ elseif(IOS AND NOT LIBRETRO) ios/CameraHelper.mm ios/AudioEngine.mm ios/LocationHelper.mm + ios/AccessibilityBridge.mm ios/Controls.mm Core/Util/DarwinFileSystemServices.mm Common/Battery/AppleBatteryClient.m diff --git a/Common/System/NativeApp.h b/Common/System/NativeApp.h index 02841f87e50e..e1ee1a159060 100644 --- a/Common/System/NativeApp.h +++ b/Common/System/NativeApp.h @@ -51,6 +51,8 @@ bool NativeIsRestarting(); void NativeTouch(const TouchInput &touch); bool NativeKey(const KeyInput &key); void NativeAxis(const AxisInput *axis, size_t count); +bool NativeAccessibilityFocus(int id); +bool NativeAccessibilityClick(int id); void NativeAccelerometer(float tiltX, float tiltY, float tiltZ); void NativeMouseDelta(float dx, float dy); diff --git a/Common/UI/Accessibility.cpp b/Common/UI/Accessibility.cpp new file mode 100644 index 000000000000..b9ba2eaeb988 --- /dev/null +++ b/Common/UI/Accessibility.cpp @@ -0,0 +1,196 @@ +#include "Common/UI/Accessibility.h" + +#include +#include +#include +#include + +#include "Common/Input/InputState.h" +#include "Common/Log.h" +#include "Common/System/NativeApp.h" +#include "Common/UI/Screen.h" +#include "Common/UI/UIScreen.h" + +namespace UI { + +static std::mutex g_accessibilitySnapshotLock; +static std::mutex g_accessibilityInputLock; +static std::vector g_accessibilitySnapshot; +static uint64_t g_accessibilitySnapshotVersion; +static bool g_accessibilityEnabled; +static std::map g_heldAccessibilityPointers; + +static constexpr int ACCESSIBILITY_TAP_POINTER = 7; +static constexpr int ACCESSIBILITY_FIRST_HOLD_POINTER = 8; +static constexpr int ACCESSIBILITY_LAST_HOLD_POINTER = TOUCH_MAX_POINTERS - 1; + +static bool BoundsEqual(const Bounds &a, const Bounds &b) { + return a.x == b.x && a.y == b.y && a.w == b.w && a.h == b.h; +} + +static bool AccessibilitySnapshotsEqual(const std::vector &a, const std::vector &b) { + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (a[i].id != b[i].id || a[i].label != b[i].label || !BoundsEqual(a[i].bounds, b[i].bounds) || + a[i].role != b[i].role || a[i].enabled != b[i].enabled || a[i].checked != b[i].checked || a[i].selected != b[i].selected || + a[i].clickable != b[i].clickable || a[i].longClickable != b[i].longClickable || + a[i].touchX != b[i].touchX || a[i].touchY != b[i].touchY) { + return false; + } + } + return true; +} + +std::vector BuildAccessibilitySnapshot(ScreenManager *screenManager) { + std::vector elements; + if (!screenManager) { + return elements; + } + + Screen *screen = screenManager->topScreen(); + UIScreen *uiScreen = dynamic_cast(screen); + if (!uiScreen) { + return elements; + } + + uiScreen->GetAccessibilityElements(elements); + return elements; +} + +void UpdateCachedAccessibilitySnapshot(ScreenManager *screenManager) { + std::vector snapshot; + try { + snapshot = BuildAccessibilitySnapshot(screenManager); + } catch (const std::exception &e) { + WARN_LOG(Log::UI, "Accessibility snapshot failed: %s", e.what()); + } catch (...) { + WARN_LOG(Log::UI, "Accessibility snapshot failed with unknown exception"); + } + bool releaseInputs = false; + { + std::lock_guard guard(g_accessibilitySnapshotLock); + if (!AccessibilitySnapshotsEqual(g_accessibilitySnapshot, snapshot)) { + const bool hadGamepadControls = std::any_of(g_accessibilitySnapshot.begin(), g_accessibilitySnapshot.end(), [](const AccessibilityElementInfo &info) { + return info.role == AccessibilityRole::GamepadControl; + }); + const bool hasGamepadControls = std::any_of(snapshot.begin(), snapshot.end(), [](const AccessibilityElementInfo &info) { + return info.role == AccessibilityRole::GamepadControl; + }); + releaseInputs = hadGamepadControls && !hasGamepadControls; + g_accessibilitySnapshot = std::move(snapshot); + ++g_accessibilitySnapshotVersion; + NOTICE_LOG(Log::UI, "Accessibility snapshot updated: version=%llu elements=%zu", + (unsigned long long)g_accessibilitySnapshotVersion, g_accessibilitySnapshot.size()); + } + } + if (releaseInputs) { + ReleaseAccessibilityInputs(); + } +} + +std::vector GetCachedAccessibilitySnapshot() { + std::lock_guard guard(g_accessibilitySnapshotLock); + return g_accessibilitySnapshot; +} + +uint64_t GetCachedAccessibilitySnapshotVersion() { + std::lock_guard guard(g_accessibilitySnapshotLock); + return g_accessibilitySnapshotVersion; +} + +void ClearCachedAccessibilitySnapshot() { + ReleaseAccessibilityInputs(); + std::lock_guard guard(g_accessibilitySnapshotLock); + if (!g_accessibilitySnapshot.empty()) { + g_accessibilitySnapshot.clear(); + ++g_accessibilitySnapshotVersion; + } +} + +void SetAccessibilityEnabled(bool enabled) { + if (!enabled) { + ClearCachedAccessibilitySnapshot(); + } + std::lock_guard guard(g_accessibilitySnapshotLock); + g_accessibilityEnabled = enabled; +} + +bool IsAccessibilityEnabled() { + std::lock_guard guard(g_accessibilitySnapshotLock); + return g_accessibilityEnabled; +} + +bool FocusAccessibilityElement(ScreenManager *screenManager, int id) { + if (!screenManager) { + return false; + } + UIScreen *screen = dynamic_cast(screenManager->topScreen()); + return screen && screen->FocusAccessibilityElement(id); +} + +static void SendAccessibilityTouch(const AccessibilityElementInfo &info, int pointerId, TouchInputFlags flags) { + TouchInput touch{}; + touch.id = pointerId; + touch.x = info.touchX; + touch.y = info.touchY; + touch.flags = flags; + NativeTouch(touch); +} + +bool PerformAccessibilityClick(int id, bool longClick) { + AccessibilityElementInfo info; + { + std::lock_guard guard(g_accessibilitySnapshotLock); + auto iter = std::find_if(g_accessibilitySnapshot.begin(), g_accessibilitySnapshot.end(), [id](const AccessibilityElementInfo &candidate) { + return candidate.id == id; + }); + if (iter == g_accessibilitySnapshot.end() || !iter->enabled || !iter->clickable) { + return false; + } + info = *iter; + } + + { + std::lock_guard guard(g_accessibilityInputLock); + auto held = g_heldAccessibilityPointers.find(id); + if (held != g_heldAccessibilityPointers.end()) { + SendAccessibilityTouch(info, held->second, TouchInputFlags::UP); + g_heldAccessibilityPointers.erase(held); + return true; + } + if (!longClick || !info.longClickable) { + if (info.role == AccessibilityRole::Tab) { + return NativeAccessibilityClick(id); + } + SendAccessibilityTouch(info, ACCESSIBILITY_TAP_POINTER, TouchInputFlags::DOWN); + SendAccessibilityTouch(info, ACCESSIBILITY_TAP_POINTER, TouchInputFlags::UP); + return true; + } + + for (int pointerId = ACCESSIBILITY_FIRST_HOLD_POINTER; pointerId <= ACCESSIBILITY_LAST_HOLD_POINTER; ++pointerId) { + const bool used = std::any_of(g_heldAccessibilityPointers.begin(), g_heldAccessibilityPointers.end(), [pointerId](const auto &entry) { + return entry.second == pointerId; + }); + if (!used) { + SendAccessibilityTouch(info, pointerId, TouchInputFlags::DOWN); + g_heldAccessibilityPointers[id] = pointerId; + return true; + } + } + } + return false; +} + +void ReleaseAccessibilityInputs() { + { + std::lock_guard guard(g_accessibilityInputLock); + g_heldAccessibilityPointers.clear(); + } + TouchInput touch{}; + touch.flags = TouchInputFlags::RELEASE_ALL; + NativeTouch(touch); +} + +} // namespace UI diff --git a/Common/UI/Accessibility.h b/Common/UI/Accessibility.h new file mode 100644 index 000000000000..8f07516e9d83 --- /dev/null +++ b/Common/UI/Accessibility.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include + +#include "Common/Math/geom2d.h" + +class ScreenManager; + +namespace UI { + +enum class AccessibilityRole { + StaticText, + Button, + Choice, + Checkbox, + Radio, + Tab, + Slider, + TextField, + Progress, + Heading, + GamepadControl, + Image, +}; + +struct AccessibilityElementInfo { + int id = -1; + std::string label; + Bounds bounds; + AccessibilityRole role = AccessibilityRole::StaticText; + bool enabled = true; + bool checked = false; + bool selected = false; + bool clickable = false; + bool longClickable = false; + float touchX = 0.0f; + float touchY = 0.0f; +}; + +std::vector BuildAccessibilitySnapshot(ScreenManager *screenManager); +void UpdateCachedAccessibilitySnapshot(ScreenManager *screenManager); +std::vector GetCachedAccessibilitySnapshot(); +uint64_t GetCachedAccessibilitySnapshotVersion(); +void ClearCachedAccessibilitySnapshot(); +void SetAccessibilityEnabled(bool enabled); +bool IsAccessibilityEnabled(); +bool FocusAccessibilityElement(ScreenManager *screenManager, int id); +bool PerformAccessibilityClick(int id, bool longClick); +void ReleaseAccessibilityInputs(); + +} // namespace UI diff --git a/Common/UI/TabHolder.cpp b/Common/UI/TabHolder.cpp index 2456a19f14bf..cc25430324f4 100644 --- a/Common/UI/TabHolder.cpp +++ b/Common/UI/TabHolder.cpp @@ -231,7 +231,7 @@ void TabHolder::OnTabClick(EventParams &e) { // In that case, we make the view gone and then visible - this scrolls scrollviews to the top. if (e.b != 0) { // SetCurrentTab calls EnsureTab if needed. - SetCurrentTab((int)e.a); + SetCurrentTab((int)e.a, e.x != 0); } } @@ -275,6 +275,7 @@ void ChoiceStrip::AddChoice(std::string_view title, ImageID imageId) { orientation_ == ORIENT_HORIZONTAL ? nullptr : new LinearLayoutParams(FILL_PARENT, ITEM_HEIGHT)); + c->SetAccessibilityTab(topTabs_); c->OnClick.Handle(this, &ChoiceStrip::OnChoiceClick); Add(c); choices_.push_back(c); @@ -287,6 +288,7 @@ void ChoiceStrip::AddChoice(ImageID buttonImage) { orientation_ == ORIENT_HORIZONTAL ? nullptr : new LinearLayoutParams(FILL_PARENT, ITEM_HEIGHT)); + c->SetAccessibilityTab(topTabs_); c->OnClick.Handle(this, &ChoiceStrip::OnChoiceClick); Add(c); choices_.push_back(c); @@ -294,6 +296,13 @@ void ChoiceStrip::AddChoice(ImageID buttonImage) { c->Press(); } +void ChoiceStrip::SetTopTabs(bool tabs) { + topTabs_ = tabs; + for (StickyChoice *choice : choices_) { + choice->SetAccessibilityTab(tabs); + } +} + void ChoiceStrip::OnChoiceClick(EventParams &e) { // Unstick the other choices that weren't clicked. for (int i = 0; i < (int)choices_.size(); i++) { @@ -309,6 +318,7 @@ void ChoiceStrip::OnChoiceClick(EventParams &e) { e2.a = selected_; // Set to 1 to indicate an explicit click. e2.b = 1; + e2.x = e2.v && static_cast(e2.v)->ConsumeAccessibilityActivation() ? 1 : 0; // Dispatch immediately (we're already on the UI thread as we're in an event handler). OnChoice.Dispatch(e2); } diff --git a/Common/UI/TabHolder.h b/Common/UI/TabHolder.h index a4ca99ccbdf2..cfea60f31ec5 100644 --- a/Common/UI/TabHolder.h +++ b/Common/UI/TabHolder.h @@ -86,7 +86,7 @@ class ChoiceStrip : public LinearLayout { bool Key(const KeyInput &input) override; - void SetTopTabs(bool tabs) { topTabs_ = tabs; } + void SetTopTabs(bool tabs); std::string DescribeLog() const override { return "ChoiceStrip: " + View::DescribeLog(); } std::string DescribeText() const override; diff --git a/Common/UI/UIScreen.cpp b/Common/UI/UIScreen.cpp index 694212f353ea..4709a13a5d48 100644 --- a/Common/UI/UIScreen.cpp +++ b/Common/UI/UIScreen.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include "Common/Log.h" #include "Common/System/Display.h" @@ -6,6 +8,7 @@ #include "Common/System/Request.h" #include "Common/Input/InputState.h" #include "Common/Input/KeyCodes.h" +#include "Common/UI/Accessibility.h" #include "Common/UI/UIScreen.h" #include "Common/UI/Context.h" #include "Common/UI/Screen.h" @@ -14,6 +17,73 @@ static constexpr bool ClickDebug = false; +static UI::AccessibilityRole AccessibilityRoleForView(UI::View *view) { + if (UI::StickyChoice *stickyChoice = dynamic_cast(view); stickyChoice && stickyChoice->IsAccessibilityTab()) { + return UI::AccessibilityRole::Tab; + } + if (dynamic_cast(view)) { + return UI::AccessibilityRole::Button; + } + if (dynamic_cast(view)) { + return UI::AccessibilityRole::Checkbox; + } + if (dynamic_cast(view)) { + return UI::AccessibilityRole::Radio; + } + if (dynamic_cast(view) || dynamic_cast(view)) { + return UI::AccessibilityRole::Slider; + } + if (dynamic_cast(view)) { + return UI::AccessibilityRole::TextField; + } + if (dynamic_cast(view)) { + return UI::AccessibilityRole::Progress; + } + if (dynamic_cast(view) || dynamic_cast(view)) { + return UI::AccessibilityRole::Heading; + } + if (dynamic_cast(view)) { + return UI::AccessibilityRole::GamepadControl; + } + if (dynamic_cast(view) || + dynamic_cast(view) || dynamic_cast(view)) { + return UI::AccessibilityRole::Choice; + } + return UI::AccessibilityRole::StaticText; +} + +static bool ShouldIncludeAccessibilityView(UI::AccessibilityRole role) { + switch (role) { + case UI::AccessibilityRole::Button: + case UI::AccessibilityRole::Choice: + case UI::AccessibilityRole::Checkbox: + case UI::AccessibilityRole::Radio: + case UI::AccessibilityRole::Tab: + case UI::AccessibilityRole::Slider: + case UI::AccessibilityRole::TextField: + case UI::AccessibilityRole::Progress: + case UI::AccessibilityRole::Heading: + case UI::AccessibilityRole::GamepadControl: + case UI::AccessibilityRole::StaticText: + return true; + default: + return false; + } +} + +static bool IsUsefulAccessibilityLabel(const std::string &label) { + return !label.empty() && label != "button" && label != "choice" && label != "radio button" && label != "text field"; +} + +static std::string TrimAccessibilityLabel(std::string label) { + const auto begin = std::find_if(label.begin(), label.end(), [](unsigned char c) { return !std::isspace(c); }); + const auto end = std::find_if(label.rbegin(), label.rend(), [](unsigned char c) { return !std::isspace(c); }).base(); + if (begin >= end) { + return ""; + } + return std::string(begin, end); +} + UIScreen::UIScreen() : Screen() { lastOrientation_ = GetDeviceOrientation(); } @@ -114,6 +184,14 @@ void UIScreen::UnsyncAxis(const AxisInput *axes, size_t count) { } } +void UIScreen::UnsyncAccessibilityActivate(int id) { + QueuedEvent ev{}; + ev.type = QueuedEventType::ACCESSIBILITY_ACTIVATE; + ev.accessibilityId = id; + std::lock_guard guard(eventQueueLock_); + eventQueue_.push_back(ev); +} + bool UIScreen::UnsyncKey(const KeyInput &key) { bool retval = false; if (root_) { @@ -222,6 +300,9 @@ void UIScreen::update() { case QueuedEventType::AXIS: axis(ev.axis); break; + case QueuedEventType::ACCESSIBILITY_ACTIVATE: + ActivateAccessibilityElement(ev.accessibilityId); + break; } } @@ -289,6 +370,157 @@ void UIScreen::TriggerFinish(DialogResult result) { screenManager()->finishDialog(this, result); } +void UIScreen::GetAccessibilityElements(std::vector &elements) { + if (!root_) { + return; + } + + std::lock_guard guard(screenManager()->inputLock_); + int nextId = 1; + root_->RecurseVisible([&](UI::View *view) { + try { + if (!view || view->GetVisibility() != UI::V_VISIBLE) { + return; + } + const Bounds &bounds = view->GetBounds(); + if (bounds.w <= 0.0f || bounds.h <= 0.0f) { + return; + } + const size_t customElementCount = elements.size(); + view->GetAccessibilityElements(elements); + if (elements.size() != customElementCount || view->SuppressDefaultAccessibilityElement()) { + return; + } + if (view->IsViewGroup() && !view->CanBeFocused()) { + return; + } + UI::AccessibilityRole role = AccessibilityRoleForView(view); + if (!ShouldIncludeAccessibilityView(role)) { + return; + } + UI::RadioButton *radio = dynamic_cast(view); + UI::Choice *choice = dynamic_cast(view); + UI::CheckBox *checkbox = dynamic_cast(view); + UI::ItemHeader *itemHeader = dynamic_cast(view); + UI::PopupHeader *popupHeader = dynamic_cast(view); + std::string label = TrimAccessibilityLabel(radio ? std::string(radio->Text()) : + checkbox ? checkbox->AccessibilityText() : + itemHeader ? std::string(itemHeader->Text()) : + popupHeader ? std::string(popupHeader->Text()) : + choice ? std::string(choice->Text()) : view->DescribeText()); + if (!IsUsefulAccessibilityLabel(label)) { + return; + } + + UI::AccessibilityElementInfo info; + info.id = nextId++; + info.label = label; + info.bounds = bounds; + info.role = role; + info.enabled = view->IsEnabled(); + info.checked = dynamic_cast(view) && static_cast(view)->Toggled(); + if (radio) { + info.checked = radio->Selected(); + } + if (UI::StickyChoice *stickyChoice = dynamic_cast(view); stickyChoice && stickyChoice->IsAccessibilityTab()) { + info.selected = stickyChoice->IsDown(); + } + info.clickable = dynamic_cast(view) != nullptr; + info.touchX = bounds.centerX(); + info.touchY = bounds.centerY(); + elements.push_back(info); + } catch (const std::exception &e) { + WARN_LOG(Log::UI, "Skipping accessibility element after exception: %s", e.what()); + } catch (...) { + WARN_LOG(Log::UI, "Skipping accessibility element after unknown exception"); + } + }); +} + +bool UIScreen::FocusAccessibilityElement(int id) { + if (!root_) { + return false; + } + std::lock_guard guard(screenManager()->inputLock_); + int nextId = 1; + bool focused = false; + root_->RecurseVisible([&](UI::View *view) { + if (focused || !view || view->GetVisibility() != UI::V_VISIBLE) { + return; + } + const Bounds &bounds = view->GetBounds(); + if (bounds.w <= 0.0f || bounds.h <= 0.0f) { + return; + } + std::vector customElements; + view->GetAccessibilityElements(customElements); + if (!customElements.empty() || view->SuppressDefaultAccessibilityElement() || (view->IsViewGroup() && !view->CanBeFocused())) { + return; + } + const UI::AccessibilityRole role = AccessibilityRoleForView(view); + UI::RadioButton *radio = dynamic_cast(view); + UI::Choice *choice = dynamic_cast(view); + UI::CheckBox *checkbox = dynamic_cast(view); + UI::ItemHeader *itemHeader = dynamic_cast(view); + UI::PopupHeader *popupHeader = dynamic_cast(view); + const std::string label = TrimAccessibilityLabel(radio ? std::string(radio->Text()) : + checkbox ? checkbox->AccessibilityText() : + itemHeader ? std::string(itemHeader->Text()) : + popupHeader ? std::string(popupHeader->Text()) : + choice ? std::string(choice->Text()) : view->DescribeText()); + if (!ShouldIncludeAccessibilityView(role) || !IsUsefulAccessibilityLabel(label)) { + return; + } + if (nextId++ == id && view->CanBeFocused()) { + focused = view->SetFocus(UI::FocusFlags::CAUSE_OTHER); + if (focused) { + root_->SubviewFocused(view); + } + } + }); + return focused; +} + +bool UIScreen::ActivateAccessibilityElement(int id) { + if (!root_) { + return false; + } + int nextId = 1; + bool activated = false; + root_->RecurseVisible([&](UI::View *view) { + if (activated || !view || view->GetVisibility() != UI::V_VISIBLE) { + return; + } + const Bounds &bounds = view->GetBounds(); + if (bounds.w <= 0.0f || bounds.h <= 0.0f) { + return; + } + std::vector customElements; + view->GetAccessibilityElements(customElements); + if (!customElements.empty() || view->SuppressDefaultAccessibilityElement() || (view->IsViewGroup() && !view->CanBeFocused())) { + return; + } + const UI::AccessibilityRole role = AccessibilityRoleForView(view); + UI::RadioButton *radio = dynamic_cast(view); + UI::Choice *choice = dynamic_cast(view); + UI::CheckBox *checkbox = dynamic_cast(view); + UI::ItemHeader *itemHeader = dynamic_cast(view); + UI::PopupHeader *popupHeader = dynamic_cast(view); + const std::string label = TrimAccessibilityLabel(radio ? std::string(radio->Text()) : + checkbox ? checkbox->AccessibilityText() : + itemHeader ? std::string(itemHeader->Text()) : + popupHeader ? std::string(popupHeader->Text()) : + choice ? std::string(choice->Text()) : view->DescribeText()); + if (!ShouldIncludeAccessibilityView(role) || !IsUsefulAccessibilityLabel(label)) { + return; + } + if (nextId++ == id) { + activated = view->ActivateAccessibility(); + } + }); + return activated; +} + bool UIDialogScreen::key(const KeyInput &key) { bool retval = UIScreen::key(key); if (!retval && (key.flags & KeyInputFlags::DOWN) && UI::IsEscapeKey(key)) { diff --git a/Common/UI/UIScreen.h b/Common/UI/UIScreen.h index 046c4a9f1d58..77f856f14845 100644 --- a/Common/UI/UIScreen.h +++ b/Common/UI/UIScreen.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "Common/Math/lin/vec3.h" #include "Common/UI/Screen.h" @@ -16,11 +17,15 @@ class I18NCategory; namespace Draw { class DrawContext; } +namespace UI { + struct AccessibilityElementInfo; +} enum class QueuedEventType : u8 { KEY, AXIS, TOUCH, + ACCESSIBILITY_ACTIVATE, }; struct QueuedEvent { @@ -29,6 +34,7 @@ struct QueuedEvent { TouchInput touch; KeyInput key; AxisInput axis; + int accessibilityId; }; }; @@ -62,6 +68,7 @@ class UIScreen : public Screen { bool UnsyncTouch(const TouchInput &touch) override; bool UnsyncKey(const KeyInput &key) override; void UnsyncAxis(const AxisInput *axes, size_t count) override; + void UnsyncAccessibilityActivate(int id); TouchInput transformTouch(const TouchInput &touch) override; @@ -79,6 +86,10 @@ class UIScreen : public Screen { modifiersPressed_ = Modifier::NONE; } + virtual void GetAccessibilityElements(std::vector &elements); + bool FocusAccessibilityElement(int id); + bool ActivateAccessibilityElement(int id); + protected: virtual void CreateViews() = 0; diff --git a/Common/UI/View.h b/Common/UI/View.h index 8782e362436d..92f929960e87 100644 --- a/Common/UI/View.h +++ b/Common/UI/View.h @@ -45,6 +45,7 @@ namespace Draw { namespace UI { class View; +struct AccessibilityElementInfo; enum class FocusFlags; enum DrawableType { @@ -384,6 +385,7 @@ class View { // touch response from the frame rate. Same with Key and Axis. virtual bool Key(const KeyInput &input) { return false; } virtual bool Touch(const TouchInput &input) { return true; } + virtual bool ActivateAccessibility() { return false; } virtual void Axis(const AxisInput &input) {} virtual void Update(); @@ -396,6 +398,8 @@ class View { virtual std::string DescribeLog() const; // Accessible/searchable description. virtual std::string DescribeText() const { return ""; } + virtual void GetAccessibilityElements(std::vector &elements) const {} + virtual bool SuppressDefaultAccessibilityElement() const { return false; } virtual void FocusChanged(FocusFlags focusFlags) {} virtual void PersistData(PersistStatus status, std::string anonId, PersistMap &storage); @@ -544,6 +548,10 @@ class Clickable : public View { bool Key(const KeyInput &input) override; bool Touch(const TouchInput &input) override; + bool ActivateAccessibility() override { + ClickInternal(); + return true; + } void FocusChanged(FocusFlags focusFlags) override; @@ -612,6 +620,8 @@ class RadioButton : public Clickable { void Draw(UIContext &dc) override; void GetContentDimensions(const UIContext &dc, float &w, float &h) const override; std::string DescribeText() const override; + bool Selected() const { return *value_ == thisButtonValue_; } + std::string_view Text() const { return text_; } private: void ClickInternal() override; @@ -779,6 +789,9 @@ class Choice : public ClickableItem { void SetText(std::string_view text) { text_ = text; } + std::string_view Text() const { + return text_; + } void SetIconOnly(bool iconOnly) { iconOnly_ = iconOnly; } @@ -825,15 +838,32 @@ class StickyChoice : public Choice { bool Key(const KeyInput &key) override; bool Touch(const TouchInput &touch) override; + bool ActivateAccessibility() override { + Press(); + accessibilityActivation_ = true; + ClickInternal(); + return true; + } void FocusChanged(FocusFlags focusFlags) override; void Press() { down_ = true; dragging_ = false; } void Release() { down_ = false; dragging_ = false; } - bool IsDown() { return down_; } + bool IsDown() const { return down_; } + void SetAccessibilityTab(bool tab) { accessibilityTab_ = tab; } + bool IsAccessibilityTab() const { return accessibilityTab_; } + bool ConsumeAccessibilityActivation() { + const bool activated = accessibilityActivation_; + accessibilityActivation_ = false; + return activated; + } protected: // hackery bool IsSticky() const override { return true; } + +private: + bool accessibilityTab_ = false; + bool accessibilityActivation_ = false; }; class InfoItem : public Item { @@ -908,6 +938,7 @@ class ItemHeader : public Item { std::string DescribeText() const override; void GetContentDimensionsBySpec(const UIContext &dc, MeasureSpec horiz, MeasureSpec vert, float &w, float &h) const override; void SetLarge(bool large) { large_ = large; } + std::string_view Text() const { return text_; } private: std::string text_; bool large_ = false; @@ -922,6 +953,7 @@ class PopupHeader : public Item { } void Draw(UIContext &dc) override; std::string DescribeText() const override; + std::string_view Text() const { return text_; } private: std::string text_; @@ -945,6 +977,9 @@ class CheckBox : public ClickableItem { //allow external agents to toggle the checkbox virtual void Toggle(); virtual bool Toggled() const; + std::string AccessibilityText() const { + return smallText_.empty() ? text_ : text_ + "\n" + smallText_; + } // we don't allow these for checkboxes. void SetAutoResult(DialogResult result) override {} diff --git a/Common/UI/ViewGroup.cpp b/Common/UI/ViewGroup.cpp index d39a36f848d3..be16f7f6f1aa 100644 --- a/Common/UI/ViewGroup.cpp +++ b/Common/UI/ViewGroup.cpp @@ -55,6 +55,18 @@ void ViewGroup::Recurse(std::function func) { } } +void ViewGroup::RecurseVisible(std::function func) { + for (View *view : views_) { + if (view->GetVisibility() != V_VISIBLE) { + continue; + } + func(view); + if (ViewGroup *group = dynamic_cast(view)) { + group->RecurseVisible(func); + } + } +} + void ViewGroup::RemoveSubview(View *subView) { // loop counter needed, so can't convert loop. for (size_t i = 0; i < views_.size(); i++) { diff --git a/Common/UI/ViewGroup.h b/Common/UI/ViewGroup.h index 37c54dec9e88..2a23c9b94b30 100644 --- a/Common/UI/ViewGroup.h +++ b/Common/UI/ViewGroup.h @@ -87,6 +87,7 @@ class ViewGroup : public View { std::string DescribeText() const override; void Recurse(std::function func) override; + void RecurseVisible(std::function func); protected: std::string DescribeListUnordered(std::string_view heading) const; diff --git a/Core/Config.cpp b/Core/Config.cpp index 81304f6b0ad3..26b0d461e5c8 100644 --- a/Core/Config.cpp +++ b/Core/Config.cpp @@ -971,6 +971,7 @@ static const ConfigSetting controlSettings[] = { #endif ConfigSetting("ShowTouchControls", SETTING(g_Config, bShowTouchControls), &DefaultShowTouchControls, CfgFlag::PER_GAME), + ConfigSetting("AccessibleTouchControls", SETTING(g_Config, bAccessibleTouchControls), true, CfgFlag::DEFAULT), // ConfigSetting("KeyMapping", SETTING(g_Config, iMappingMap), 0), ConfigSetting("Custom0Mapping", "Custom0Image", "Custom0Shape", "Custom0Toggle", "Custom0Repeat", SETTING_IDX(g_Config, CustomButton, 0), {0, 0, 0, false, false}, CfgFlag::PER_GAME), diff --git a/Core/Config.h b/Core/Config.h index 87f9f4d4d17a..eb78d53447a2 100644 --- a/Core/Config.h +++ b/Core/Config.h @@ -472,6 +472,7 @@ struct Config : public ConfigBlock { // Controls Visibility bool bShowTouchControls = false; + bool bAccessibleTouchControls = true; // Disable diagonals bool bDisableDpadDiagonals; diff --git a/UI/EmuScreen.cpp b/UI/EmuScreen.cpp index 67c25d9dace0..d8b8b0af44cf 100644 --- a/UI/EmuScreen.cpp +++ b/UI/EmuScreen.cpp @@ -32,6 +32,7 @@ using namespace std::placeholders; #include "Common/UI/Tween.h" #include "Common/UI/View.h" #include "Common/UI/AsyncImageFileView.h" +#include "Common/UI/Accessibility.h" #include "Common/VR/PPSSPPVR.h" #include "Common/Data/Text/I18n.h" @@ -1368,6 +1369,19 @@ void EmuScreen::CreateViews() { loadingBG->SetVisibility(V_INVISIBLE); } +void EmuScreen::GetAccessibilityElements(std::vector &elements) { + UIScreen::GetAccessibilityElements(elements); + if (GetUIState() != UISTATE_INGAME || !screenManager() || !screenManager()->getUIContext()) { + return; + } + UI::AccessibilityElementInfo viewport; + viewport.id = 90000; + viewport.label = "Game view"; + viewport.bounds = GetLayoutBounds(*screenManager()->getUIContext()); + viewport.role = UI::AccessibilityRole::Image; + elements.insert(elements.begin(), std::move(viewport)); +} + void EmuScreen::deviceLost() { // If we are currently in the middle of boot, we have to block here! // Otherwise the boot thread will encounter draw_ == nullptr and weird stuff like that. diff --git a/UI/EmuScreen.h b/UI/EmuScreen.h index 60459ca66f6d..8a95ed31bc92 100644 --- a/UI/EmuScreen.h +++ b/UI/EmuScreen.h @@ -51,6 +51,7 @@ class EmuScreen : public UIScreen, protected ControlListener { void sendMessage(UIMessage message, const char *value) override; void resized() override; ScreenRenderRole renderRole(bool isTop) const override; + void GetAccessibilityElements(std::vector &elements) override; // Note: Unlike your average boring UIScreen, here we override the Unsync* functions // to get minimal latency and full control. We forward to UIScreen when needed. diff --git a/UI/GameSettingsScreen.cpp b/UI/GameSettingsScreen.cpp index c789a09a55a6..67e16e35fff2 100644 --- a/UI/GameSettingsScreen.cpp +++ b/UI/GameSettingsScreen.cpp @@ -861,6 +861,9 @@ void GameSettingsScreen::CreateControlsSettings(UI::ViewGroup *controlsSettings) if ((deviceType != DEVICE_TYPE_TV) && (deviceType != DEVICE_TYPE_VR)) { controlsSettings->Add(new ItemHeader(co->T("On-screen touch controls"))); controlsSettings->Add(new CheckBox(&g_Config.bShowTouchControls, co->T("On-screen touch controls"))); +#if PPSSPP_PLATFORM(ANDROID) + controlsSettings->Add(new CheckBox(&g_Config.bAccessibleTouchControls, co->T("Accessibility controller"))); +#endif Choice *layoutEditorChoice = controlsSettings->Add(new Choice(co->T("Edit touch control layout"))); layoutEditorChoice->OnClick.Add([this](UI::EventParams &e) { screenManager()->push(new TouchControlLayoutScreen(gamePath_)); diff --git a/UI/GamepadEmu.cpp b/UI/GamepadEmu.cpp index fd37328a12f6..35fac2f0f17f 100644 --- a/UI/GamepadEmu.cpp +++ b/UI/GamepadEmu.cpp @@ -24,6 +24,7 @@ #include "Common/Render/TextureAtlas.h" #include "Common/Math/math_util.h" #include "Common/UI/Context.h" +#include "Common/UI/Accessibility.h" #include "Common/Log.h" #include "Common/TimeUtil.h" @@ -90,6 +91,27 @@ static u32 GetButtonColor() { return g_Config.iTouchButtonStyle != 0 ? 0xFFFFFF : 0xc0b080; } +static int AccessibilityGamepadId(std::string_view label) { + uint32_t hash = 2166136261u; + for (char c : label) { + hash = (hash ^ (uint8_t)c) * 16777619u; + } + return 100000 + (int)(hash & 0x3fffffff); +} + +static void AddAccessibilityTouchElement(std::vector &elements, std::string label, const Bounds &bounds, float touchX, float touchY, bool longClickable = true) { + UI::AccessibilityElementInfo info; + info.id = AccessibilityGamepadId(label); + info.label = std::move(label); + info.bounds = bounds; + info.role = UI::AccessibilityRole::GamepadControl; + info.clickable = true; + info.longClickable = longClickable; + info.touchX = touchX; + info.touchY = touchY; + elements.push_back(std::move(info)); +} + GamepadComponent::GamepadComponent(std::string_view key, UI::LayoutParams *layoutParams) : UI::View(layoutParams), key_(key) {} static void rotateTouchHelper(float &dx, float &dy) { @@ -115,6 +137,13 @@ std::string GamepadComponent::DescribeText() const { return key_; } +void GamepadComponent::GetAccessibilityElements(std::vector &elements) const { + if (!g_Config.bAccessibleTouchControls) { + return; + } + AddAccessibilityTouchElement(elements, key_, bounds_, bounds_.centerX(), bounds_.centerY()); +} + void MultiTouchButton::GetContentDimensions(const UIContext &dc, float &w, float &h) const { const AtlasImage *image = dc.Draw()->GetAtlas()->getImage(bgImg_); if (image) { @@ -494,6 +523,20 @@ void PSPDpad::Draw(UIContext &dc) { } } +void PSPDpad::GetAccessibilityElements(std::vector &elements) const { + if (!g_Config.bAccessibleTouchControls) { + return; + } + const float x = bounds_.centerX(); + const float y = bounds_.centerY(); + const float r = D_pad_Radius * spacing_; + const float size = std::max(32.0f, r); + AddAccessibilityTouchElement(elements, "D-pad up", Bounds::FromCenter(x, y - r, size * 0.5f), x, y - r); + AddAccessibilityTouchElement(elements, "D-pad down", Bounds::FromCenter(x, y + r, size * 0.5f), x, y + r); + AddAccessibilityTouchElement(elements, "D-pad left", Bounds::FromCenter(x - r, y, size * 0.5f), x - r, y); + AddAccessibilityTouchElement(elements, "D-pad right", Bounds::FromCenter(x + r, y, size * 0.5f), x + r, y); +} + PSPStick::PSPStick(ImageID bgImg, std::string_view key, ImageID stickImg, ImageID stickDownImg, int stick, float scale, UI::LayoutParams *layoutParams) : GamepadComponent(key, layoutParams), bgImg_(bgImg), stickImageIndex_(stickImg), stickDownImg_(stickDownImg), stick_(stick), scale_(scale) { stick_size_ = 50; @@ -539,6 +582,20 @@ void PSPStick::Draw(UIContext &dc) { dc.Draw()->DrawImage(stickImageIndex_, stickX + dx * stick_size_ * scale_, stickY - dy * stick_size_ * scale_, 1.0f * scale_ * headScale, colorBg, ALIGN_CENTER); } +void PSPStick::GetAccessibilityElements(std::vector &elements) const { + if (!g_Config.bAccessibleTouchControls) { + return; + } + const float x = bounds_.centerX(); + const float y = bounds_.centerY(); + const float r = stick_size_ * scale_; + const float size = std::max(32.0f, r); + AddAccessibilityTouchElement(elements, key_ + " up", Bounds::FromCenter(x, y - r, size * 0.5f), x, y - r); + AddAccessibilityTouchElement(elements, key_ + " down", Bounds::FromCenter(x, y + r, size * 0.5f), x, y + r); + AddAccessibilityTouchElement(elements, key_ + " left", Bounds::FromCenter(x - r, y, size * 0.5f), x - r, y); + AddAccessibilityTouchElement(elements, key_ + " right", Bounds::FromCenter(x + r, y, size * 0.5f), x + r, y); +} + bool PSPStick::Touch(const TouchInput &input) { bool retval = GamepadComponent::Touch(input); if (input.flags & TouchInputFlags::RELEASE_ALL) { diff --git a/UI/GamepadEmu.h b/UI/GamepadEmu.h index 6b106f6dae84..d45998ebe2b3 100644 --- a/UI/GamepadEmu.h +++ b/UI/GamepadEmu.h @@ -43,6 +43,8 @@ class GamepadComponent : public UI::View { return false; } std::string DescribeText() const override; + void GetAccessibilityElements(std::vector &elements) const override; + bool SuppressDefaultAccessibilityElement() const override { return true; } virtual bool IsDownByTouch() const { return false; } @@ -120,6 +122,7 @@ class PSPDpad : public GamepadComponent { void Draw(UIContext &dc) override; void GetContentDimensions(const UIContext &dc, float &w, float &h) const override; bool IsDownByTouch() const override { return down_ != 0; } + void GetAccessibilityElements(std::vector &elements) const override; private: void ProcessTouch(float x, float y, bool down, bool ignorePress); @@ -142,6 +145,7 @@ class PSPStick : public GamepadComponent { void Draw(UIContext &dc) override; void GetContentDimensions(const UIContext &dc, float &w, float &h) const override; bool IsDownByTouch() const override { return dragPointerId_ != -1; } + void GetAccessibilityElements(std::vector &elements) const override; protected: int dragPointerId_ = -1; diff --git a/UI/NativeApp.cpp b/UI/NativeApp.cpp index ab3b410d3d1b..0d7160e0d046 100644 --- a/UI/NativeApp.cpp +++ b/UI/NativeApp.cpp @@ -54,6 +54,7 @@ #include "Common/GPU/thin3d.h" #include "Common/UI/UI.h" #include "Common/UI/Screen.h" +#include "Common/UI/Accessibility.h" #include "Common/UI/Context.h" #include "Common/UI/View.h" #include "Common/UI/IconCache.h" @@ -916,6 +917,9 @@ void NativeShutdownGraphics() { if (g_screenManager) { g_screenManager->deviceLost(); } +#if PPSSPP_PLATFORM(IOS) || PPSSPP_PLATFORM(ANDROID) + UI::ClearCachedAccessibilitySnapshot(); +#endif g_iconCache.ClearTextures(); // TODO: This is not really necessary with Vulkan on Android - could keep shaders etc in memory @@ -1084,6 +1088,13 @@ void NativeFrame(GraphicsContext *graphicsContext) { // All actual rendering (and also emulation) happens in here. ScreenRenderFlags renderFlags = g_screenManager->render(); +#if PPSSPP_PLATFORM(IOS) || PPSSPP_PLATFORM(ANDROID) + static double lastAccessibilitySnapshotTime = 0.0; + if (UI::IsAccessibilityEnabled() && startTime - lastAccessibilitySnapshotTime >= 0.5) { + UI::UpdateCachedAccessibilitySnapshot(g_screenManager); + lastAccessibilitySnapshotTime = startTime; + } +#endif if (g_screenManager->getUIContext()->Text()) { g_screenManager->getUIContext()->Text()->OncePerFrame(); } @@ -1263,6 +1274,22 @@ void NativeTouch(const TouchInput &touch) { g_screenManager->touch(touch); } +bool NativeAccessibilityFocus(int id) { + return UI::FocusAccessibilityElement(g_screenManager, id); +} + +bool NativeAccessibilityClick(int id) { + if (!g_screenManager) { + return false; + } + UIScreen *screen = dynamic_cast(g_screenManager->topScreen()); + if (!screen) { + return false; + } + screen->UnsyncAccessibilityActivate(id); + return true; +} + // up, down static double g_wheelReleaseTime[2]{}; diff --git a/android/jni/app-android.cpp b/android/jni/app-android.cpp index 84daae722413..4f18a567baa9 100644 --- a/android/jni/app-android.cpp +++ b/android/jni/app-android.cpp @@ -90,6 +90,7 @@ struct JNIEnv {}; #include "Common/GraphicsContext.h" #include "Common/StringUtils.h" #include "Common/TimeUtil.h" +#include "Common/UI/Accessibility.h" #include "AndroidGraphicsContext.h" #include "AndroidVulkanContext.h" @@ -669,6 +670,95 @@ extern "C" jstring Java_org_ppsspp_ppsspp_NativeApp_queryConfig return jresult; } +static const char *AccessibilityRoleName(UI::AccessibilityRole role) { + switch (role) { + case UI::AccessibilityRole::Button: return "button"; + case UI::AccessibilityRole::Choice: return "choice"; + case UI::AccessibilityRole::Checkbox: return "checkbox"; + case UI::AccessibilityRole::Radio: return "radio"; + case UI::AccessibilityRole::Tab: return "tab"; + case UI::AccessibilityRole::Slider: return "slider"; + case UI::AccessibilityRole::TextField: return "text_field"; + case UI::AccessibilityRole::Progress: return "progress"; + case UI::AccessibilityRole::Heading: return "heading"; + case UI::AccessibilityRole::GamepadControl: return "gamepad_control"; + case UI::AccessibilityRole::Image: return "image"; + case UI::AccessibilityRole::StaticText: + default: + return "text"; + } +} + +static std::string JsonEscape(std::string_view value) { + std::string escaped; + escaped.reserve(value.size() + 8); + for (char c : value) { + switch (c) { + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + case '\n': escaped += "\\n"; break; + case '\r': escaped += "\\r"; break; + case '\t': escaped += "\\t"; break; + default: escaped += c; break; + } + } + return escaped; +} + +extern "C" void Java_org_ppsspp_ppsspp_NativeApp_setAccessibilityEnabled(JNIEnv *, jclass, jboolean enabled) { + UI::SetAccessibilityEnabled(enabled); +} + +extern "C" jlong Java_org_ppsspp_ppsspp_NativeApp_getAccessibilitySnapshotVersion(JNIEnv *, jclass) { + return (jlong)UI::GetCachedAccessibilitySnapshotVersion(); +} + +extern "C" jstring Java_org_ppsspp_ppsspp_NativeApp_getAccessibilitySnapshotJson(JNIEnv *env, jclass) { + const std::vector snapshot = UI::GetCachedAccessibilitySnapshot(); + std::ostringstream json; + json << "{\"version\":" << UI::GetCachedAccessibilitySnapshotVersion() + << ",\"width\":" << g_display.dp_xres + << ",\"height\":" << g_display.dp_yres + << ",\"nodes\":["; + bool first = true; + for (const UI::AccessibilityElementInfo &info : snapshot) { + if (info.role == UI::AccessibilityRole::GamepadControl && !g_Config.bAccessibleTouchControls) { + continue; + } + if (!first) { + json << ','; + } + first = false; + json << "{\"id\":" << info.id + << ",\"label\":\"" << JsonEscape(info.label) << '"' + << ",\"role\":\"" << AccessibilityRoleName(info.role) << '"' + << ",\"left\":" << info.bounds.x + << ",\"top\":" << info.bounds.y + << ",\"right\":" << info.bounds.x2() + << ",\"bottom\":" << info.bounds.y2() + << ",\"enabled\":" << (info.enabled ? "true" : "false") + << ",\"checked\":" << (info.checked ? "true" : "false") + << ",\"selected\":" << (info.selected ? "true" : "false") + << ",\"clickable\":" << (info.clickable ? "true" : "false") + << ",\"longClickable\":" << (info.longClickable ? "true" : "false") + << '}'; + } + json << "]}"; + return env->NewStringUTF(json.str().c_str()); +} + +extern "C" jboolean Java_org_ppsspp_ppsspp_NativeApp_performAccessibilityClick(JNIEnv *, jclass, jint id, jboolean longClick) { + return UI::PerformAccessibilityClick(id, longClick); +} + +extern "C" jboolean Java_org_ppsspp_ppsspp_NativeApp_focusAccessibilityElement(JNIEnv *, jclass, jint id) { + return NativeAccessibilityFocus(id); +} + +extern "C" void Java_org_ppsspp_ppsspp_NativeApp_releaseAccessibilityInputs(JNIEnv *, jclass) { + UI::ReleaseAccessibilityInputs(); +} + static void parse_args(std::vector &args, const std::string value) { // Simple argument parser so we can take args from extra params. const char *p = value.c_str(); diff --git a/android/res/values/accessibility_strings.xml b/android/res/values/accessibility_strings.xml new file mode 100644 index 000000000000..3cd839877c39 --- /dev/null +++ b/android/res/values/accessibility_strings.xml @@ -0,0 +1,6 @@ + + + Release all held controls + All held controls released + %1$s hold toggled + diff --git a/android/src/org/ppsspp/ppsspp/NativeApp.java b/android/src/org/ppsspp/ppsspp/NativeApp.java index d3fe00337910..cb9bdc26cebb 100644 --- a/android/src/org/ppsspp/ppsspp/NativeApp.java +++ b/android/src/org/ppsspp/ppsspp/NativeApp.java @@ -72,6 +72,12 @@ public class NativeApp { public static native void sendMessageFromJava(String msg, String arg); public static native void sendRequestResult(int seqID, boolean result, String value, int iValue); public static native String queryConfig(String queryName); + public static native void setAccessibilityEnabled(boolean enabled); + public static native long getAccessibilitySnapshotVersion(); + public static native String getAccessibilitySnapshotJson(); + public static native boolean performAccessibilityClick(int virtualId, boolean longClick); + public static native boolean focusAccessibilityElement(int virtualId); + public static native void releaseAccessibilityInputs(); public static native int getSelectedCamera(); public static native int getDisplayFramerateMode(); diff --git a/android/src/org/ppsspp/ppsspp/PPSSPPAccessibilityDelegate.java b/android/src/org/ppsspp/ppsspp/PPSSPPAccessibilityDelegate.java new file mode 100644 index 000000000000..a61689732764 --- /dev/null +++ b/android/src/org/ppsspp/ppsspp/PPSSPPAccessibilityDelegate.java @@ -0,0 +1,448 @@ +package org.ppsspp.ppsspp; + +import android.graphics.Rect; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; +import android.view.KeyEvent; +import android.view.MotionEvent; +import android.view.View; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityNodeProvider; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class PPSSPPAccessibilityDelegate extends View.AccessibilityDelegate { + private static final String TAG = "PPSSPPAccessibility"; + static final int ACTION_RELEASE_ALL = 0x01020001; + private static final long REFRESH_INTERVAL_MS = 500; + + private final View host; + private final Handler handler = new Handler(Looper.getMainLooper()); + private final Provider provider = new Provider(); + private final Runnable refreshRunnable = new Runnable() { + @Override + public void run() { + refresh(); + if (enabled) { + handler.postDelayed(this, REFRESH_INTERVAL_MS); + } + } + }; + + private boolean enabled; + private boolean hardwareControllerConnected; + private long lastVersion = -1; + private String lastHeading = ""; + private float nativeWidth = 1.0f; + private float nativeHeight = 1.0f; + private int accessibilityFocusedId = View.NO_ID; + private int hoveredId = View.NO_ID; + private final List nodes = new ArrayList<>(); + private final Map nodesById = new HashMap<>(); + private final List lastNonEmptyNodes = new ArrayList<>(); + + PPSSPPAccessibilityDelegate(View host) { + this.host = host; + host.setImportantForAccessibility(View.IMPORTANT_FOR_ACCESSIBILITY_YES); + host.setOnHoverListener((view, event) -> handleHover(event)); + } + + void setEnabled(boolean enabled) { + if (this.enabled == enabled) { + return; + } + this.enabled = enabled; + handler.removeCallbacks(refreshRunnable); + if (enabled) { + lastVersion = -1; + refresh(); + handler.postDelayed(refreshRunnable, REFRESH_INTERVAL_MS); + } else { + clearNodes(); + } + } + + void setHardwareControllerConnected(boolean connected) { + if (hardwareControllerConnected != connected) { + hardwareControllerConnected = connected; + NativeApp.releaseAccessibilityInputs(); + lastVersion = -1; + refresh(); + } + } + + void resetInputs() { + NativeApp.releaseAccessibilityInputs(); + } + + @Override + public AccessibilityNodeProvider getAccessibilityNodeProvider(View host) { + return provider; + } + + @Override + public void onInitializeAccessibilityNodeInfo(View host, AccessibilityNodeInfo info) { + super.onInitializeAccessibilityNodeInfo(host, info); + info.setClassName(View.class.getName()); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + info.setScreenReaderFocusable(false); + } + for (Node node : nodes) { + info.addChild(host, node.id); + } + info.addAction(new AccessibilityNodeInfo.AccessibilityAction(ACTION_RELEASE_ALL, host.getContext().getString(R.string.accessibility_release_all))); + } + + @Override + public boolean performAccessibilityAction(View host, int action, Bundle args) { + if (action == ACTION_RELEASE_ALL) { + NativeApp.releaseAccessibilityInputs(); + host.announceForAccessibility(host.getContext().getString(R.string.accessibility_all_released)); + return true; + } + return super.performAccessibilityAction(host, action, args); + } + + private void refresh() { + if (!enabled) { + return; + } + long version = NativeApp.getAccessibilitySnapshotVersion(); + if (version == lastVersion) { + return; + } + try { + String snapshotJson = NativeApp.getAccessibilitySnapshotJson(); + JSONObject root = new JSONObject(snapshotJson); + nativeWidth = (float)root.optDouble("width", 1.0); + nativeHeight = (float)root.optDouble("height", 1.0); + Node previousSelectedTab = selectedTab(nodes); + nodes.clear(); + nodesById.clear(); + String heading = ""; + String firstLabel = ""; + JSONArray array = root.getJSONArray("nodes"); + for (int i = 0; i < array.length(); ++i) { + Node node = new Node(array.getJSONObject(i)); + if (hardwareControllerConnected && "gamepad_control".equals(node.role)) { + continue; + } + nodes.add(node); + nodesById.put(node.id, node); + if (firstLabel.isEmpty() && !node.label.isEmpty()) { + firstLabel = node.label; + } + if (heading.isEmpty() && "heading".equals(node.role)) { + heading = node.label; + } + } + lastVersion = version; + if (!nodesById.containsKey(hoveredId)) { + updateHoveredId(View.NO_ID); + } + sendEvent(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED, View.NO_ID); + Node selectedTab = selectedTab(nodes); + if (previousSelectedTab != null && selectedTab != null && !previousSelectedTab.label.equals(selectedTab.label)) { + sendEvent(AccessibilityEvent.TYPE_VIEW_SELECTED, selectedTab.id); + } + if (!heading.isEmpty() && !heading.equals(lastHeading)) { + lastHeading = heading; + host.announceForAccessibility(heading); + } + if (!nodes.isEmpty()) { + if (heading.isEmpty() && isSubstantialTreeReplacement(lastNonEmptyNodes, nodes)) { + accessibilityFocusedId = View.NO_ID; + sendEvent(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED, View.NO_ID); + if (!firstLabel.isEmpty()) { + host.announceForAccessibility(firstLabel); + } + } + lastNonEmptyNodes.clear(); + lastNonEmptyNodes.addAll(nodes); + } + } catch (Exception e) { + Log.e(TAG, "Failed to load accessibility snapshot version=" + version, e); + clearNodes(); + } + } + + private Node selectedTab(List candidates) { + for (Node node : candidates) { + if ("tab".equals(node.role) && node.selected) { + return node; + } + } + return null; + } + + private boolean isSubstantialTreeReplacement(List previous, List current) { + if (previous.isEmpty() || current.isEmpty()) { + return false; + } + int matchingNodes = 0; + for (Node oldNode : previous) { + for (Node newNode : current) { + if (oldNode.label.equals(newNode.label) && oldNode.role.equals(newNode.role)) { + ++matchingNodes; + break; + } + } + } + return matchingNodes * 2 < Math.min(previous.size(), current.size()); + } + + private void clearNodes() { + updateHoveredId(View.NO_ID); + nodes.clear(); + nodesById.clear(); + accessibilityFocusedId = View.NO_ID; + lastVersion = -1; + sendEvent(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED, View.NO_ID); + } + + private void sendEvent(int type, int virtualId) { + if (!host.isAttachedToWindow()) { + return; + } + AccessibilityEvent event = AccessibilityEvent.obtain(type); + event.setPackageName(host.getContext().getPackageName()); + if (virtualId == View.NO_ID) { + event.setSource(host); + } else { + event.setSource(host, virtualId); + Node node = nodesById.get(virtualId); + if (node != null) { + event.setClassName(node.className()); + event.setContentDescription(node.label); + } + } + if (type == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED) { + event.setContentChangeTypes(AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE); + } + if (host.getParent() != null) { + host.getParent().requestSendAccessibilityEvent(host, event); + } + } + + private Rect boundsFor(Node node) { + float sx = host.getWidth() / Math.max(1.0f, nativeWidth); + float sy = host.getHeight() / Math.max(1.0f, nativeHeight); + return new Rect(Math.round(node.left * sx), Math.round(node.top * sy), + Math.round(node.right * sx), Math.round(node.bottom * sy)); + } + + private boolean handleHover(MotionEvent event) { + if (!enabled) { + return false; + } + if (event.getActionMasked() == MotionEvent.ACTION_HOVER_EXIT) { + updateHoveredId(View.NO_ID); + return true; + } + if (event.getActionMasked() != MotionEvent.ACTION_HOVER_ENTER && event.getActionMasked() != MotionEvent.ACTION_HOVER_MOVE) { + return false; + } + int newHoveredId = View.NO_ID; + for (Node node : nodes) { + if (boundsFor(node).contains(Math.round(event.getX()), Math.round(event.getY()))) { + newHoveredId = node.id; + break; + } + } + updateHoveredId(newHoveredId); + return newHoveredId != View.NO_ID; + } + + private void updateHoveredId(int newHoveredId) { + if (hoveredId == newHoveredId) { + return; + } + if (hoveredId != View.NO_ID) { + sendEvent(AccessibilityEvent.TYPE_VIEW_HOVER_EXIT, hoveredId); + } + hoveredId = newHoveredId; + if (hoveredId != View.NO_ID) { + sendEvent(AccessibilityEvent.TYPE_VIEW_HOVER_ENTER, hoveredId); + } + } + + private Rect screenBoundsFor(Node node) { + Rect bounds = boundsFor(node); + int[] location = new int[2]; + host.getLocationOnScreen(location); + bounds.offset(location[0], location[1]); + return bounds; + } + + private void tapKey(int keyCode) { + NativeApp.keyDown(NativeApp.DEVICE_ID_DEFAULT, keyCode, false); + NativeApp.keyUp(NativeApp.DEVICE_ID_DEFAULT, keyCode); + } + + private class Provider extends AccessibilityNodeProvider { + @Override + public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { + if (virtualViewId == View.NO_ID) { + AccessibilityNodeInfo info = AccessibilityNodeInfo.obtain(host); + PPSSPPAccessibilityDelegate.this.onInitializeAccessibilityNodeInfo(host, info); + return info; + } + Node node = nodesById.get(virtualViewId); + if (node == null) { + return null; + } + AccessibilityNodeInfo info = AccessibilityNodeInfo.obtain(); + info.setSource(host, node.id); + info.setParent(host); + info.setPackageName(host.getContext().getPackageName()); + info.setClassName(node.className()); + info.setContentDescription(node.label); + info.setBoundsInParent(boundsFor(node)); + info.setBoundsInScreen(screenBoundsFor(node)); + info.setEnabled(node.enabled); + info.setFocusable(true); + info.setVisibleToUser(true); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + info.setScreenReaderFocusable(true); + } + info.setClickable(node.clickable); + info.setLongClickable(node.longClickable); + if ("checkbox".equals(node.role) || "radio".equals(node.role)) { + info.setCheckable(true); + info.setChecked(node.checked); + } + if ("tab".equals(node.role)) { + info.setSelected(node.selected); + } + if ("heading".equals(node.role)) { + info.setHeading(true); + } + if (node.clickable) { + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK); + } + if (node.longClickable) { + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_LONG_CLICK); + } + if ("slider".equals(node.role)) { + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_FORWARD); + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_BACKWARD); + } + if (node.id == accessibilityFocusedId) { + info.setAccessibilityFocused(true); + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLEAR_ACCESSIBILITY_FOCUS); + } else { + info.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_ACCESSIBILITY_FOCUS); + } + return info; + } + + @Override + public boolean performAction(int virtualViewId, int action, Bundle args) { + Node node = nodesById.get(virtualViewId); + if (node == null) { + return false; + } + if (action == AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS) { + accessibilityFocusedId = virtualViewId; + NativeApp.focusAccessibilityElement(virtualViewId); + sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, virtualViewId); + return true; + } + if (action == AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS) { + accessibilityFocusedId = View.NO_ID; + sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED, virtualViewId); + return true; + } + if (action == AccessibilityNodeInfo.ACTION_CLICK) { + return NativeApp.performAccessibilityClick(virtualViewId, false); + } + if (action == AccessibilityNodeInfo.ACTION_LONG_CLICK) { + boolean handled = NativeApp.performAccessibilityClick(virtualViewId, true); + if (handled) { + host.announceForAccessibility(host.getContext().getString(R.string.accessibility_hold_toggled, node.label)); + } + return handled; + } + if (action == AccessibilityNodeInfo.ACTION_SCROLL_FORWARD) { + tapKey(KeyEvent.KEYCODE_DPAD_RIGHT); + return true; + } + if (action == AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD) { + tapKey(KeyEvent.KEYCODE_DPAD_LEFT); + return true; + } + return false; + } + + @Override + public AccessibilityNodeInfo findFocus(int focus) { + if (focus == AccessibilityNodeInfo.FOCUS_ACCESSIBILITY && accessibilityFocusedId != View.NO_ID) { + return createAccessibilityNodeInfo(accessibilityFocusedId); + } + return null; + } + } + + private static class Node { + final int id; + final String label; + final String role; + final float left; + final float top; + final float right; + final float bottom; + final boolean enabled; + final boolean checked; + final boolean selected; + final boolean clickable; + final boolean longClickable; + + Node(JSONObject json) { + id = json.optInt("id", -1); + label = json.optString("label", ""); + role = json.optString("role", "text"); + left = (float)json.optDouble("left", 0.0); + top = (float)json.optDouble("top", 0.0); + right = (float)json.optDouble("right", 0.0); + bottom = (float)json.optDouble("bottom", 0.0); + enabled = json.optBoolean("enabled", true); + checked = json.optBoolean("checked", false); + selected = json.optBoolean("selected", false); + clickable = json.optBoolean("clickable", false); + longClickable = json.optBoolean("longClickable", false); + } + + String className() { + switch (role) { + case "button": + case "choice": + case "gamepad_control": + return android.widget.Button.class.getName(); + case "checkbox": + return android.widget.CheckBox.class.getName(); + case "radio": + return android.widget.RadioButton.class.getName(); + case "tab": + return "android.app.ActionBar$Tab"; + case "slider": + return android.widget.SeekBar.class.getName(); + case "text_field": + return android.widget.EditText.class.getName(); + case "image": + return android.widget.ImageView.class.getName(); + default: + return android.widget.TextView.class.getName(); + } + } + } +} diff --git a/android/src/org/ppsspp/ppsspp/PpssppActivity.java b/android/src/org/ppsspp/ppsspp/PpssppActivity.java index 55ada7d4245a..50097e0e7f46 100644 --- a/android/src/org/ppsspp/ppsspp/PpssppActivity.java +++ b/android/src/org/ppsspp/ppsspp/PpssppActivity.java @@ -48,6 +48,8 @@ import android.view.View; import android.view.Window; import android.view.WindowManager; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityManager; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputMethodManager; import android.widget.EditText; @@ -115,6 +117,10 @@ public class PpssppActivity extends AppCompatActivity implements SensorEventList private AudioFocusChangeListener audioFocusChangeListener; private AudioManager audioManager; private InputManager.InputDeviceListener inputDeviceListener; + private AccessibilityManager accessibilityManager; + private AccessibilityManager.AccessibilityStateChangeListener accessibilityStateChangeListener; + private AccessibilityManager.TouchExplorationStateChangeListener touchExplorationStateChangeListener; + private PPSSPPAccessibilityDelegate accessibilityDelegate; // This is to avoid losing the game/menu state etc when we are just // switched-away from or rotated etc. @@ -158,6 +164,40 @@ public class PpssppActivity extends AppCompatActivity implements SensorEventList // for the right mouse button. public static boolean useModernMouseEventsB2 = false; + private boolean isTouchExplorationActive() { + return accessibilityManager != null && accessibilityManager.isEnabled() && accessibilityManager.isTouchExplorationEnabled(); + } + + private boolean hasHardwareController() { + for (int deviceId : InputDevice.getDeviceIds()) { + InputDevice device = InputDevice.getDevice(deviceId); + if (device != null && InputDeviceState.inputSourceIsJoystick(device.getSources()) && !device.isVirtual()) { + return true; + } + } + return false; + } + + private void updateAccessibilityState() { + boolean enabled = isTouchExplorationActive(); + NativeApp.setAccessibilityEnabled(enabled); + if (accessibilityDelegate != null) { + accessibilityDelegate.setHardwareControllerConnected(hasHardwareController()); + accessibilityDelegate.setEnabled(enabled); + } + } + + private void setupAccessibility(View surfaceView) { + accessibilityDelegate = new PPSSPPAccessibilityDelegate(surfaceView); + surfaceView.setAccessibilityDelegate(accessibilityDelegate); + accessibilityManager = (AccessibilityManager)getSystemService(Context.ACCESSIBILITY_SERVICE); + accessibilityStateChangeListener = enabled -> updateAccessibilityState(); + touchExplorationStateChangeListener = enabled -> updateAccessibilityState(); + accessibilityManager.addAccessibilityStateChangeListener(accessibilityStateChangeListener); + accessibilityManager.addTouchExplorationStateChangeListener(touchExplorationStateChangeListener); + updateAccessibilityState(); + } + // Functions for the app activity to override to change behaviour. public native void registerCallbacks(); @@ -773,6 +813,7 @@ public void run() { mGLSurfaceView.setRenderer(nativeRenderer); setContentView(mGLSurfaceView); + setupAccessibility(mGLSurfaceView); } else { updateSystemUiVisibility(); @@ -780,6 +821,7 @@ public void run() { sizeManager.setSurfaceView(mSurfaceView); setInsetsListener(mSurfaceView); setContentView(mSurfaceView); + setupAccessibility(mSurfaceView); // render loop thread will be started once we get a surface. } @@ -835,6 +877,7 @@ public void onInputDeviceAdded(int deviceId) { InputDeviceState state = new InputDeviceState(device, true); inputPlayers.add(state); Log.i(TAG, "Input player registered on connect: desc = " + device.getDescriptor()); + updateAccessibilityState(); } @Override @@ -851,6 +894,7 @@ public void onInputDeviceRemoved(int deviceId) { // This is important so the C++ side can clear button states NativeApp.sendMessageFromJava("inputDeviceDisconnectedID", String.valueOf(state.getDeviceId())); inputPlayers.remove(i); + updateAccessibilityState(); break; } } @@ -995,6 +1039,13 @@ void setupSystemUiCallback() { protected void onDestroy() { super.onDestroy(); lifeCycle.onDestroy(); + if (accessibilityManager != null && accessibilityStateChangeListener != null) { + accessibilityManager.removeAccessibilityStateChangeListener(accessibilityStateChangeListener); + } + if (accessibilityManager != null && touchExplorationStateChangeListener != null) { + accessibilityManager.removeTouchExplorationStateChangeListener(touchExplorationStateChangeListener); + } + NativeApp.releaseAccessibilityInputs(); if (javaGL) { nativeRenderer = null; @@ -1053,6 +1104,9 @@ protected void onStop() { protected void onPause() { super.onPause(); lifeCycle.onPause(); + if (accessibilityDelegate != null) { + accessibilityDelegate.resetInputs(); + } InputManager inputManager = (InputManager)getSystemService(Context.INPUT_SERVICE); inputManager.unregisterInputDeviceListener(inputDeviceListener); @@ -1099,6 +1153,7 @@ protected void onResume() { InputManager inputManager = (InputManager)getSystemService(Context.INPUT_SERVICE); inputManager.registerInputDeviceListener(inputDeviceListener, null); + updateAccessibilityState(); if (!javaGL) { // Restart the render loop. @@ -1449,6 +1504,7 @@ public void inputBox(final int requestId, final String title, String defaultText input.setInputType(InputType.TYPE_CLASS_TEXT); input.setImeOptions(EditorInfo.IME_ACTION_DONE); input.setText(defaultText); + input.setHint(title); input.setFocusableInTouchMode(true); input.requestFocus(); input.selectAll(); @@ -1490,6 +1546,10 @@ public void inputBox(final int requestId, final String title, String defaultText try { dlg.show(); input.requestFocus(); + Window dialogWindow = dlg.getWindow(); + if (dialogWindow != null) { + dialogWindow.getDecorView().sendAccessibilityEvent(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED); + } } catch (Exception e) { NativeApp.reportException(e, "AlertDialog"); } @@ -1627,6 +1687,9 @@ public boolean processCommand(String command, String params) { Toast toast = Toast.makeText(this, params, Toast.LENGTH_LONG); toast.show(); Log.i(TAG, params); + if (surfView != null && isTouchExplorationActive()) { + surfView.announceForAccessibility(params); + } return true; } else if (command.equals("showKeyboard") && surfView != null) { InputMethodManager inputMethodManager = (InputMethodManager) getSystemService(Context.INPUT_METHOD_SERVICE); diff --git a/ios/AccessibilityBridge.h b/ios/AccessibilityBridge.h new file mode 100644 index 000000000000..6958c232d51f --- /dev/null +++ b/ios/AccessibilityBridge.h @@ -0,0 +1,18 @@ +#pragma once + +#import + +@class PPSSPPAccessibilityBridge; + +@interface PPSSPPAccessibilityBridge : NSObject + +- (instancetype)initWithView:(UIView *)view; +- (void)scheduleRefresh; +- (void)refresh; +- (void)reset; +- (void)uiStateChanged; +- (void)willResignActive; +- (BOOL)accessibilityPerformEscape; +- (BOOL)accessibilityPerformMagicTap; + +@end diff --git a/ios/AccessibilityBridge.mm b/ios/AccessibilityBridge.mm new file mode 100644 index 000000000000..63b3c4c04bc4 --- /dev/null +++ b/ios/AccessibilityBridge.mm @@ -0,0 +1,497 @@ +#import "ios/AccessibilityBridge.h" + +#include +#include + +#include "Common/Input/InputState.h" +#include "Common/Log.h" +#include "Common/System/Display.h" +#include "Common/System/NativeApp.h" +#include "Common/UI/Accessibility.h" +#include "Core/System.h" + +@class PPSSPPAccessibilityBridge; + +typedef NS_ENUM(NSInteger, PPSSPPAccessibilityAction) { + PPSSPPAccessibilityActionActivateUI, + PPSSPPAccessibilityActionDPad, + PPSSPPAccessibilityActionLeftStick, + PPSSPPAccessibilityActionRightStick, + PPSSPPAccessibilityActionFaceButtons, + PPSSPPAccessibilityActionShoulders, + PPSSPPAccessibilityActionSelect, + PPSSPPAccessibilityActionEmulatorMenu, +}; + +@interface PPSSPPAccessibilityElement : UIAccessibilityElement +@property(nonatomic, weak) PPSSPPAccessibilityBridge *bridge; +@property(nonatomic) PPSSPPAccessibilityAction action; +@property(nonatomic) CGRect dpFrame; +@end + +@interface PPSSPPAccessibilityBridge () { + __weak UIView *_view; + NSMutableArray *_elements; + NSTimer *_refreshTimer; + NSString *_lastSignature; + uint64_t _lastSnapshotVersion; + int _lastUIState; + CGRect _lastViewBounds; + float _lastDPXRes; + float _lastDPYRes; + InputKeyCode _lastShoulderKey; + InputKeyCode _heldShoulderKey; + BOOL _refreshQueued; + BOOL _hasBuiltElements; +} +- (BOOL)activateElement:(PPSSPPAccessibilityElement *)element; +- (BOOL)scrollElement:(PPSSPPAccessibilityElement *)element direction:(UIAccessibilityScrollDirection)direction; +- (void)adjustElement:(PPSSPPAccessibilityElement *)element increment:(BOOL)increment; +- (void)logRefreshWithReason:(NSString *)reason elementCount:(NSUInteger)elementCount snapshotVersion:(uint64_t)snapshotVersion; +@end + +@implementation PPSSPPAccessibilityElement + +- (BOOL)accessibilityActivate { + return [self.bridge activateElement:self]; +} + +- (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction { + return [self.bridge scrollElement:self direction:direction]; +} + +- (void)accessibilityIncrement { + [self.bridge adjustElement:self increment:YES]; +} + +- (void)accessibilityDecrement { + [self.bridge adjustElement:self increment:NO]; +} + +@end + +static void SendKey(InputKeyCode keyCode, bool down, InputDeviceID deviceId = DEVICE_ID_TOUCH) { + KeyInput key{}; + key.deviceId = deviceId; + key.keyCode = keyCode; + key.flags = down ? KeyInputFlags::DOWN : KeyInputFlags::UP; + NativeKey(key); +} + +static void TapKey(InputKeyCode keyCode, InputDeviceID deviceId = DEVICE_ID_TOUCH) { + SendKey(keyCode, true, deviceId); + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(80 * NSEC_PER_MSEC)), dispatch_get_main_queue(), ^{ + SendKey(keyCode, false, deviceId); + }); +} + +static void SendAxis(InputAxis axisId, float value) { + AxisInput axis{}; + axis.deviceId = DEVICE_ID_PAD_0; + axis.axisId = axisId; + axis.value = value; + NativeAxis(&axis, 1); +} + +static void TapAxis(InputAxis axisId, float value) { + SendAxis(axisId, value); + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(80 * NSEC_PER_MSEC)), dispatch_get_main_queue(), ^{ + SendAxis(axisId, 0.0f); + }); +} + +static UIAccessibilityTraits TraitsForRole(UI::AccessibilityRole role) { + switch (role) { + case UI::AccessibilityRole::Button: + case UI::AccessibilityRole::Choice: + case UI::AccessibilityRole::GamepadControl: + return UIAccessibilityTraitButton; + case UI::AccessibilityRole::Checkbox: + return UIAccessibilityTraitButton; + case UI::AccessibilityRole::Slider: + return UIAccessibilityTraitAdjustable; + case UI::AccessibilityRole::TextField: + return UIAccessibilityTraitNone; + case UI::AccessibilityRole::Progress: + return UIAccessibilityTraitUpdatesFrequently; + case UI::AccessibilityRole::Heading: + return UIAccessibilityTraitHeader; + case UI::AccessibilityRole::StaticText: + default: + return UIAccessibilityTraitStaticText; + } +} + +@implementation PPSSPPAccessibilityBridge + +- (instancetype)initWithView:(UIView *)view { + self = [super init]; + if (self) { + _view = view; + _elements = [[NSMutableArray alloc] init]; + _lastSnapshotVersion = 0; + _lastUIState = -1; + _lastViewBounds = CGRectNull; + _lastDPXRes = 0.0f; + _lastDPYRes = 0.0f; + _lastShoulderKey = NKCODE_UNKNOWN; + _heldShoulderKey = NKCODE_UNKNOWN; + view.isAccessibilityElement = NO; + view.accessibilityElements = _elements; + [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(voiceOverStatusChanged:) name:UIAccessibilityVoiceOverStatusDidChangeNotification object:nil]; + _refreshTimer = [NSTimer scheduledTimerWithTimeInterval:1.0 target:self selector:@selector(periodicRefresh:) userInfo:nil repeats:YES]; + } + return self; +} + +- (void)dealloc { + [_refreshTimer invalidate]; + [[NSNotificationCenter defaultCenter] removeObserver:self]; + [self releaseHeldShoulder]; +} + +- (void)voiceOverStatusChanged:(NSNotification *)notification { + [self scheduleRefresh]; +} + +- (void)periodicRefresh:(NSTimer *)timer { + if (UIAccessibilityIsVoiceOverRunning()) { + [self scheduleRefresh]; + } +} + +- (void)scheduleRefresh { + if (_refreshQueued) { + return; + } + _refreshQueued = YES; + dispatch_async(dispatch_get_main_queue(), ^{ + self->_refreshQueued = NO; + [self refresh]; + }); +} + +- (CGRect)uiFrameFromDPBounds:(const Bounds &)bounds { + UIView *view = _view; + if (!view || g_display.dp_xres <= 0 || g_display.dp_yres <= 0) { + return CGRectZero; + } + const CGFloat xScale = view.bounds.size.width / (CGFloat)g_display.dp_xres; + const CGFloat yScale = view.bounds.size.height / (CGFloat)g_display.dp_yres; + CGRect local = CGRectMake(bounds.x * xScale, bounds.y * yScale, bounds.w * xScale, bounds.h * yScale); + return [view.window convertRect:local fromView:view]; +} + +- (PPSSPPAccessibilityElement *)makeElementWithLabel:(NSString *)label + frame:(CGRect)frame + dpFrame:(CGRect)dpFrame + action:(PPSSPPAccessibilityAction)action + traits:(UIAccessibilityTraits)traits { + PPSSPPAccessibilityElement *element = [[PPSSPPAccessibilityElement alloc] initWithAccessibilityContainer:_view]; + element.bridge = self; + element.action = action; + element.accessibilityLabel = label; + element.accessibilityFrame = frame; + element.dpFrame = dpFrame; + element.accessibilityTraits = traits; + return element; +} + +- (void)addInGameControls { + UIView *view = _view; + if (!view || g_display.dp_xres <= 0 || g_display.dp_yres <= 0) { + return; + } + + struct ControlArea { + __unsafe_unretained NSString *label; + PPSSPPAccessibilityAction action; + CGRect dpFrame; + }; + const CGFloat w = (CGFloat)g_display.dp_xres; + const CGFloat h = (CGFloat)g_display.dp_yres; + const CGFloat thirdW = w / 3.0f; + const CGFloat halfH = h / 2.0f; + const ControlArea controls[] = { + { @"D-pad", PPSSPPAccessibilityActionDPad, CGRectMake(0, halfH, thirdW, halfH) }, + { @"Left stick", PPSSPPAccessibilityActionLeftStick, CGRectMake(0, 0, thirdW, halfH) }, + { @"Right stick", PPSSPPAccessibilityActionRightStick, CGRectMake(thirdW * 2.0f, 0, thirdW, halfH) }, + { @"Face buttons", PPSSPPAccessibilityActionFaceButtons, CGRectMake(thirdW * 2.0f, halfH, thirdW, halfH) }, + { @"Shoulder buttons", PPSSPPAccessibilityActionShoulders, CGRectMake(thirdW, 0, thirdW, h * 0.25f) }, + { @"Select", PPSSPPAccessibilityActionSelect, CGRectMake(thirdW, h * 0.72f, thirdW * 0.5f, h * 0.16f) }, + { @"Emulator menu", PPSSPPAccessibilityActionEmulatorMenu, CGRectMake(thirdW * 1.5f, h * 0.72f, thirdW * 0.5f, h * 0.16f) }, + }; + + for (const ControlArea &control : controls) { + Bounds bounds(control.dpFrame.origin.x, control.dpFrame.origin.y, control.dpFrame.size.width, control.dpFrame.size.height); + PPSSPPAccessibilityElement *element = [self makeElementWithLabel:control.label + frame:[self uiFrameFromDPBounds:bounds] + dpFrame:control.dpFrame + action:control.action + traits:UIAccessibilityTraitButton]; + if (control.action == PPSSPPAccessibilityActionDPad || + control.action == PPSSPPAccessibilityActionLeftStick || + control.action == PPSSPPAccessibilityActionRightStick || + control.action == PPSSPPAccessibilityActionFaceButtons || + control.action == PPSSPPAccessibilityActionShoulders) { + element.accessibilityHint = @"Swipe up, down, left, or right."; + } + [_elements addObject:element]; + } +} + +- (void)refresh { + UIView *view = _view; + if (!view) { + return; + } + if (!UIAccessibilityIsVoiceOverRunning()) { + [_elements removeAllObjects]; + _lastSignature = nil; + _hasBuiltElements = NO; + view.accessibilityElements = _elements; + return; + } + const int uiState = (int)GetUIState(); + const CGRect viewBounds = view.bounds; + const bool geometryChanged = !CGRectEqualToRect(_lastViewBounds, viewBounds) || + _lastDPXRes != g_display.dp_xres || _lastDPYRes != g_display.dp_yres; + + if (uiState == UISTATE_INGAME) { + if (_hasBuiltElements && _lastUIState == uiState && !geometryChanged) { + return; + } + NSMutableArray *newElements = [[NSMutableArray alloc] init]; + NSMutableString *signature = [[NSMutableString alloc] init]; + NSMutableArray *oldElements = _elements; + _elements = newElements; + [self addInGameControls]; + [signature appendString:@"ingame"]; + for (PPSSPPAccessibilityElement *element in _elements) { + [signature appendFormat:@"|%@:%@", element.accessibilityLabel, NSStringFromCGRect(element.accessibilityFrame)]; + } + if (![_lastSignature isEqualToString:signature]) { + _lastSignature = [signature copy]; + view.accessibilityElements = _elements; + UIAccessibilityPostNotification(UIAccessibilityLayoutChangedNotification, nil); + [self logRefreshWithReason:@"ingame" elementCount:_elements.count snapshotVersion:_lastSnapshotVersion]; + } else { + _elements = oldElements; + } + _hasBuiltElements = YES; + _lastUIState = uiState; + _lastViewBounds = viewBounds; + _lastDPXRes = g_display.dp_xres; + _lastDPYRes = g_display.dp_yres; + return; + } + + [self releaseHeldShoulder]; + const uint64_t snapshotVersion = UI::GetCachedAccessibilitySnapshotVersion(); + if (_hasBuiltElements && _lastUIState == uiState && !geometryChanged && _lastSnapshotVersion == snapshotVersion) { + return; + } + NSMutableArray *newElements = [[NSMutableArray alloc] init]; + NSMutableString *signature = [[NSMutableString alloc] init]; + NSMutableArray *oldElements = _elements; + _elements = newElements; + if (g_display.dp_xres > 0 && g_display.dp_yres > 0) { + std::vector snapshot = UI::GetCachedAccessibilitySnapshot(); + for (const UI::AccessibilityElementInfo &info : snapshot) { + NSString *label = [NSString stringWithUTF8String:info.label.c_str()]; + CGRect frame = [self uiFrameFromDPBounds:info.bounds]; + CGRect dpFrame = CGRectMake(info.bounds.x, info.bounds.y, info.bounds.w, info.bounds.h); + UIAccessibilityTraits traits = TraitsForRole(info.role); + if (!info.enabled) { + traits |= UIAccessibilityTraitNotEnabled; + } + PPSSPPAccessibilityElement *element = [self makeElementWithLabel:label + frame:frame + dpFrame:dpFrame + action:PPSSPPAccessibilityActionActivateUI + traits:traits]; + [_elements addObject:element]; + [signature appendFormat:@"|%@:%@:%llu", label, NSStringFromCGRect(frame), (unsigned long long)traits]; + } + } + if (![_lastSignature isEqualToString:signature]) { + _lastSignature = [signature copy]; + view.accessibilityElements = _elements; + UIAccessibilityPostNotification(UIAccessibilityLayoutChangedNotification, nil); + [self logRefreshWithReason:@"ui" elementCount:_elements.count snapshotVersion:snapshotVersion]; + } else { + _elements = oldElements; + } + _hasBuiltElements = YES; + _lastSnapshotVersion = snapshotVersion; + _lastUIState = uiState; + _lastViewBounds = viewBounds; + _lastDPXRes = g_display.dp_xres; + _lastDPYRes = g_display.dp_yres; +} + +- (void)logRefreshWithReason:(NSString *)reason elementCount:(NSUInteger)elementCount snapshotVersion:(uint64_t)snapshotVersion { + NSMutableString *labels = [[NSMutableString alloc] init]; + const NSUInteger limit = MIN(elementCount, (NSUInteger)20); + for (NSUInteger i = 0; i < limit; ++i) { + PPSSPPAccessibilityElement *element = [_elements objectAtIndex:i]; + NSString *label = element.accessibilityLabel ?: @""; + if (label.length == 0) { + label = @""; + } + [labels appendFormat:@"%@%@", + i == 0 ? @"" : @", ", + label]; + } + NSLog(@"PPSSPPAccessibility refresh reason=%@ state=%d version=%llu count=%lu labels=[%@]", + reason, _lastUIState, (unsigned long long)snapshotVersion, (unsigned long)elementCount, labels); + NOTICE_LOG(Log::UI, "PPSSPPAccessibility refresh reason=%s state=%d version=%llu count=%lu", + [reason UTF8String], _lastUIState, (unsigned long long)snapshotVersion, (unsigned long)elementCount); +} + +- (void)releaseHeldShoulder { + if (_heldShoulderKey != NKCODE_UNKNOWN) { + SendKey(_heldShoulderKey, false, DEVICE_ID_PAD_0); + _heldShoulderKey = NKCODE_UNKNOWN; + } +} + +- (void)reset { + [self releaseHeldShoulder]; + _lastShoulderKey = NKCODE_UNKNOWN; + [_elements removeAllObjects]; + _lastSignature = nil; + _hasBuiltElements = NO; + if (_view) { + _view.accessibilityElements = _elements; + } +} + +- (void)uiStateChanged { + if (GetUIState() != UISTATE_INGAME) { + [self releaseHeldShoulder]; + } + [self scheduleRefresh]; +} + +- (void)willResignActive { + [self releaseHeldShoulder]; +} + +- (BOOL)activateElement:(PPSSPPAccessibilityElement *)element { + switch (element.action) { + case PPSSPPAccessibilityActionActivateUI: { + const CGFloat x = CGRectGetMidX(element.dpFrame); + const CGFloat y = CGRectGetMidY(element.dpFrame); + TouchInput down{}; + down.x = x; + down.y = y; + down.id = 9; + down.flags = TouchInputFlags::DOWN; + NativeTouch(down); + TouchInput up = down; + up.flags = TouchInputFlags::UP; + NativeTouch(up); + return YES; + } + case PPSSPPAccessibilityActionShoulders: + if (_lastShoulderKey == NKCODE_UNKNOWN) { + return NO; + } + if (_heldShoulderKey == _lastShoulderKey) { + [self releaseHeldShoulder]; + } else { + [self releaseHeldShoulder]; + SendKey(_lastShoulderKey, true, DEVICE_ID_PAD_0); + _heldShoulderKey = _lastShoulderKey; + } + return YES; + case PPSSPPAccessibilityActionSelect: + TapKey(NKCODE_BUTTON_SELECT, DEVICE_ID_PAD_0); + return YES; + case PPSSPPAccessibilityActionEmulatorMenu: + TapKey(NKCODE_BACK); + return YES; + default: + return NO; + } +} + +- (void)adjustElement:(PPSSPPAccessibilityElement *)element increment:(BOOL)increment { + if (element.action != PPSSPPAccessibilityActionActivateUI) { + return; + } + [self activateElement:element]; + TapKey(increment ? NKCODE_DPAD_RIGHT : NKCODE_DPAD_LEFT); +} + +- (BOOL)scrollElement:(PPSSPPAccessibilityElement *)element direction:(UIAccessibilityScrollDirection)direction { + switch (element.action) { + case PPSSPPAccessibilityActionDPad: + switch (direction) { + case UIAccessibilityScrollDirectionLeft: TapKey(NKCODE_DPAD_LEFT, DEVICE_ID_PAD_0); return YES; + case UIAccessibilityScrollDirectionRight: TapKey(NKCODE_DPAD_RIGHT, DEVICE_ID_PAD_0); return YES; + case UIAccessibilityScrollDirectionUp: TapKey(NKCODE_DPAD_UP, DEVICE_ID_PAD_0); return YES; + case UIAccessibilityScrollDirectionDown: TapKey(NKCODE_DPAD_DOWN, DEVICE_ID_PAD_0); return YES; + default: return NO; + } + case PPSSPPAccessibilityActionLeftStick: + switch (direction) { + case UIAccessibilityScrollDirectionLeft: TapAxis(JOYSTICK_AXIS_X, -1.0f); return YES; + case UIAccessibilityScrollDirectionRight: TapAxis(JOYSTICK_AXIS_X, 1.0f); return YES; + case UIAccessibilityScrollDirectionUp: TapAxis(JOYSTICK_AXIS_Y, -1.0f); return YES; + case UIAccessibilityScrollDirectionDown: TapAxis(JOYSTICK_AXIS_Y, 1.0f); return YES; + default: return NO; + } + case PPSSPPAccessibilityActionRightStick: + switch (direction) { + case UIAccessibilityScrollDirectionLeft: TapAxis(JOYSTICK_AXIS_Z, -1.0f); return YES; + case UIAccessibilityScrollDirectionRight: TapAxis(JOYSTICK_AXIS_Z, 1.0f); return YES; + case UIAccessibilityScrollDirectionUp: TapAxis(JOYSTICK_AXIS_RZ, -1.0f); return YES; + case UIAccessibilityScrollDirectionDown: TapAxis(JOYSTICK_AXIS_RZ, 1.0f); return YES; + default: return NO; + } + case PPSSPPAccessibilityActionFaceButtons: + switch (direction) { + case UIAccessibilityScrollDirectionLeft: TapKey(NKCODE_BUTTON_4, DEVICE_ID_PAD_0); return YES; + case UIAccessibilityScrollDirectionRight: TapKey(NKCODE_BUTTON_3, DEVICE_ID_PAD_0); return YES; + case UIAccessibilityScrollDirectionUp: TapKey(NKCODE_BUTTON_1, DEVICE_ID_PAD_0); return YES; + case UIAccessibilityScrollDirectionDown: TapKey(NKCODE_BUTTON_2, DEVICE_ID_PAD_0); return YES; + default: return NO; + } + case PPSSPPAccessibilityActionShoulders: + switch (direction) { + case UIAccessibilityScrollDirectionLeft: + _lastShoulderKey = NKCODE_BUTTON_L1; + if (_heldShoulderKey != NKCODE_BUTTON_L1) { + TapKey(NKCODE_BUTTON_L1, DEVICE_ID_PAD_0); + } + return YES; + case UIAccessibilityScrollDirectionRight: + _lastShoulderKey = NKCODE_BUTTON_R1; + if (_heldShoulderKey != NKCODE_BUTTON_R1) { + TapKey(NKCODE_BUTTON_R1, DEVICE_ID_PAD_0); + } + return YES; + default: + return NO; + } + default: + return NO; + } +} + +- (BOOL)accessibilityPerformEscape { + TapKey(NKCODE_BACK); + return YES; +} + +- (BOOL)accessibilityPerformMagicTap { + if (GetUIState() != UISTATE_INGAME) { + return NO; + } + TapKey(NKCODE_BUTTON_START, DEVICE_ID_PAD_0); + return YES; +} + +@end diff --git a/ios/ViewControllerCommon.mm b/ios/ViewControllerCommon.mm index 4964db869a97..3fb08b4d820f 100644 --- a/ios/ViewControllerCommon.mm +++ b/ios/ViewControllerCommon.mm @@ -1,4 +1,5 @@ #import "ios/CameraHelper.h" +#import "ios/AccessibilityBridge.h" #import "ios/ViewControllerCommon.h" #import "ios/Controls.h" #import "ios/IAPManager.h" @@ -24,6 +25,7 @@ @interface PPSSPPBaseViewController () { @property (strong, nonatomic) NSOperationQueue *accelerometerQueue; @property (nonatomic) GCController *gameController __attribute__((weak_import)); @property (strong, nonatomic) CMMotionManager *motionManager; +@property (strong, nonatomic) PPSSPPAccessibilityBridge *accessibilityBridge; @end @@ -86,6 +88,8 @@ - (id)init { - (void)shutdown { self.gameController = nil; + [self.accessibilityBridge reset]; + self.accessibilityBridge = nil; [[NSNotificationCenter defaultCenter] removeObserver:self]; _dbg_assert_(sharedViewController != nil); @@ -119,6 +123,8 @@ - (void)didBecomeActive { } - (void)willResignActive { + [self.accessibilityBridge willResignActive]; + // Stop accelerometer updates if (self.motionManager.accelerometerActive) { INFO_LOG(Log::G3D, "Stopping accelerometer updates"); @@ -325,6 +331,7 @@ - (void)uiStateChanged { [self setNeedsUpdateOfScreenEdgesDeferringSystemGestures]; [self hideKeyboard]; [self updateGesture]; + [self.accessibilityBridge uiStateChanged]; } - (void)startVideo:(int)width height:(int)height { @@ -366,6 +373,7 @@ - (void)viewDidLoad { [locationHelper setDelegate:self]; self.motionManager = [[CMMotionManager alloc] init]; + self.accessibilityBridge = [[PPSSPPAccessibilityBridge alloc] initWithView:self.view]; } extern float g_safeInsetLeft; @@ -541,9 +549,18 @@ - (void)updateResolutionWithView:(UIView *)view { PSP_CoreParameter().pixelHeight = g_display.pixel_yres; NativeResized(); + [self.accessibilityBridge scheduleRefresh]; NSLog(@"Updated display resolution: (%d, %d) @%.1fx", g_display.pixel_xres, g_display.pixel_yres, (float)scale); } +- (BOOL)accessibilityPerformEscape { + return [self.accessibilityBridge accessibilityPerformEscape]; +} + +- (BOOL)accessibilityPerformMagicTap { + return [self.accessibilityBridge accessibilityPerformMagicTap]; +} + @end