diff --git a/.github/actions/spell-check/allow/code.txt b/.github/actions/spell-check/allow/code.txt index 5efa608b2861..c8a6837a2d3e 100644 --- a/.github/actions/spell-check/allow/code.txt +++ b/.github/actions/spell-check/allow/code.txt @@ -309,6 +309,11 @@ pwa AOT Aot ify +LAF +Laf +languagemodel +philm +phisilica TFM # YML diff --git a/.github/actions/spell-check/patterns.txt b/.github/actions/spell-check/patterns.txt index 43b506d6ce8e..5f8677b892db 100644 --- a/.github/actions/spell-check/patterns.txt +++ b/.github/actions/spell-check/patterns.txt @@ -313,6 +313,9 @@ ms-windows-store://\S+ # ANSI color codes (?:\\(?:u00|x)1[Bb]|\\03[1-7]|\x1b|\\u\{1[Bb]\})\[\d+(?:;\d+)*m +# Phi Silica internal token/ID literals +]*>[^<]+ +\bdjwsxzxb4ksa8\b # Special licenses text from RNNoise (BSD-style disclaimer: ``AS IS'') ``AS IS'' diff --git a/.pipelines/v2/release.yml b/.pipelines/v2/release.yml index 72e16b72c92f..594e3a0c84f9 100644 --- a/.pipelines/v2/release.yml +++ b/.pipelines/v2/release.yml @@ -102,7 +102,7 @@ extends: useManagedIdentity: $(SigningUseManagedIdentity) clientId: $(SigningOriginalClientId) # Have msbuild use the release nuget config profile - additionalBuildOptions: /p:RestoreConfigFile="$(Build.SourcesDirectory)\.pipelines\release-nuget.config" /p:EnableCmdPalAOT=true + additionalBuildOptions: /p:RestoreConfigFile="$(Build.SourcesDirectory)\.pipelines\release-nuget.config" /p:EnableCmdPalAOT=true /p:PhiSilicaLafToken=$(PhiSilicaLafToken) /p:PhiSilicaLafAttestation="$(PhiSilicaLafAttestation)" beforeBuildSteps: # Install the Terrapin retrieval tool, which replaces vcpkg's download handler # to redirect it to a safe Microsoft-controlled location diff --git a/.pipelines/v2/templates/job-build-project.yml b/.pipelines/v2/templates/job-build-project.yml index 8ca7a4ef5045..a54670409ce2 100644 --- a/.pipelines/v2/templates/job-build-project.yml +++ b/.pipelines/v2/templates/job-build-project.yml @@ -266,6 +266,17 @@ jobs: VCWhereExtraVersionTarget: '-prerelease' - ${{ if eq(parameters.official, true) }}: + # M.W.T.V Setup.ps1 sets the pipeline-level XES_APPXMANIFESTVERSION env var + # from the supplied -ProjectDirectory's custom.props. Whichever invocation + # runs last wins. cmdpal MUST run last because $(CmdPalVersion) (used by the + # VNext installer and CmdPal's AppxPackageTestDir) falls back to that env + # var; if AP wins, CmdPal's folder name and MSIX filename disagree on + # VersionMinor and the installer fails with WIX0103. Each project's own + # custom.props is still applied at MSBuild time, so AP versioning is + # unaffected by the order. + - template: .\steps-setup-versioning.yml + parameters: + directory: $(build.sourcesdirectory)\src\modules\AdvancedPaste - template: .\steps-setup-versioning.yml parameters: directory: $(build.sourcesdirectory)\src\modules\cmdpal diff --git a/Directory.Build.props b/Directory.Build.props index 8745c1c311e1..0c0a6752a082 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -3,6 +3,7 @@ $(MSBuildThisFileDirectory) + Copyright (C) Microsoft Corporation. All rights reserved. Copyright (C) Microsoft Corporation. All rights reserved. diff --git a/doc/devdocs/modules/advancedpaste-phisilica-local-testing.md b/doc/devdocs/modules/advancedpaste-phisilica-local-testing.md new file mode 100644 index 000000000000..79e9d7df8024 --- /dev/null +++ b/doc/devdocs/modules/advancedpaste-phisilica-local-testing.md @@ -0,0 +1,100 @@ +# Advanced Paste – Phi Silica local testing + +How to build, register, and test **Phi Silica** in **Advanced Paste (AP)** on a dev machine, +plus the few things that actually break it. + +## How it fits together + +AP ships as an **unpackaged, self-contained WinUI 3 exe** (`PowerToys.AdvancedPaste.exe`). +The Windows AI `LanguageModel` (Phi Silica) API is a **Limited Access Feature (LAF)**. For it +to work, all of these must line up: + +1. **Package identity** — AP runs with identity granted by the sparse MSIX + `Microsoft.PowerToys.SparseApp`. +2. **Matching LAF creds** — the token/attestation baked into the exe match the registered + sparse package's publisher. +3. **AI metadata deployed** — the `Microsoft.Windows.AI*.winmd` files ship next to the exe; + the AI runtime resolves them **at runtime**. +4. **Model ready** — supported hardware and the on-device model downloaded + (`GetReadyState() == Ready`). + +Two identities — the baked token must match the registered package's publisher: + +| Build | Publisher Id | LAF creds | +|-------|--------------|-----------| +| **Dev** | `djwsxzxb4ksa8` | dev default in [`src/PhiSilicaLaf.props`](../../../src/PhiSilicaLaf.props) | +| **Prod** | `8wekyb3d8bbwe` | secret, injected only by `.pipelines/v2/release.yml` | + +Non-secret pairing check: the exe's baked **Attestation** must equal the registered package's +**PublisherId**. + +## Build + register (dev loop) + +```powershell +$repo = "X:\GitHub\PowerToys"; $Plat = "ARM64"; $Cfg = "Debug" # or x64 / Release + +# Build AP only (C#; reuses existing C++ outputs): +dotnet restore "$repo\src\modules\AdvancedPaste\AdvancedPaste\AdvancedPaste.csproj" /p:Platform=$Plat +& "$repo\tools\build\build.cmd" -Path "$repo\src\modules\AdvancedPaste\AdvancedPaste" ` + -Platform $Plat -Configuration $Cfg /p:BuildProjectReferences=false + +# Register the dev sparse package (creates + trusts a dev cert, grants identity): +pwsh -ExecutionPolicy Bypass -File "$repo\src\PackageIdentity\BuildSparsePackage.ps1" ` + -Platform $Plat -Configuration $Cfg -DevRegister +# Expect: PublisherId djwsxzxb4ksa8, IsDevelopmentMode True +``` + +## Check the API + +`PowerToys.AdvancedPaste.exe` is a **GUI-subsystem** app — run directly in a console it prints +nothing and returns no exit code. **Redirect** stdout/stderr and wait: + +```powershell +$exe = "$repo\$Plat\$Cfg\WinUI3Apps\PowerToys.AdvancedPaste.exe" +$o = "$env:TEMP\ap.out"; $e = "$env:TEMP\ap.err" +$p = Start-Process $exe '--check-phi-silica' -Wait -PassThru -WindowStyle Hidden ` + -RedirectStandardOutput $o -RedirectStandardError $e +"exit=$($p.ExitCode) stdout=$((Get-Content $o -Raw).Trim())" +Get-Content $e -Raw # stderr: [phi-silica] LAF unlock status: <…>; ReadyState: <…> +``` + +| `--check-phi-silica` | `--prepare-phi-silica` (downloads the model) | +|----------------------|----------------------------------------------| +| `0` Available · `1` NotReady · `2` NotSupported / unlock failed | `0` Ready · `1` Failed · `2` NotSupported | + +`--check` only reads state; use `--prepare` to trigger the model download (`EnsureReadyAsync`). +On failure it prints the `HRESULT` to stderr. + +Confirm the running AP has identity: + +```powershell +$apPid = (Get-Process PowerToys.AdvancedPaste -EA SilentlyContinue | Select-Object -First 1).Id +if ($apPid) { & "$repo\src\PackageIdentity\Check-ProcessIdentity.ps1" -ProcessId $apPid } +# Expect a PFN ending in the publisher id that matches the baked attestation +``` + +## What actually breaks it + +- **Missing `.winmd` (most important).** The Windows AI runtime resolves + `Microsoft.Windows.AI*.winmd` from the app folder at runtime. If they aren't deployed, + `GetReadyState()` returns `NotReady` and `EnsureReadyAsync()` fails with + `RO_E_METADATA_NAME_NOT_FOUND` (`0x8000000F`) — even though identity, token, and the AI DLLs + are all correct. The build emits these winmd into `WinUI3Apps\`; the **installer must harvest + them** (`*.winmd` is in the inclusion list of + [`generateAllFileComponents.ps1`](../../../installer/PowerToysSetupVNext/generateAllFileComponents.ps1)). + Classic symptom: "works from the build output but not from the installer" → check that the + installed `WinUI3Apps\` contains `Microsoft.Windows.AI*.winmd`. +- **Dev/prod mismatch.** A dev-cred exe running against a prod sparse package (or vice versa) + makes the LAF unlock silently return `Unavailable`. Keep the exe and the registered package + the same flavor, and verify with the attestation == publisherId check above. +- **Forgot to redirect.** `--check-phi-silica` in a console prints nothing — that's the + GUI-subsystem quirk, not a result. + +## Cleanup + +```powershell +pwsh -ExecutionPolicy Bypass -File "$repo\src\PackageIdentity\BuildSparsePackage.ps1" -Unregister +``` + +⚠️ This removes any `Microsoft.PowerToys.SparseApp` registration, **including a prod one** from +an installer — reinstall/repair PowerToys to restore it. diff --git a/doc/devdocs/modules/advancedpaste.md b/doc/devdocs/modules/advancedpaste.md index b2ab24443245..861a370095b6 100644 --- a/doc/devdocs/modules/advancedpaste.md +++ b/doc/devdocs/modules/advancedpaste.md @@ -33,7 +33,81 @@ See the `ExecutePasteFormatAsync(PasteFormat, PasteActionSource)` method in `Opt ## Debugging -TODO: Add debugging information +Advanced Paste is an unpackaged, self-contained WinUI 3 app (`PowerToys.AdvancedPaste.exe`). To call Windows AI APIs (Phi Silica / `Microsoft.Windows.AI.Text.LanguageModel`) it acquires **package identity** at runtime via a shared sparse MSIX package (`Microsoft.PowerToys.SparseApp`). + +### Running and attaching the debugger + +1. Set the **Runner** project (`src/runner`) as the startup project in Visual Studio. +2. Launch the Runner (F5). This starts the PowerToys tray icon and loads all module interfaces. +3. Open Settings (right-click tray icon → Settings) and enable the **Advanced Paste** module if it isn't already. The module launches `PowerToys.AdvancedPaste.exe` in the background immediately. +4. In Visual Studio, go to **Debug → Attach to Process** (`Ctrl+Alt+P`) and attach to `PowerToys.AdvancedPaste.exe` (select **Managed (.NET Core)** debugger). + +Alternatively, use the VS Code launch configuration **"Run AdvancedPaste"** from [.vscode/launch.json](/.vscode/launch.json) to launch the exe directly — but note that without the Runner, IPC and hotkeys won't work. + +### Sparse package identity (local development) + +#### Why is this needed? + +- The `LanguageModel` API requires a Limited Access Feature (LAF) unlock, which only succeeds when the calling process has a matching package identity. +- Advanced Paste is an unpackaged, self-contained WinUI 3 app. The sparse package grants it identity without converting it to a full MSIX. +- The csproj uses `PowerToys.AdvancedPaste.pri` (matching the convention of other WinUI3 apps like ImageResizer). This requires WindowsAppSDK Foundation >= 2.0.22 ([PR #6376](https://github.com/microsoft/WindowsAppSDK/pull/6376)) which fixes MRT PRI lookup under sparse identity so `Application.LoadComponent` resolves custom-named PRI files instead of hard-coding `resources.pri`. + +#### One-step dev setup + +```powershell +pwsh src/PackageIdentity/BuildSparsePackage.ps1 -Platform ARM64 -Configuration Debug -DevRegister +``` + +`-DevRegister`: +1. Generates a dev certificate under `src/PackageIdentity/.user/` (first run only). +2. Auto-imports that certificate into `CurrentUser\TrustedPeople` and `CurrentUser\Root` so the OS grants sparse identity to AP (without trust, `GetPackageFamilyName` returns `APPMODEL_ERROR_NO_PACKAGE` and LAF unlock silently fails). +3. Removes any prior registration. +4. Rewrites the publisher in a temp copy of `AppxManifest.xml` to match the dev cert subject. +5. Registers via `Add-AppxPackage -Register … -ExternalLocation X:\…\\\WinUI3Apps`. + +After registration verify: + +```powershell +$pkg = Get-AppxPackage -Name '*SparseApp*' +$pkg.PackageFamilyName # Microsoft.PowerToys.SparseApp_ +$pkg.PublisherId # djwsxzxb4ksa8 +$pkg.IsDevelopmentMode # True +``` + +Confirm AP picks up sparse identity at runtime: + +```powershell +& 'ARM64\Debug\WinUI3Apps\PowerToys.AdvancedPaste.exe' --check-phi-silica +# Exit 0 = Available, 1 = NotReady, 2 = NotSupported +``` + +Re-register after rebuilding AP, changing `src/PackageIdentity/AppxManifest.xml`, or switching platforms/configurations by re-running the same command. Unregister with `-Unregister`. + +#### Troubleshooting + +| Problem | Cause | Fix | +|---------|-------|-----| +| `GetPackageFamilyName` returns `APPMODEL_ERROR_NO_PACKAGE` (15700) at runtime; LAF unlock returns `Unavailable` | Dev certificate not trusted (or sparse package not registered) | Re-run `BuildSparsePackage.ps1 -DevRegister` — auto-imports the cert into `TrustedPeople` and `Root`. | +| `Microsoft.UI.Xaml.dll` crash with `0xC000027B` (class-not-registered) on AP or Settings startup | `` `Executable` path in `src/PackageIdentity/AppxManifest.xml` does not resolve under the registered `ExternalLocation` (`\WinUI3Apps\`) | Confirm every `Executable` is relative to `WinUI3Apps\` (per #47177) and the file exists under the build output. | +| AP launches but never shows a window when triggered via hotkey | Runner's pipe-server wait timed out before AP's cold-start finished bootstrapping WinAppSDK + DI host | Already mitigated by the 15 s pipe timeout in `AdvancedPasteProcessManager.cpp`; warm-start launches connect in well under 1 s. | +| `XamlParseException` / `ms-appx:///Microsoft.UI.Xaml/Themes/…` not found | WindowsAppSDK Foundation < 2.0.22; MRT can't resolve custom PRI name under sparse identity | Ensure `Microsoft.WindowsAppSDK.Foundation` >= 2.0.22 in `Directory.Packages.props`. | + +### How Settings UI checks Phi Silica availability + +Settings UI does not have sparse package identity. To check whether Phi Silica is available, it launches Advanced Paste as a short-lived subprocess: + +``` +PowerToys.AdvancedPaste.exe --check-phi-silica +``` + +`Program.Main` recognizes this flag, calls `PhiSilicaLafHelper.TryUnlock()` + `LanguageModel.GetReadyState()`, prints one of `Available` / `NotReady` / `NotSupported` to stdout, and exits with the matching code (0/1/2). Settings reads stdout with a 10 s wait. Because each call is a fresh process, transient `Unavailable` results are not cached across checks. + +### See also + +- [Phi Silica local testing & troubleshooting guide](advancedpaste-phisilica-local-testing.md) — layer-by-layer diagnostics for Phi Silica availability +- [`src/PackageIdentity/readme.md`](/src/PackageIdentity/readme.md) — full sparse package documentation +- [microsoft/microsoft-ui-xaml#10856](https://github.com/microsoft/microsoft-ui-xaml/issues/10856) — original WinUI sparse-identity PRI bug +- [microsoft/WindowsAppSDK#6376](https://github.com/microsoft/WindowsAppSDK/pull/6376) — MRT sparse PRI fix (Foundation >= 2.0.22) ## Settings diff --git a/installer/PowerToysSetupVNext/generateAllFileComponents.ps1 b/installer/PowerToysSetupVNext/generateAllFileComponents.ps1 index fcdfdf6b0ef4..403ec0137abf 100644 --- a/installer/PowerToysSetupVNext/generateAllFileComponents.ps1 +++ b/installer/PowerToysSetupVNext/generateAllFileComponents.ps1 @@ -28,12 +28,23 @@ Function Generate-FileList() { $fileExclusionList = @("*.pdb", "*.lastcodeanalysissucceeded", "createdump.exe", "powertoys.exe") - $fileInclusionList = @("*.dll", "*.exe", "*.json", "*.msix", "*.png", "*.gif", "*.ico", "*.cur", "*.svg", "index.html", "reg.js", "gitignore.js", "srt.js", "monacoSpecialLanguages.js", "customTokenThemeRules.js", "*.pri", "*.yml") + # *.winmd: WinRT metadata for the Windows App SDK AI APIs (Phi Silica, Imaging, etc.). The AI + # runtime resolves these from the app directory at runtime, so they must ship with the product. + # Without them GetReadyState() reports NotReady and EnsureReadyAsync() fails with + # RO_E_METADATA_NAME_NOT_FOUND (0x8000000F). The build already emits them into the app output + # (e.g. WinUI3Apps); they were previously dropped here because the harvest didn't include them. + $fileInclusionList = @("*.dll", "*.exe", "*.json", "*.msix", "*.png", "*.gif", "*.ico", "*.cur", "*.svg", "index.html", "reg.js", "gitignore.js", "srt.js", "monacoSpecialLanguages.js", "customTokenThemeRules.js", "*.pri", "*.yml", "*.winmd") # MFC DLLs leak into the output via WindowsAppSDKSelfContained but no PowerToys binary imports them. # Verified with dumpbin /dependents across all 2176 binaries — zero consumers. $fileExclusionList += @("mfc140.dll", "mfc140u.dll", "mfcm140.dll", "mfcm140u.dll") + # Microsoft.CommandPalette.Extensions.winmd already has a dedicated WiX component + # (Microsoft_CommandPalette_Extensions_winmd in BaseApplications.wxs, placed in WinUI3Apps for + # CmdPal's WinRT resolution). Exclude it from the generic *.winmd harvest so it isn't declared + # by two components (WIX ICE30 "installed by two different components" breaks ref-counting). + $fileExclusionList += @("Microsoft.CommandPalette.Extensions.winmd") + $dllsToIgnore = @("System.CodeDom.dll", "WindowsBase.dll") if ($fileDepsJson -eq [string]::Empty) { diff --git a/src/PackageIdentity/AppxManifest.xml b/src/PackageIdentity/AppxManifest.xml index edfa0af5b7d2..4d65e3b6c817 100644 --- a/src/PackageIdentity/AppxManifest.xml +++ b/src/PackageIdentity/AppxManifest.xml @@ -58,6 +58,16 @@ AppListEntry="none"> + + + + + + + + RmToMMYJHZkQSrKP5lWesA== + djwsxzxb4ksa8 + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/IntegrationTestUserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/IntegrationTestUserSettings.cs index 4446e24dde29..c02bb5eb8c22 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/IntegrationTestUserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/IntegrationTestUserSettings.cs @@ -55,6 +55,22 @@ public IntegrationTestUserSettings() public IReadOnlyList AdditionalActions => _additionalActions; + public string FixSpellingAndGrammarPrompt => string.Empty; + + public string FixSpellingAndGrammarSystemPrompt => string.Empty; + + public string FixSpellingAndGrammarProviderId => string.Empty; + + public bool FixSpellingAndGrammarCoachingEnabled => false; + + public bool FixSpellingAndGrammarCoachingShortcutSet => false; + + public string FixSpellingAndGrammarCoachingPrompt => string.Empty; + + public string FixSpellingAndGrammarCoachingSystemPrompt => string.Empty; + + public string FixSpellingAndGrammarCoachingProviderId => string.Empty; + public PasteAIConfiguration PasteAIConfiguration => _configuration; public event EventHandler Changed; diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AdvancedAIProviderResolverTests.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AdvancedAIProviderResolverTests.cs new file mode 100644 index 000000000000..59c902334db9 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AdvancedAIProviderResolverTests.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.ObjectModel; +using AdvancedPaste.Services; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace AdvancedPaste.UnitTests.ServicesTests; + +[TestClass] +public sealed class AdvancedAIProviderResolverTests +{ + [TestMethod] + public void TryResolveAdvancedProvider_WithPhiSilicaOverrideAndAdvancedActiveProvider_ReturnsFalse() + { + var advancedProvider = CreateAdvancedProvider("advanced", AIServiceType.OpenAI); + var phiSilicaProvider = new PasteAIProviderDefinition { Id = "phi", ServiceTypeKind = AIServiceType.PhiSilica }; + var configuration = CreateConfiguration(advancedProvider, advancedProvider, phiSilicaProvider); + + var result = AdvancedAIProviderResolver.TryResolveAdvancedProvider(configuration, phiSilicaProvider.Id, out var provider); + + Assert.IsFalse(result); + Assert.IsNull(provider); + } + + [TestMethod] + public void TryResolveAdvancedProvider_WithNonActiveAdvancedOverride_ReturnsOverride() + { + var activeProvider = CreateAdvancedProvider("active", AIServiceType.OpenAI); + var overrideProvider = CreateAdvancedProvider("override", AIServiceType.AzureOpenAI); + var configuration = CreateConfiguration(activeProvider, activeProvider, overrideProvider); + + var result = AdvancedAIProviderResolver.TryResolveAdvancedProvider(configuration, overrideProvider.Id, out var provider); + + Assert.IsTrue(result); + Assert.AreSame(overrideProvider, provider); + } + + private static PasteAIProviderDefinition CreateAdvancedProvider(string id, AIServiceType serviceType) => + new() + { + Id = id, + ServiceTypeKind = serviceType, + EnableAdvancedAI = true, + }; + + private static PasteAIConfiguration CreateConfiguration(PasteAIProviderDefinition activeProvider, params PasteAIProviderDefinition[] providers) => + new() + { + ActiveProviderId = activeProvider.Id, + Providers = new ObservableCollection(providers), + }; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/CustomActionKernelQueryCacheServiceTests.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/CustomActionKernelQueryCacheServiceTests.cs index b93fda488443..b72b2c2c2222 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/CustomActionKernelQueryCacheServiceTests.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/CustomActionKernelQueryCacheServiceTests.cs @@ -30,6 +30,16 @@ public sealed class CustomActionKernelQueryCacheServiceTests private static readonly CacheValue TestValue = new([new(PasteFormats.PlainText, [])]); private static readonly CacheValue TestValue2 = new([new(PasteFormats.KernelQuery, new() { { "a", "b" }, { "c", "d" } })]); + private static string LocalizeResourceId(string resourceId) => resourceId switch + { + "PasteAsPlainText" => "Paste as plain text", + "PasteAsMarkdown" => MarkdownTestKey.Prompt, + "PasteAsJson" => JSONTestKey.Prompt, + "PasteAsTxtFile" => PasteAsTxtFileKey.Prompt, + "PasteAsPngFile" => PasteAsPngFileKey.Prompt, + _ => resourceId, + }; + private CustomActionKernelQueryCacheService _cacheService; private Mock _userSettings; private MockFileSystem _fileSystem; @@ -41,7 +51,7 @@ public void TestInitialize() UpdateUserActions([], []); _fileSystem = new(); - _cacheService = new(_userSettings.Object, _fileSystem); + _cacheService = new(_userSettings.Object, _fileSystem, LocalizeResourceId); } [TestMethod] @@ -122,7 +132,7 @@ public async Task Test_Cache_Is_Persistent() await _cacheService.WriteAsync(JSONTestKey, TestValue); await _cacheService.WriteAsync(MarkdownTestKey, TestValue2); - _cacheService = new(_userSettings.Object, _fileSystem); // recreate using same mock file-system to simulate app restart + _cacheService = new(_userSettings.Object, _fileSystem, LocalizeResourceId); // recreate using same mock file-system to simulate app restart AssertAreEqual(TestValue, _cacheService.ReadOrNull(JSONTestKey)); AssertAreEqual(TestValue2, _cacheService.ReadOrNull(MarkdownTestKey)); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj index 12f38cd61713..5718a3b19a0a 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj @@ -1,5 +1,6 @@ + @@ -8,7 +9,7 @@ $(RepoRoot)$(Platform)\$(Configuration)\WinUI3Apps true Assets\AdvancedPaste\AdvancedPaste.ico - app.manifest + AdvancedPaste.dev.manifest true false false @@ -20,9 +21,13 @@ AdvancedPaste true true - PowerToys.AdvancedPaste.pri DISABLE_XAML_GENERATED_MAIN,TRACE + $(AdvancedPasteVersion) + + + + AdvancedPaste.prod.manifest @@ -32,6 +37,25 @@ false + + + + $(ApplicationManifest.Replace("$(MSBuildProjectDirectory)\","")) + + + + + + + + $(IntermediateOutputPath)PhiSilicaLafCredentials.g.cs + + + + + + + @@ -66,9 +90,9 @@ + - @@ -85,7 +109,8 @@ - VSTHRD002;VSTHRD110;VSTHRD100;VSTHRD200;VSTHRD101 + + VSTHRD002;VSTHRD110;VSTHRD100;VSTHRD200;VSTHRD101;CS8305 - @@ -154,4 +181,5 @@ PreserveNewest + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.dev.manifest b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.dev.manifest new file mode 100644 index 000000000000..878c9f225504 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.dev.manifest @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + PerMonitorV2 + + + + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.prod.manifest b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.prod.manifest new file mode 100644 index 000000000000..7688bae41a78 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.prod.manifest @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + PerMonitorV2 + + + + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs index 3fa940952ecd..dc46a254d840 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs @@ -188,14 +188,23 @@ private async Task OnAdvancedPasteAdditionalActionHotkey(string[] messageParts) } else { - if (!AdditionalActionIPCKeys.TryGetValue(messageParts[1], out PasteFormats pasteFormat)) + const string coachingSuffix = "-coaching"; + var actionKey = messageParts[1]; + bool forceCoaching = actionKey.EndsWith(coachingSuffix, StringComparison.OrdinalIgnoreCase); + + if (forceCoaching) + { + actionKey = actionKey[..^coachingSuffix.Length]; + } + + if (!AdditionalActionIPCKeys.TryGetValue(actionKey, out PasteFormats pasteFormat)) { Logger.LogWarning($"Unexpected additional action type {messageParts[1]}"); } else { await ShowWindow(); - await viewModel.ExecutePasteFormatAsync(pasteFormat, PasteActionSource.GlobalKeyboardShortcut); + await viewModel.ExecutePasteFormatAsync(pasteFormat, PasteActionSource.GlobalKeyboardShortcut, forceCoaching); } } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml index 6303564d9b92..254edd26c2a8 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml @@ -382,6 +382,7 @@ + - + + + + + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs index d692263dc167..afa958faf83a 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs @@ -25,6 +25,22 @@ public interface IUserSettings public IReadOnlyList AdditionalActions { get; } + public string FixSpellingAndGrammarPrompt { get; } + + public string FixSpellingAndGrammarSystemPrompt { get; } + + public string FixSpellingAndGrammarProviderId { get; } + + public bool FixSpellingAndGrammarCoachingEnabled { get; } + + public bool FixSpellingAndGrammarCoachingShortcutSet { get; } + + public string FixSpellingAndGrammarCoachingPrompt { get; } + + public string FixSpellingAndGrammarCoachingSystemPrompt { get; } + + public string FixSpellingAndGrammarCoachingProviderId { get; } + public PasteAIConfiguration PasteAIConfiguration { get; } public event EventHandler Changed; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/NativeMethods.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/NativeMethods.cs index 08293d4be078..0074164242ff 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/NativeMethods.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/NativeMethods.cs @@ -157,8 +157,6 @@ internal struct PointInter { public int X; public int Y; - - public static explicit operator System.Windows.Point(PointInter point) => new System.Windows.Point(point.X, point.Y); } [DllImport("user32.dll")] diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs index 59f31f0e99c7..4c7622cf74b3 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs @@ -46,6 +46,22 @@ internal sealed partial class UserSettings : IUserSettings, IDisposable public IReadOnlyList CustomActions => _customActions; + public string FixSpellingAndGrammarPrompt { get; private set; } = string.Empty; + + public string FixSpellingAndGrammarSystemPrompt { get; private set; } = string.Empty; + + public string FixSpellingAndGrammarProviderId { get; private set; } = string.Empty; + + public bool FixSpellingAndGrammarCoachingEnabled { get; private set; } + + public bool FixSpellingAndGrammarCoachingShortcutSet { get; private set; } + + public string FixSpellingAndGrammarCoachingPrompt { get; private set; } = string.Empty; + + public string FixSpellingAndGrammarCoachingSystemPrompt { get; private set; } = string.Empty; + + public string FixSpellingAndGrammarCoachingProviderId { get; private set; } = string.Empty; + public PasteAIConfiguration PasteAIConfiguration { get; private set; } public UserSettings(IFileSystem fileSystem) @@ -113,10 +129,21 @@ void UpdateSettings() EnableClipboardPreview = properties.EnableClipboardPreview; PasteAIConfiguration = properties.PasteAIConfiguration ?? new PasteAIConfiguration(); + var fixSpellingAction = properties.AdditionalActions.FixSpellingAndGrammar; + FixSpellingAndGrammarPrompt = fixSpellingAction.Prompt ?? string.Empty; + FixSpellingAndGrammarSystemPrompt = fixSpellingAction.SystemPrompt ?? string.Empty; + FixSpellingAndGrammarProviderId = fixSpellingAction.ProviderId ?? string.Empty; + FixSpellingAndGrammarCoachingEnabled = fixSpellingAction.CoachingEnabled; + FixSpellingAndGrammarCoachingShortcutSet = fixSpellingAction.CoachingShortcut?.Code > 0; + FixSpellingAndGrammarCoachingPrompt = fixSpellingAction.CoachingPrompt ?? string.Empty; + FixSpellingAndGrammarCoachingSystemPrompt = fixSpellingAction.CoachingSystemPrompt ?? string.Empty; + FixSpellingAndGrammarCoachingProviderId = fixSpellingAction.CoachingProviderId ?? string.Empty; + var sourceAdditionalActions = properties.AdditionalActions; (PasteFormats Format, IAdvancedPasteAction[] Actions)[] additionalActionFormats = [ (PasteFormats.ImageToText, [sourceAdditionalActions.ImageToText]), + (PasteFormats.FixSpellingAndGrammar, [sourceAdditionalActions.FixSpellingAndGrammar]), (PasteFormats.PasteAsTxtFile, [sourceAdditionalActions.PasteAsFile, sourceAdditionalActions.PasteAsFile.PasteAsTxtFile]), (PasteFormats.PasteAsPngFile, [sourceAdditionalActions.PasteAsFile, sourceAdditionalActions.PasteAsFile.PasteAsPngFile]), (PasteFormats.PasteAsHtmlFile, [sourceAdditionalActions.PasteAsFile, sourceAdditionalActions.PasteAsFile.PasteAsHtmlFile]), diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs index e1df90897e35..da956ffeed5c 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs @@ -24,20 +24,22 @@ private PasteFormat(PasteFormats format, ClipboardFormat clipboardFormats, bool IsEnabled = SupportsClipboardFormats(clipboardFormats) && (isAIServiceEnabled || !Metadata.RequiresAIService); } - public static PasteFormat CreateStandardFormat(PasteFormats format, ClipboardFormat clipboardFormats, bool isAIServiceEnabled, Func resourceLoader) => + public static PasteFormat CreateStandardFormat(PasteFormats format, ClipboardFormat clipboardFormats, bool isAIServiceEnabled, Func resourceLoader, string providerId = null) => new(format, clipboardFormats, isAIServiceEnabled) { Name = MetadataDict[format].ResourceId == null ? string.Empty : resourceLoader(MetadataDict[format].ResourceId), Prompt = string.Empty, IsSavedQuery = false, + ProviderId = providerId ?? string.Empty, }; - public static PasteFormat CreateCustomAIFormat(PasteFormats format, string name, string prompt, bool isSavedQuery, ClipboardFormat clipboardFormats, bool isAIServiceEnabled) => + public static PasteFormat CreateCustomAIFormat(PasteFormats format, string name, string prompt, bool isSavedQuery, ClipboardFormat clipboardFormats, bool isAIServiceEnabled, string providerId = null) => new(format, clipboardFormats, isAIServiceEnabled) { Name = name, Prompt = prompt, IsSavedQuery = isSavedQuery, + ProviderId = providerId ?? string.Empty, }; public PasteFormatMetadataAttribute Metadata => MetadataDict[Format]; @@ -50,6 +52,8 @@ public static PasteFormat CreateCustomAIFormat(PasteFormats format, string name, public string Prompt { get; private init; } + public string ProviderId { get; private init; } = string.Empty; + public bool IsSavedQuery { get; private init; } public bool IsEnabled { get; private init; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs index 1479912e66f7..8b06128798f4 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs @@ -38,6 +38,17 @@ public enum PasteFormats KernelFunctionDescription = "Takes clipboard text and formats it as JSON text.")] Json, + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "FixSpellingAndGrammar", + IconGlyph = "\uE8E2", + RequiresAIService = true, + CanPreview = true, + SupportedClipboardFormats = ClipboardFormat.Text, + IPCKey = AdvancedPasteAdditionalActions.PropertyNames.FixSpellingAndGrammar, + KernelFunctionDescription = "Fixes all spelling and grammar errors in the clipboard text and returns the corrected version.")] + FixSpellingAndGrammar, + [PasteFormatMetadata( IsCoreAction = false, ResourceId = "ImageToText", diff --git a/src/modules/AdvancedPaste/AdvancedPaste/PhiSilicaLafHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/PhiSilicaLafHelper.cs new file mode 100644 index 000000000000..7655f4dc7d89 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/PhiSilicaLafHelper.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Diagnostics; +using Windows.ApplicationModel; + +namespace AdvancedPaste; + +internal static class PhiSilicaLafHelper +{ + private const string FeatureId = "com.microsoft.windows.ai.languagemodel"; + + private static readonly object _lock = new(); + private static bool _unlocked; + + /// + /// Gets the status of the most recent attempt + /// (e.g. Available, AvailableWithoutToken, Unavailable, or "Exception: ..."). + /// Exposed so callers can surface the real LAF result for diagnostics; the + /// generic "Access is denied" from downstream model calls does not reveal it. + /// + public static string LastUnlockStatus { get; private set; } = "NotAttempted"; + + public static bool TryUnlock() + { + // Only cache a successful unlock. Negative results (Unavailable, Unknown, exceptions) + // are often transient — e.g., AI feature stack not yet initialized after sign-in or + // sparse identity not fully applied to a freshly-started process — and retrying on + // the next call lets AP recover without restart. + if (_unlocked) + { + return true; + } + + lock (_lock) + { + if (_unlocked) + { + return true; + } + + try + { + var access = LimitedAccessFeatures.TryUnlockFeature( + FeatureId, + PhiSilicaLafCredentials.Token, + PhiSilicaLafCredentials.Attestation + " has registered their use of com.microsoft.windows.ai.languagemodel with Microsoft and agrees to the terms of use."); + + _unlocked = access.Status == LimitedAccessFeatureStatus.Available + || access.Status == LimitedAccessFeatureStatus.AvailableWithoutToken; + + LastUnlockStatus = access.Status.ToString(); + Debug.WriteLine($"Phi Silica LAF unlock status: {access.Status}"); + } + catch (Exception ex) + { + LastUnlockStatus = "Exception: " + ex.Message; + Debug.WriteLine($"Phi Silica LAF unlock failed: {ex.Message}"); + _unlocked = false; + } + + return _unlocked; + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Program.cs b/src/modules/AdvancedPaste/AdvancedPaste/Program.cs index ef089f9511ff..d1a77f077119 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Program.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Program.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Linq; using System.Threading; using ManagedCommon; @@ -14,16 +15,26 @@ namespace AdvancedPaste public static class Program { [STAThread] - public static void Main(string[] args) + public static int Main(string[] args) { Logger.InitializeLogger("\\AdvancedPaste\\Logs"); WinRT.ComWrappersSupport.InitializeComWrappers(); + if (args.Contains("--check-phi-silica", StringComparer.OrdinalIgnoreCase)) + { + return CheckPhiSilicaAvailability(); + } + + if (args.Contains("--prepare-phi-silica", StringComparer.OrdinalIgnoreCase)) + { + return PreparePhiSilica(); + } + if (PowerToys.GPOWrapper.GPOWrapper.GetConfiguredAdvancedPasteEnabledValue() == PowerToys.GPOWrapper.GpoRuleConfigured.Disabled) { Logger.LogWarning("Tried to start with a GPO policy setting the utility to always be disabled. Please contact your systems administrator."); - return; + return 1; } var instanceKey = AppInstance.FindOrRegisterForKey("PowerToys_AdvancedPaste_Instance"); @@ -41,6 +52,112 @@ public static void Main(string[] args) { Logger.LogWarning("Another instance of AdvancedPasteUI is running. Exiting."); } + + return 0; + } + + /// + /// Checks Phi Silica availability without starting the WinUI app. + /// Used by Settings UI to probe API status via subprocess. + /// Exit codes: 0 = available, 1 = not ready (model needs download), 2 = not supported or error. + /// + private static int CheckPhiSilicaAvailability() + { + try + { + if (!PhiSilicaLafHelper.TryUnlock()) + { + Console.Error.WriteLine($"[phi-silica] LAF unlock status: {PhiSilicaLafHelper.LastUnlockStatus}"); + Console.Out.WriteLine("NotSupported"); + return 2; + } + + var readyState = Microsoft.Windows.AI.Text.LanguageModel.GetReadyState(); + + Console.Error.WriteLine($"[phi-silica] LAF unlock status: {PhiSilicaLafHelper.LastUnlockStatus}; ReadyState: {readyState}"); + + switch (readyState) + { + case Microsoft.Windows.AI.AIFeatureReadyState.Ready: + Console.Out.WriteLine("Available"); + return 0; + case Microsoft.Windows.AI.AIFeatureReadyState.NotReady: + Console.Out.WriteLine("NotReady"); + return 1; + default: + // NotSupportedOnCurrentSystem, DisabledByUser, CapabilityMissing, + // NotCompatibleWithSystemHardware, OSUpdateNeeded, or any future state: + // the model isn't usable and "Download model" (EnsureReadyAsync) won't fix it. + // CapabilityMissing in particular means the systemAIModels capability isn't + // authorized for the app, so EnsureReadyAsync throws E_ACCESSDENIED (0x80070005). + Console.Out.WriteLine("NotSupported"); + return 2; + } + } + catch (Exception ex) + { + Console.Error.WriteLine(ex.Message); + Console.Out.WriteLine("NotSupported"); + return 2; + } + } + + /// + /// Triggers Phi Silica model preparation (download) without starting the WinUI app. + /// Moves the model from NotReady to Ready by calling EnsureReadyAsync. + /// Exit codes: 0 = ready, 1 = preparation failed, 2 = not supported or error. + /// + private static int PreparePhiSilica() + { + try + { + if (!PhiSilicaLafHelper.TryUnlock()) + { + Console.Error.WriteLine($"[phi-silica] LAF unlock status: {PhiSilicaLafHelper.LastUnlockStatus}"); + Console.Out.WriteLine("NotSupported"); + return 2; + } + + var readyState = Microsoft.Windows.AI.Text.LanguageModel.GetReadyState(); + + Console.Error.WriteLine($"[phi-silica] LAF unlock status: {PhiSilicaLafHelper.LastUnlockStatus}; ReadyState: {readyState}"); + + if (readyState is Microsoft.Windows.AI.AIFeatureReadyState.NotSupportedOnCurrentSystem + or Microsoft.Windows.AI.AIFeatureReadyState.DisabledByUser) + { + Console.Out.WriteLine("NotSupported"); + return 2; + } + + if (readyState == Microsoft.Windows.AI.AIFeatureReadyState.Ready) + { + Console.Out.WriteLine("Ready"); + return 0; + } + + // Run on a thread-pool (MTA) thread: the WinRT async operation does not + // marshal correctly when blocked on from the [STAThread] entry point. + var result = System.Threading.Tasks.Task.Run( + () => Microsoft.Windows.AI.Text.LanguageModel.EnsureReadyAsync().AsTask()).GetAwaiter().GetResult(); + + if (result.Status != Microsoft.Windows.AI.AIFeatureReadyResultState.Success) + { + int hresult = result.ExtendedError?.HResult ?? 0; + Console.Error.WriteLine($"[phi-silica] EnsureReadyAsync Status: {result.Status}; HRESULT: 0x{hresult:X8}; Message: {result.ExtendedError?.Message}"); + Console.Error.WriteLine(result.ExtendedError?.Message ?? result.Status.ToString()); + Console.Out.WriteLine("Failed"); + return 1; + } + + Console.Out.WriteLine("Ready"); + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine(ex.Message); + Console.Out.WriteLine("NotSupported"); + return 2; + } } } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIKernelService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIKernelService.cs index c886bcef43d3..ae9199e41508 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIKernelService.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIKernelService.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using System; -using System.Linq; using AdvancedPaste.Helpers; using AdvancedPaste.Models; using AdvancedPaste.Services.CustomActions; @@ -18,6 +17,7 @@ namespace AdvancedPaste.Services; public sealed class AdvancedAIKernelService : KernelServiceBase { private sealed record RuntimeConfiguration( + string ProviderId, AIServiceType ServiceType, string ModelName, string Endpoint, @@ -41,23 +41,18 @@ public AdvancedAIKernelService( this.credentialsProvider = credentialsProvider; } - protected override string AdvancedAIModelName => GetRuntimeConfiguration().ModelName; - - protected override PromptExecutionSettings PromptExecutionSettings => CreatePromptExecutionSettings(); - - protected override void AddChatCompletionService(IKernelBuilder kernelBuilder) + protected override void AddChatCompletionService(IKernelBuilder kernelBuilder, IKernelRuntimeConfiguration runtimeConfig) { ArgumentNullException.ThrowIfNull(kernelBuilder); + ArgumentNullException.ThrowIfNull(runtimeConfig); - var runtimeConfig = GetRuntimeConfiguration(); var serviceType = runtimeConfig.ServiceType; var modelName = runtimeConfig.ModelName; var requiresApiKey = RequiresApiKey(serviceType); var apiKey = string.Empty; if (requiresApiKey) { - this.credentialsProvider.Refresh(); - apiKey = (this.credentialsProvider.GetKey() ?? string.Empty).Trim(); + apiKey = (this.credentialsProvider.GetKey(serviceType, runtimeConfig.ProviderId) ?? string.Empty).Trim(); if (string.IsNullOrWhiteSpace(apiKey)) { throw new InvalidOperationException($"An API key is required for {serviceType} but none was found in the credential vault."); @@ -85,13 +80,8 @@ protected override AIServiceUsage GetAIServiceUsage(ChatMessageContent chatMessa return AIServiceUsageHelper.GetOpenAIServiceUsage(chatMessage); } - protected override bool ShouldModerateAdvancedAI() + protected override bool ShouldModerateAdvancedAI(IKernelRuntimeConfiguration runtimeConfig) { - if (!TryGetRuntimeConfiguration(out var runtimeConfig)) - { - return false; - } - return runtimeConfig.ModerationEnabled && (runtimeConfig.ServiceType == AIServiceType.OpenAI || runtimeConfig.ServiceType == AIServiceType.AzureOpenAI); } @@ -105,9 +95,9 @@ private static string GetModelName(PasteAIProviderDefinition config) return "gpt-4o"; } - protected override IKernelRuntimeConfiguration GetRuntimeConfiguration() + protected override IKernelRuntimeConfiguration GetRuntimeConfiguration(string providerIdOverride) { - if (TryGetRuntimeConfiguration(out var runtimeConfig)) + if (TryGetRuntimeConfiguration(providerIdOverride, out var runtimeConfig)) { return runtimeConfig; } @@ -115,11 +105,11 @@ protected override IKernelRuntimeConfiguration GetRuntimeConfiguration() throw new InvalidOperationException("No Advanced AI provider is configured."); } - private bool TryGetRuntimeConfiguration(out IKernelRuntimeConfiguration runtimeConfig) + private bool TryGetRuntimeConfiguration(string providerIdOverride, out IKernelRuntimeConfiguration runtimeConfig) { runtimeConfig = null; - if (!TryResolveAdvancedProvider(out var provider)) + if (!AdvancedAIProviderResolver.TryResolveAdvancedProvider(this.UserSettings?.PasteAIConfiguration, providerIdOverride, out var provider)) { return false; } @@ -131,6 +121,7 @@ private bool TryGetRuntimeConfiguration(out IKernelRuntimeConfiguration runtimeC } runtimeConfig = new RuntimeConfiguration( + provider.Id, serviceType, GetModelName(provider), provider.EndpointUrl, @@ -141,49 +132,6 @@ private bool TryGetRuntimeConfiguration(out IKernelRuntimeConfiguration runtimeC return true; } - private bool TryResolveAdvancedProvider(out PasteAIProviderDefinition provider) - { - provider = null; - - var configuration = this.UserSettings?.PasteAIConfiguration; - if (configuration is null) - { - return false; - } - - var activeProvider = configuration.ActiveProvider; - if (IsAdvancedProvider(activeProvider)) - { - provider = activeProvider; - return true; - } - - if (activeProvider is not null) - { - return false; - } - - var fallback = configuration.Providers?.FirstOrDefault(IsAdvancedProvider); - if (fallback is not null) - { - provider = fallback; - return true; - } - - return false; - } - - private static bool IsAdvancedProvider(PasteAIProviderDefinition provider) - { - if (provider is null || !provider.EnableAdvancedAI) - { - return false; - } - - var serviceType = NormalizeServiceType(provider.ServiceTypeKind); - return IsServiceTypeSupported(serviceType); - } - private static bool IsServiceTypeSupported(AIServiceType serviceType) { return serviceType is AIServiceType.OpenAI or AIServiceType.AzureOpenAI; @@ -209,9 +157,8 @@ private static string RequireEndpoint(string endpoint, AIServiceType serviceType throw new InvalidOperationException($"Endpoint is required for {serviceType} configuration but was not provided."); } - private PromptExecutionSettings CreatePromptExecutionSettings() + protected override PromptExecutionSettings GetPromptExecutionSettings(IKernelRuntimeConfiguration runtimeConfig) { - var serviceType = GetRuntimeConfiguration().ServiceType; return new OpenAIPromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(), diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIProviderResolver.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIProviderResolver.cs new file mode 100644 index 000000000000..af6d1d139bac --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIProviderResolver.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; +using Microsoft.PowerToys.Settings.UI.Library; + +namespace AdvancedPaste.Services; + +internal static class AdvancedAIProviderResolver +{ + public static bool TryResolveAdvancedProvider(PasteAIConfiguration configuration, string providerIdOverride, out PasteAIProviderDefinition provider) + { + provider = null; + + if (configuration is null) + { + return false; + } + + if (!string.IsNullOrWhiteSpace(providerIdOverride)) + { + var configuredProvider = configuration.Providers?.FirstOrDefault(candidate => string.Equals(candidate.Id, providerIdOverride, StringComparison.OrdinalIgnoreCase)); + if (configuredProvider is not null) + { + if (!IsAdvancedProvider(configuredProvider)) + { + return false; + } + + provider = configuredProvider; + return true; + } + } + + var activeProvider = configuration.ActiveProvider; + if (IsAdvancedProvider(activeProvider)) + { + provider = activeProvider; + return true; + } + + if (activeProvider is not null) + { + return false; + } + + provider = configuration.Providers?.FirstOrDefault(IsAdvancedProvider); + return provider is not null; + } + + private static bool IsAdvancedProvider(PasteAIProviderDefinition provider) + { + if (provider is null || !provider.EnableAdvancedAI) + { + return false; + } + + var serviceType = provider.ServiceTypeKind == AIServiceType.Unknown ? AIServiceType.OpenAI : provider.ServiceTypeKind; + return serviceType is AIServiceType.OpenAI or AIServiceType.AzureOpenAI; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActionKernelQueryCacheService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActionKernelQueryCacheService.cs index f7d888cf10f5..3c06bb56155f 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActionKernelQueryCacheService.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActionKernelQueryCacheService.cs @@ -33,14 +33,21 @@ public sealed class CustomActionKernelQueryCacheService : IKernelQueryCacheServi private readonly IUserSettings _userSettings; private readonly IFileSystem _fileSystem; private readonly SettingsUtils _settingsUtil; + private readonly Func _getLocalizedString; private static string Version => Assembly.GetExecutingAssembly()?.GetName()?.Version?.ToString() ?? string.Empty; public CustomActionKernelQueryCacheService(IUserSettings userSettings, IFileSystem fileSystem) + : this(userSettings, fileSystem, ResourceLoaderInstance.ResourceLoader.GetString) + { + } + + internal CustomActionKernelQueryCacheService(IUserSettings userSettings, IFileSystem fileSystem, Func getLocalizedString) { _userSettings = userSettings; _fileSystem = fileSystem; _settingsUtil = new SettingsUtils(fileSystem); + _getLocalizedString = getLocalizedString; _userSettings.Changed += OnUserSettingsChanged; @@ -112,7 +119,7 @@ private void UpdateCacheablePrompts() let metadata = pair.Value where !string.IsNullOrEmpty(metadata.ResourceId) where metadata.IsCoreAction || _userSettings.AdditionalActions.Contains(format) - select ResourceLoaderInstance.ResourceLoader.GetString(metadata.ResourceId); + select _getLocalizedString(metadata.ResourceId); var customActionPrompts = from customAction in _userSettings.CustomActions select customAction.Prompt; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs index 05cdcbe81fa0..3cefc56b4c81 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs @@ -40,10 +40,15 @@ public CustomActionTransformService(IPromptModerationService promptModerationSer this.userSettings = userSettings; } - public async Task TransformAsync(string prompt, string inputText, byte[] imageBytes, CancellationToken cancellationToken, IProgress progress) + public async Task TransformAsync(string prompt, string inputText, byte[] imageBytes, CancellationToken cancellationToken, IProgress progress, string systemPromptOverride = null, string providerIdOverride = null) { var pasteConfig = userSettings?.PasteAIConfiguration; - var providerConfig = BuildProviderConfig(pasteConfig); + var providerConfig = BuildProviderConfig(pasteConfig, providerIdOverride); + + if (systemPromptOverride != null) + { + providerConfig.SystemPrompt = systemPromptOverride; + } return await TransformAsync(prompt, inputText, imageBytes, providerConfig, cancellationToken, progress); } @@ -148,13 +153,26 @@ private static AIServiceType NormalizeServiceType(AIServiceType serviceType) return serviceType == AIServiceType.Unknown ? AIServiceType.OpenAI : serviceType; } - private PasteAIConfig BuildProviderConfig(PasteAIConfiguration config) + private PasteAIConfig BuildProviderConfig(PasteAIConfiguration config, string providerIdOverride = null) { config ??= new PasteAIConfiguration(); - var provider = config.ActiveProvider ?? config.Providers?.FirstOrDefault() ?? new PasteAIProviderDefinition(); + PasteAIProviderDefinition provider; + + if (!string.IsNullOrWhiteSpace(providerIdOverride)) + { + provider = config.Providers?.FirstOrDefault(p => string.Equals(p.Id, providerIdOverride, StringComparison.OrdinalIgnoreCase)) + ?? config.ActiveProvider + ?? config.Providers?.FirstOrDefault() + ?? new PasteAIProviderDefinition(); + } + else + { + provider = config.ActiveProvider ?? config.Providers?.FirstOrDefault() ?? new PasteAIProviderDefinition(); + } + var serviceType = NormalizeServiceType(provider.ServiceTypeKind); var systemPrompt = string.IsNullOrWhiteSpace(provider.SystemPrompt) ? DefaultSystemPrompt : provider.SystemPrompt; - var apiKey = AcquireApiKey(serviceType); + var apiKey = AcquireApiKey(serviceType, provider.Id); var modelName = provider.ModelName; var providerConfig = new PasteAIConfig @@ -173,15 +191,14 @@ private PasteAIConfig BuildProviderConfig(PasteAIConfiguration config) return providerConfig; } - private string AcquireApiKey(AIServiceType serviceType) + private string AcquireApiKey(AIServiceType serviceType, string providerId) { if (!RequiresApiKey(serviceType)) { return string.Empty; } - credentialsProvider.Refresh(); - return credentialsProvider.GetKey() ?? string.Empty; + return credentialsProvider.GetKey(serviceType, providerId ?? string.Empty); } private static bool RequiresApiKey(AIServiceType serviceType) @@ -190,6 +207,8 @@ private static bool RequiresApiKey(AIServiceType serviceType) { AIServiceType.Onnx => false, AIServiceType.Ollama => false, + AIServiceType.FoundryLocal => false, + AIServiceType.PhiSilica => false, _ => true, }; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/ICustomActionTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/ICustomActionTransformService.cs index 564db3fdc56e..361d96d4c7a3 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/ICustomActionTransformService.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/ICustomActionTransformService.cs @@ -12,6 +12,6 @@ namespace AdvancedPaste.Services.CustomActions { public interface ICustomActionTransformService { - Task TransformAsync(string prompt, string inputText, byte[] imageBytes, CancellationToken cancellationToken, IProgress progress); + Task TransformAsync(string prompt, string inputText, byte[] imageBytes, CancellationToken cancellationToken, IProgress progress, string systemPromptOverride = null, string providerIdOverride = null); } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderFactory.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderFactory.cs index 7339b4e4e38f..4f7e02fdc303 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderFactory.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderFactory.cs @@ -15,6 +15,7 @@ public sealed class PasteAIProviderFactory : IPasteAIProviderFactory SemanticKernelPasteProvider.Registration, LocalModelPasteProvider.Registration, FoundryLocalPasteProvider.Registration, + PhiSilicaPasteProvider.Registration, }; private static readonly IReadOnlyDictionary> ProviderFactories = CreateProviderFactories(); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PhiSilicaPasteProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PhiSilicaPasteProvider.cs new file mode 100644 index 000000000000..6310319bcf56 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PhiSilicaPasteProvider.cs @@ -0,0 +1,219 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AdvancedPaste.Models; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.Windows.AI; +using Microsoft.Windows.AI.ContentSafety; +using Microsoft.Windows.AI.Text; +using PhiSilicaLanguageModel = Microsoft.Windows.AI.Text.LanguageModel; + +namespace AdvancedPaste.Services.CustomActions; + +public sealed class PhiSilicaPasteProvider : IPasteAIProvider +{ + private static readonly IReadOnlyCollection SupportedTypes = new[] + { + AIServiceType.PhiSilica, + }; + + public static PasteAIProviderRegistration Registration { get; } = new(SupportedTypes, config => new PhiSilicaPasteProvider(config)); + + private static readonly SemaphoreSlim _initLock = new(1, 1); + private static PhiSilicaLanguageModel _cachedModel; + + private readonly PasteAIConfig _config; + + public PhiSilicaPasteProvider(PasteAIConfig config) + { + ArgumentNullException.ThrowIfNull(config); + _config = config; + } + + public Task IsAvailableAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + if (!PhiSilicaLafHelper.TryUnlock()) + { + return Task.FromResult(false); + } + + var readyState = PhiSilicaLanguageModel.GetReadyState(); + return Task.FromResult(readyState is not (AIFeatureReadyState.NotSupportedOnCurrentSystem or AIFeatureReadyState.DisabledByUser)); + } + catch (Exception) + { + return Task.FromResult(false); + } + } + + public async Task ProcessPasteAsync(PasteAIRequest request, CancellationToken cancellationToken, IProgress progress) + { + ArgumentNullException.ThrowIfNull(request); + + try + { + var systemPrompt = request.SystemPrompt; + if (string.IsNullOrWhiteSpace(systemPrompt)) + { + throw new PasteActionException( + "System prompt is required for Phi Silica", + new ArgumentException("System prompt must be provided", nameof(request))); + } + + var prompt = request.Prompt; + var inputText = request.InputText; + if (string.IsNullOrWhiteSpace(prompt) || string.IsNullOrWhiteSpace(inputText)) + { + throw new PasteActionException( + "Prompt and input text are required", + new ArgumentException("Prompt and input text must be provided", nameof(request))); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var languageModel = await GetOrCreateModelAsync(cancellationToken).ConfigureAwait(false); + + progress?.Report(0.1); + + var contentFilterOptions = new ContentFilterOptions(); + using var context = languageModel.CreateContext(systemPrompt, contentFilterOptions); + + var userPrompt = $""" + User instructions: + {prompt} + + Text: + {inputText} + + Output: + """; + + if ((ulong)userPrompt.Length > languageModel.GetUsablePromptLength(context, userPrompt)) + { + throw new PasteActionException( + "Prompt is too large for the Phi Silica model context", + new InvalidOperationException("Prompt exceeds usable prompt length"), + aiServiceMessage: "The input text is too large for on-device processing. Try with shorter text."); + } + + var options = new LanguageModelOptions + { + ContentFilterOptions = contentFilterOptions, + }; + + var result = await languageModel.GenerateResponseAsync(context, userPrompt, options).AsTask(cancellationToken).ConfigureAwait(false); + + progress?.Report(0.8); + + if (result.Status != LanguageModelResponseStatus.Complete) + { + var statusMessage = result.Status switch + { + LanguageModelResponseStatus.BlockedByPolicy => "Response was blocked by policy.", + LanguageModelResponseStatus.PromptBlockedByContentModeration => "Prompt was blocked by content moderation.", + LanguageModelResponseStatus.ResponseBlockedByContentModeration => "Response was blocked by content moderation.", + LanguageModelResponseStatus.PromptLargerThanContext => "Prompt is too large for the model context.", + _ => $"Unexpected status: {result.Status}", + }; + + throw new PasteActionException( + $"Phi Silica returned status: {result.Status}", + new InvalidOperationException($"LanguageModel response status: {result.Status}"), + aiServiceMessage: statusMessage); + } + + var responseText = result.Text ?? string.Empty; + request.Usage = AIServiceUsage.None; + + progress?.Report(1.0); + + return responseText; + } + catch (OperationCanceledException) + { + throw; + } + catch (PasteActionException) + { + throw; + } + catch (Exception ex) + { + throw new PasteActionException( + "Failed to generate response using Phi Silica", + ex, + aiServiceMessage: $"Error details: {ex.Message}"); + } + } + + private static async Task GetOrCreateModelAsync(CancellationToken cancellationToken) + { + if (_cachedModel is not null) + { + return _cachedModel; + } + + await _initLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_cachedModel is not null) + { + return _cachedModel; + } + + if (!PhiSilicaLafHelper.TryUnlock()) + { + throw new PasteActionException( + "Phi Silica access is unavailable", + new InvalidOperationException($"Phi Silica LAF unlock failed: {PhiSilicaLafHelper.LastUnlockStatus}"), + aiServiceMessage: "Phi Silica access is unavailable on this device."); + } + + var readyState = PhiSilicaLanguageModel.GetReadyState(); + + if (readyState is AIFeatureReadyState.NotSupportedOnCurrentSystem or AIFeatureReadyState.DisabledByUser) + { + throw new PasteActionException( + "Phi Silica is not supported on this device. A Copilot+ PC is required.", + new InvalidOperationException("Phi Silica requires a Copilot+ PC with an NPU."), + aiServiceMessage: "Phi Silica requires a Copilot+ PC with an NPU. For on-device AI on any Windows PC, consider using Foundry Local."); + } + + if (readyState is AIFeatureReadyState.NotReady) + { + var ensureResult = await PhiSilicaLanguageModel.EnsureReadyAsync().AsTask(cancellationToken).ConfigureAwait(false); + if (ensureResult.Status != AIFeatureReadyResultState.Success) + { + throw new PasteActionException( + "Failed to prepare Phi Silica model", + ensureResult.ExtendedError, + aiServiceMessage: $"Model preparation failed (status: {ensureResult.Status})"); + } + } + + if (PhiSilicaLanguageModel.GetReadyState() is not AIFeatureReadyState.Ready) + { + throw new PasteActionException( + "Phi Silica model is not ready", + new InvalidOperationException("Phi Silica model is not in Ready state after preparation."), + aiServiceMessage: "Phi Silica model is not available. Please ensure the model is downloaded and ready."); + } + + _cachedModel = await PhiSilicaLanguageModel.CreateAsync().AsTask(cancellationToken).ConfigureAwait(false); + return _cachedModel; + } + finally + { + _initLock.Release(); + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/SemanticKernelPasteProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/SemanticKernelPasteProvider.cs index 636d2e3e78b3..90c8d58b1bcc 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/SemanticKernelPasteProvider.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/SemanticKernelPasteProvider.cs @@ -175,6 +175,7 @@ private PromptExecutionSettings CreateExecutionSettings() AIServiceType.OpenAI or AIServiceType.AzureOpenAI => new OpenAIPromptExecutionSettings { FunctionChoiceBehavior = null, + ReasoningEffort = "minimal", }, _ => new PromptExecutionSettings(), }; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/EnhancedVaultCredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/EnhancedVaultCredentialsProvider.cs index 648881fba04b..27bf71092842 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/EnhancedVaultCredentialsProvider.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/EnhancedVaultCredentialsProvider.cs @@ -55,6 +55,13 @@ public bool IsConfigured() return !string.IsNullOrEmpty(GetKey()); } + public string GetKey(AIServiceType serviceType, string providerId) + { + var normalizedType = NormalizeServiceType(serviceType); + var entry = BuildCredentialEntry(normalizedType, providerId ?? string.Empty); + return LoadKey(entry); + } + public bool Refresh() { using (_syncRoot.EnterScope()) @@ -121,6 +128,7 @@ private static string LoadKey((string Resource, string Username)? entry) try { var credential = new PasswordVault().Retrieve(entry.Value.Resource, entry.Value.Username); + credential?.RetrievePassword(); return credential?.Password ?? string.Empty; } catch (Exception) @@ -160,6 +168,7 @@ private static (string Resource, string Username)? BuildCredentialEntry(AIServic case AIServiceType.ML: case AIServiceType.Onnx: case AIServiceType.Ollama: + case AIServiceType.PhiSilica: return null; default: return null; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs index 7aa6f63b198a..db9739697b43 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs @@ -2,6 +2,8 @@ // The Microsoft Corporation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.PowerToys.Settings.UI.Library; + namespace AdvancedPaste.Services; /// @@ -21,6 +23,14 @@ public interface IAICredentialsProvider /// Credential string or when missing. string GetKey(); + /// + /// Retrieves the credential for a specific AI provider. + /// + /// The AI service type. + /// The provider identifier. + /// Credential string or when missing. + string GetKey(AIServiceType serviceType, string providerId); + /// /// Refreshes the cached credential for the active AI provider. /// diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelRuntimeConfiguration.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelRuntimeConfiguration.cs index d634c13e30ef..32934f0bd9c3 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelRuntimeConfiguration.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelRuntimeConfiguration.cs @@ -11,6 +11,8 @@ namespace AdvancedPaste.Services; /// public interface IKernelRuntimeConfiguration { + string ProviderId { get; } + AIServiceType ServiceType { get; } string ModelName { get; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs index beb62fb293d2..6cef3b120836 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs @@ -12,5 +12,5 @@ namespace AdvancedPaste.Services; public interface IKernelService { - Task TransformClipboardAsync(string prompt, DataPackageView clipboardData, bool isSavedQuery, CancellationToken cancellationToken, IProgress progress); + Task TransformClipboardAsync(string prompt, DataPackageView clipboardData, bool isSavedQuery, CancellationToken cancellationToken, IProgress progress, string providerIdOverride = null); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs index 0d753d1ec327..5c0b775e541e 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs @@ -36,21 +36,20 @@ public abstract class KernelServiceBase( private readonly IUserSettings _userSettings = userSettings; private readonly ICustomActionTransformService _customActionTransformService = customActionTransformService; - protected abstract string AdvancedAIModelName { get; } + protected abstract PromptExecutionSettings GetPromptExecutionSettings(IKernelRuntimeConfiguration runtimeConfig); - protected abstract PromptExecutionSettings PromptExecutionSettings { get; } - - protected abstract void AddChatCompletionService(IKernelBuilder kernelBuilder); + protected abstract void AddChatCompletionService(IKernelBuilder kernelBuilder, IKernelRuntimeConfiguration runtimeConfig); protected abstract AIServiceUsage GetAIServiceUsage(ChatMessageContent chatMessage); - protected abstract IKernelRuntimeConfiguration GetRuntimeConfiguration(); + protected abstract IKernelRuntimeConfiguration GetRuntimeConfiguration(string providerIdOverride); - public async Task TransformClipboardAsync(string prompt, DataPackageView clipboardData, bool isSavedQuery, CancellationToken cancellationToken, IProgress progress) + public async Task TransformClipboardAsync(string prompt, DataPackageView clipboardData, bool isSavedQuery, CancellationToken cancellationToken, IProgress progress, string providerIdOverride = null) { Logger.LogTrace(); - var kernel = CreateKernel(); + var runtimeConfig = GetRuntimeConfiguration(providerIdOverride); + var kernel = CreateKernel(runtimeConfig); kernel.SetDataPackageView(clipboardData); kernel.SetCancellationToken(cancellationToken); kernel.SetProgress(progress); @@ -63,9 +62,9 @@ public async Task TransformClipboardAsync(string prompt, DataPackag try { - (chatHistory, var usage) = cacheUsed ? await ExecuteCachedActionChain(kernel, maybeCacheValue.ActionChain) : await ExecuteAICompletion(kernel, prompt, cancellationToken); + (chatHistory, var usage) = cacheUsed ? await ExecuteCachedActionChain(kernel, maybeCacheValue.ActionChain) : await ExecuteAICompletion(kernel, prompt, runtimeConfig, cancellationToken); - LogResult(cacheUsed, isSavedQuery, kernel.GetOrAddActionChain(), usage); + LogResult(cacheUsed, isSavedQuery, kernel.GetOrAddActionChain(), usage, runtimeConfig); var outputPackage = kernel.GetDataPackage(); var hasUsableData = await outputPackage.GetView().HasUsableDataAsync(); @@ -163,10 +162,8 @@ private static string GetFullPrompt(ChatHistory initialHistory) return $"{combinedSystemMessage}{newLine}{newLine}User instructions:{newLine}{userPromptMessage.Content}"; } - private async Task<(ChatHistory ChatHistory, AIServiceUsage Usage)> ExecuteAICompletion(Kernel kernel, string prompt, CancellationToken cancellationToken) + private async Task<(ChatHistory ChatHistory, AIServiceUsage Usage)> ExecuteAICompletion(Kernel kernel, string prompt, IKernelRuntimeConfiguration runtimeConfig, CancellationToken cancellationToken) { - var runtimeConfig = GetRuntimeConfiguration(); - ChatHistory chatHistory = []; var systemPrompt = string.IsNullOrWhiteSpace(runtimeConfig.SystemPrompt) ? DefaultSystemPrompt : runtimeConfig.SystemPrompt; @@ -188,13 +185,13 @@ private static string GetFullPrompt(ChatHistory initialHistory) chatHistory.AddUserMessage(prompt); } - if (ShouldModerateAdvancedAI()) + if (ShouldModerateAdvancedAI(runtimeConfig)) { await _promptModerationService.ValidateAsync(GetFullPrompt(chatHistory), cancellationToken); } - var chatResult = await kernel.GetRequiredService(AdvancedAIModelName) - .GetChatMessageContentAsync(chatHistory, PromptExecutionSettings, kernel, cancellationToken); + var chatResult = await kernel.GetRequiredService(runtimeConfig.ModelName) + .GetChatMessageContentAsync(chatHistory, GetPromptExecutionSettings(runtimeConfig), kernel, cancellationToken); chatHistory.Add(chatResult); var totalUsage = chatHistory.Select(GetAIServiceUsage) @@ -224,32 +221,30 @@ private static string GetFullPrompt(ChatHistory initialHistory) protected IUserSettings UserSettings => _userSettings; - private void LogResult(bool cacheUsed, bool isSavedQuery, IEnumerable actionChain, AIServiceUsage usage) + private void LogResult(bool cacheUsed, bool isSavedQuery, IEnumerable actionChain, AIServiceUsage usage, IKernelRuntimeConfiguration runtimeConfig) { - var runtimeConfig = GetRuntimeConfiguration(); - AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new( cacheUsed, isSavedQuery, usage.PromptTokens, usage.CompletionTokens, - AdvancedAIModelName, + runtimeConfig.ModelName, runtimeConfig.ServiceType.ToString(), AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain)); PowerToysTelemetry.Log.WriteEvent(telemetryEvent); // Log endpoint usage - var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType, AdvancedAIModelName, isAdvanced: true); + var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType, runtimeConfig.ModelName, isAdvanced: true); PowerToysTelemetry.Log.WriteEvent(endpointEvent); var logEvent = new AIServiceFormatEvent(telemetryEvent); Logger.LogDebug($"{nameof(TransformClipboardAsync)} complete; {logEvent.ToJsonString()}"); } - private Kernel CreateKernel() + private Kernel CreateKernel(IKernelRuntimeConfiguration runtimeConfig) { var kernelBuilder = Kernel.CreateBuilder(); - AddChatCompletionService(kernelBuilder); + AddChatCompletionService(kernelBuilder, runtimeConfig); kernelBuilder.Plugins.AddFromFunctions("Actions", GetKernelFunctions()); return kernelBuilder.Build(); } @@ -436,7 +431,7 @@ static string FormatKernelContent(KernelContent kernelContent) => return $"-> {role}: {redactedContent}{usageString}"; } - protected virtual bool ShouldModerateAdvancedAI() + protected virtual bool ShouldModerateAdvancedAI(IKernelRuntimeConfiguration runtimeConfig) { return false; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs index ff64a5ad8328..0f02efa77eba 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs @@ -9,15 +9,18 @@ using AdvancedPaste.Helpers; using AdvancedPaste.Models; using AdvancedPaste.Services.CustomActions; +using AdvancedPaste.Settings; +using Microsoft.PowerToys.Settings.UI.Library; using Microsoft.PowerToys.Telemetry; using Windows.ApplicationModel.DataTransfer; namespace AdvancedPaste.Services; -public sealed class PasteFormatExecutor(IKernelService kernelService, ICustomActionTransformService customActionTransformService) : IPasteFormatExecutor +public sealed class PasteFormatExecutor(IKernelService kernelService, ICustomActionTransformService customActionTransformService, IUserSettings userSettings) : IPasteFormatExecutor { private readonly IKernelService _kernelService = kernelService; private readonly ICustomActionTransformService _customActionTransformService = customActionTransformService; + private readonly IUserSettings _userSettings = userSettings; public async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source, CancellationToken cancellationToken, IProgress progress) { @@ -36,8 +39,9 @@ public async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, return await Task.Run(async () => pasteFormat.Format switch { - PasteFormats.KernelQuery => await _kernelService.TransformClipboardAsync(pasteFormat.Prompt, clipboardData, pasteFormat.IsSavedQuery, cancellationToken, progress), - PasteFormats.CustomTextTransformation => DataPackageHelpers.CreateFromText((await _customActionTransformService.TransformAsync(pasteFormat.Prompt, await clipboardData.GetTextOrHtmlTextAsync(), await clipboardData.GetImageAsPngBytesAsync(), cancellationToken, progress))?.Content ?? string.Empty), + PasteFormats.KernelQuery => await _kernelService.TransformClipboardAsync(pasteFormat.Prompt, clipboardData, pasteFormat.IsSavedQuery, cancellationToken, progress, pasteFormat.ProviderId), + PasteFormats.CustomTextTransformation => DataPackageHelpers.CreateFromText((await _customActionTransformService.TransformAsync(pasteFormat.Prompt, await clipboardData.GetTextOrHtmlTextAsync(), await clipboardData.GetImageAsPngBytesAsync(), cancellationToken, progress, providerIdOverride: pasteFormat.ProviderId))?.Content ?? string.Empty), + PasteFormats.FixSpellingAndGrammar => DataPackageHelpers.CreateFromText((await _customActionTransformService.TransformAsync(GetFixSpellingPrompt(), await clipboardData.GetTextOrHtmlTextAsync(), null, cancellationToken, progress, GetFixSpellingSystemPrompt(), pasteFormat.ProviderId))?.Content ?? string.Empty), _ => await TransformHelpers.TransformAsync(format, clipboardData, cancellationToken, progress), }); } @@ -62,4 +66,16 @@ private static void WriteTelemetry(PasteFormats format, PasteActionSource source throw new ArgumentOutOfRangeException(nameof(format)); } } + + private string GetFixSpellingPrompt() + { + var customPrompt = _userSettings.FixSpellingAndGrammarPrompt; + return string.IsNullOrWhiteSpace(customPrompt) ? AdvancedPasteDefaultPrompts.FixSpellingAndGrammar : customPrompt; + } + + private string GetFixSpellingSystemPrompt() + { + var customSystemPrompt = _userSettings.FixSpellingAndGrammarSystemPrompt; + return string.IsNullOrWhiteSpace(customSystemPrompt) ? AdvancedPasteDefaultPrompts.FixSpellingAndGrammarSystem : customSystemPrompt; + } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw b/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw index f36577832131..388001b94d46 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw +++ b/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw @@ -232,6 +232,12 @@ Paste as plain text + + Fix spelling and grammar + + + What was changed and why + Image to text diff --git a/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs b/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs index b474b8215af6..7a280a7297e1 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs @@ -16,8 +16,8 @@ using AdvancedPaste.Helpers; using AdvancedPaste.Models; using AdvancedPaste.Services; +using AdvancedPaste.Services.CustomActions; using AdvancedPaste.Settings; -using Common.UI; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using ManagedCommon; @@ -41,6 +41,7 @@ public sealed partial class OptionsViewModel : ObservableObject, IProgress double.IsNaN(TransformProgress); - private PasteFormats CustomAIFormat => - _userSettings.IsAIEnabled && TryResolveAdvancedAIProvider(out _) + private PasteFormats CustomAIFormat => GetCustomAIFormat(); + + private PasteFormats GetCustomAIFormat(string providerIdOverride = null) => + _userSettings.IsAIEnabled && AdvancedAIProviderResolver.TryResolveAdvancedProvider(_userSettings?.PasteAIConfiguration, providerIdOverride, out _) ? PasteFormats.KernelQuery : PasteFormats.CustomTextTransformation; @@ -258,11 +261,12 @@ private bool Visible public event EventHandler PreviewRequested; - public OptionsViewModel(IFileSystem fileSystem, IAICredentialsProvider credentialsProvider, IUserSettings userSettings, IPasteFormatExecutor pasteFormatExecutor) + public OptionsViewModel(IFileSystem fileSystem, IAICredentialsProvider credentialsProvider, IUserSettings userSettings, IPasteFormatExecutor pasteFormatExecutor, ICustomActionTransformService customActionTransformService) { _credentialsProvider = credentialsProvider; _userSettings = userSettings; _pasteFormatExecutor = pasteFormatExecutor; + _customActionTransformService = customActionTransformService; GeneratedResponses = []; GeneratedResponses.CollectionChanged += (s, e) => @@ -341,11 +345,21 @@ private void EnqueueRefreshPasteFormats() }); } - private PasteFormat CreateStandardPasteFormat(PasteFormats format) => - PasteFormat.CreateStandardFormat(format, AvailableClipboardFormats, IsCustomAIServiceEnabled, ResourceLoaderInstance.ResourceLoader.GetString); + private PasteFormat CreateStandardPasteFormat(PasteFormats format) + { + var providerId = GetProviderIdForFormat(format); + return PasteFormat.CreateStandardFormat(format, AvailableClipboardFormats, IsCustomAIServiceEnabled, ResourceLoaderInstance.ResourceLoader.GetString, providerId); + } - private PasteFormat CreateCustomAIPasteFormat(string name, string prompt, bool isSavedQuery) => - PasteFormat.CreateCustomAIFormat(CustomAIFormat, name, prompt, isSavedQuery, AvailableClipboardFormats, IsCustomAIServiceEnabled); + private PasteFormat CreateCustomAIPasteFormat(string name, string prompt, bool isSavedQuery, string providerId = null) => + PasteFormat.CreateCustomAIFormat(GetCustomAIFormat(providerId), name, prompt, isSavedQuery, AvailableClipboardFormats, IsCustomAIServiceEnabled, providerId); + + private string GetProviderIdForFormat(PasteFormats format) => + format switch + { + PasteFormats.FixSpellingAndGrammar => _userSettings.FixSpellingAndGrammarProviderId, + _ => string.Empty, + }; private void UpdateAIProviderActiveFlags() { @@ -418,7 +432,7 @@ void UpdateFormats(ObservableCollection collection, IEnumerable CreateCustomAIPasteFormat(customAction.Name, customAction.Prompt, isSavedQuery: true)) : []); + IsCustomAIServiceEnabled ? _userSettings.CustomActions.Select(customAction => CreateCustomAIPasteFormat(customAction.Name, customAction.Prompt, isSavedQuery: true, customAction.ProviderId)) : []); } public void Dispose() @@ -539,6 +553,7 @@ public async Task OnShowAsync() { PasteActionError = PasteActionError.None; Query = string.Empty; + CoachingExplanation = null; await ReadClipboardAsync(); @@ -616,6 +631,12 @@ public string CustomAIUnavailableErrorText [ObservableProperty] private string _customFormatResult; + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(HasCoachingExplanation))] + private string _coachingExplanation; + + public bool HasCoachingExplanation => !string.IsNullOrEmpty(CoachingExplanation); + [RelayCommand] public async Task PasteCustomAsync() { @@ -661,17 +682,37 @@ public void NextCustomFormat() [RelayCommand] public void OpenSettings() { - SettingsDeepLink.OpenSettings(SettingsDeepLink.SettingsWindow.AdvancedPaste); + try + { + var exePath = System.IO.Path.Combine( + ManagedCommon.PowerToysPathResolver.GetPowerToysInstallPath(), + "PowerToys.exe"); + + if (exePath != null && System.IO.File.Exists(exePath)) + { + System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo + { + FileName = exePath, + Arguments = "--open-settings=AdvancedPaste", + UseShellExecute = false, + }); + } + } + catch (Exception ex) + { + Logger.LogError("Failed to open settings", ex); + } + GetMainWindow()?.Close(); } - internal async Task ExecutePasteFormatAsync(PasteFormats format, PasteActionSource source) + internal async Task ExecutePasteFormatAsync(PasteFormats format, PasteActionSource source, bool forceCoaching = false) { await ReadClipboardAsync(); - await ExecutePasteFormatAsync(CreateStandardPasteFormat(format), source); + await ExecutePasteFormatAsync(CreateStandardPasteFormat(format), source, forceCoaching); } - internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source) + internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source, bool forceCoaching = false) { if (IsBusy) { @@ -704,12 +745,30 @@ internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteAction await delayTask; var outputText = await dataPackage.GetView().GetTextOrEmptyAsync(); + bool isCoachingAction = pasteFormat.Format == PasteFormats.FixSpellingAndGrammar && + (forceCoaching || (_userSettings.FixSpellingAndGrammarCoachingEnabled && !_userSettings.FixSpellingAndGrammarCoachingShortcutSet)); bool shouldPreview = pasteFormat.Metadata.CanPreview && _userSettings.ShowCustomPreview && !string.IsNullOrEmpty(outputText) && source != PasteActionSource.GlobalKeyboardShortcut; + // Coaching mode forces preview even for global keyboard shortcuts + if (isCoachingAction && !string.IsNullOrEmpty(outputText)) + { + shouldPreview = true; + } + if (shouldPreview) { GeneratedResponses.Add(outputText); CurrentResponseIndex = GeneratedResponses.Count - 1; + + if (isCoachingAction) + { + await GenerateCoachingExplanationAsync(outputText); + } + else + { + CoachingExplanation = null; + } + PreviewRequested?.Invoke(this, EventArgs.Empty); } else @@ -730,6 +789,65 @@ internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteAction Logger.LogDebug($"Finished executing {pasteFormat.Format} from source {source}; timeTakenMs={elapsedWatch.ElapsedMilliseconds}"); } + private async Task GenerateCoachingExplanationAsync(string correctedText) + { + try + { + var originalText = ClipboardData != null ? await ClipboardData.GetTextOrEmptyAsync() : string.Empty; + + if (string.IsNullOrEmpty(originalText)) + { + CoachingExplanation = null; + return; + } + + static string NormalizeForComparison(string s) => + s.Replace('\u2018', '\'') // left single quote + .Replace('\u2019', '\'') // right single quote / apostrophe + .Replace('\u201C', '"') // left double quote + .Replace('\u201D', '"') // right double quote + .Replace('\u2013', '-') // en dash + .Replace('\u2014', '-'); // em dash + + if (string.Equals(NormalizeForComparison(originalText), NormalizeForComparison(correctedText), StringComparison.Ordinal)) + { + CoachingExplanation = null; + return; + } + + var coachingInstruction = string.IsNullOrWhiteSpace(_userSettings.FixSpellingAndGrammarCoachingPrompt) + ? AdvancedPasteDefaultPrompts.FixSpellingAndGrammarCoaching + : _userSettings.FixSpellingAndGrammarCoachingPrompt; + var coachingInputText = $"Original:\n\"{originalText}\"\n\nCorrected:\n\"{correctedText}\""; + + var coachingSystemPrompt = string.IsNullOrWhiteSpace(_userSettings.FixSpellingAndGrammarCoachingSystemPrompt) + ? AdvancedPasteDefaultPrompts.FixSpellingAndGrammarCoachingSystem + : _userSettings.FixSpellingAndGrammarCoachingSystemPrompt; + + var coachingProviderId = _userSettings.FixSpellingAndGrammarCoachingProviderId; + if (string.IsNullOrWhiteSpace(coachingProviderId)) + { + coachingProviderId = _userSettings.FixSpellingAndGrammarProviderId; + } + + var result = await _customActionTransformService.TransformAsync( + coachingInstruction, + coachingInputText, + null, + _pasteActionCancellationTokenSource?.Token ?? CancellationToken.None, + null, + coachingSystemPrompt, + string.IsNullOrWhiteSpace(coachingProviderId) ? null : coachingProviderId); + + CoachingExplanation = result?.Content; + } + catch (Exception ex) + { + Logger.LogError("Error generating coaching explanation", ex); + CoachingExplanation = null; + } + } + internal async Task ExecutePasteFormatAsync(VirtualKey key) { var pasteFormat = StandardPasteFormats.Concat(CustomActionPasteFormats) @@ -751,7 +869,7 @@ internal async Task ExecuteCustomActionAsync(int customActionId, PasteActionSour if (customAction != null) { await ReadClipboardAsync(); - await ExecutePasteFormatAsync(CreateCustomAIPasteFormat(customAction.Name, customAction.Prompt, isSavedQuery: true), source); + await ExecutePasteFormatAsync(CreateCustomAIPasteFormat(customAction.Name, customAction.Prompt, isSavedQuery: true, customAction.ProviderId), source); } } @@ -760,7 +878,7 @@ internal async Task ExecuteCustomAIFormatFromCurrentQueryAsync(PasteActionSource var customAction = _userSettings.CustomActions .FirstOrDefault(customAction => Models.KernelQueryCache.CacheKey.PromptComparer.Equals(customAction.Prompt, Query)); - await ExecutePasteFormatAsync(CreateCustomAIPasteFormat(customAction?.Name ?? "Default", Query, isSavedQuery: customAction != null), triggerSource); + await ExecutePasteFormatAsync(CreateCustomAIPasteFormat(customAction?.Name ?? "Default", Query, isSavedQuery: customAction != null, customAction?.ProviderId), triggerSource); } private void HideWindow() @@ -823,49 +941,6 @@ private bool IsProviderAllowedByGPO(PasteAIProviderDefinition provider) }; } - private bool TryResolveAdvancedAIProvider(out PasteAIProviderDefinition provider) - { - provider = null; - - var configuration = _userSettings?.PasteAIConfiguration; - if (configuration is null) - { - return false; - } - - var activeProvider = configuration.ActiveProvider; - if (IsAdvancedAIProvider(activeProvider)) - { - provider = activeProvider; - return true; - } - - if (activeProvider is not null) - { - return false; - } - - var fallback = configuration.Providers?.FirstOrDefault(IsAdvancedAIProvider); - if (fallback is not null) - { - provider = fallback; - return true; - } - - return false; - } - - private static bool IsAdvancedAIProvider(PasteAIProviderDefinition provider) - { - return provider is not null && provider.EnableAdvancedAI && SupportsAdvancedAI(provider.ServiceTypeKind); - } - - private static bool SupportsAdvancedAI(AIServiceType serviceType) - { - return serviceType is AIServiceType.OpenAI - or AIServiceType.AzureOpenAI; - } - private bool UpdateOpenAIKey() { UpdateAllowedByGPO(); diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPaste.base.rc b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPaste.base.rc index b30e3923c989..eb9e4e22d1c0 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPaste.base.rc +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPaste.base.rc @@ -1,40 +1,6 @@ #include #include "resource.h" -#include "../../../../common/version/version.h" #define APSTUDIO_READONLY_SYMBOLS #include "winres.h" #undef APSTUDIO_READONLY_SYMBOLS - -1 VERSIONINFO -FILEVERSION FILE_VERSION -PRODUCTVERSION PRODUCT_VERSION -FILEFLAGSMASK VS_FFI_FILEFLAGSMASK -#ifdef _DEBUG -FILEFLAGS VS_FF_DEBUG -#else -FILEFLAGS 0x0L -#endif -FILEOS VOS_NT_WINDOWS32 -FILETYPE VFT_DLL -FILESUBTYPE VFT2_UNKNOWN -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" // US English (0x0409), Unicode (0x04B0) charset - BEGIN - VALUE "CompanyName", COMPANY_NAME - VALUE "FileDescription", FILE_DESCRIPTION - VALUE "FileVersion", FILE_VERSION_STRING - VALUE "InternalName", INTERNAL_NAME - VALUE "LegalCopyright", COPYRIGHT_NOTE - VALUE "OriginalFilename", ORIGINAL_FILENAME - VALUE "ProductName", PRODUCT_NAME - VALUE "ProductVersion", PRODUCT_VERSION_STRING - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 // US English (0x0409), Unicode (1200) charset - END -END diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteModuleInterface.vcxproj b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteModuleInterface.vcxproj index 9f7675799ea4..ca40cf792f0a 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteModuleInterface.vcxproj +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteModuleInterface.vcxproj @@ -15,7 +15,6 @@ DynamicLibrary - diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteProcessManager.cpp b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteProcessManager.cpp index b202f93f4e10..dfa798a565a4 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteProcessManager.cpp +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteProcessManager.cpp @@ -100,25 +100,30 @@ HRESULT AdvancedPasteProcessManager::start_process(const std::wstring& pipe_name { const unsigned long powertoys_pid = GetCurrentProcessId(); - const auto executable_args = std::format(L"{} {}", std::to_wstring(powertoys_pid), pipe_name); - - SHELLEXECUTEINFOW sei{ sizeof(sei) }; - sei.fMask = { SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI }; - sei.lpFile = L"WinUI3Apps\\PowerToys.AdvancedPaste.exe"; - sei.nShow = SW_SHOWNORMAL; - sei.lpParameters = executable_args.data(); - if (ShellExecuteExW(&sei)) - { - Logger::trace("Successfully started Advanced Paste process"); - terminate_process(); - m_hProcess = sei.hProcess; - return S_OK; - } - else - { - Logger::error(L"Advanced Paste process failed to start. {}", get_last_error_or_default(GetLastError())); - return E_FAIL; - } + const auto launch_direct_exe = [&]() -> HRESULT { + // Fallback: launch exe directly (dev builds without GenerateAppxPackageOnBuild) + const auto executable_args = std::format(L"{} {}", std::to_wstring(powertoys_pid), pipe_name); + + SHELLEXECUTEINFOW sei{ sizeof(sei) }; + sei.fMask = { SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI }; + sei.lpFile = L"WinUI3Apps\\PowerToys.AdvancedPaste.exe"; + sei.nShow = SW_SHOWNORMAL; + sei.lpParameters = executable_args.data(); + if (ShellExecuteExW(&sei)) + { + Logger::trace("Successfully started Advanced Paste process (direct)"); + terminate_process(); + m_hProcess = sei.hProcess; + return S_OK; + } + else + { + Logger::error(L"Advanced Paste process failed to start. {}", get_last_error_or_default(GetLastError())); + return E_FAIL; + } + }; + + return launch_direct_exe(); } HRESULT AdvancedPasteProcessManager::start_named_pipe_server(const std::wstring& pipe_name) @@ -175,8 +180,9 @@ HRESULT AdvancedPasteProcessManager::start_named_pipe_server(const std::wstring& } } - // Wait for client. - const constexpr DWORD client_timeout_millis = 5000; + // Wait for client. AdvancedPaste under sparse identity can take >5s on cold start to + // bootstrap WinAppSDK + DI host before connecting back to this pipe. + const constexpr DWORD client_timeout_millis = 15000; switch (WaitForSingleObject(overlapped.hEvent, client_timeout_millis)) { case WAIT_OBJECT_0: diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp index 2e180c0320e7..d74a2d41af74 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp @@ -66,6 +66,8 @@ namespace const wchar_t JSON_KEY_PROVIDERS[] = L"providers"; const wchar_t JSON_KEY_SERVICE_TYPE[] = L"service-type"; const wchar_t JSON_KEY_ENABLE_ADVANCED_AI[] = L"enable-advanced-ai"; + const wchar_t JSON_KEY_COACHING_SHORTCUT[] = L"coaching-shortcut"; + const wchar_t JSON_KEY_COACHING_ENABLED[] = L"coaching-enabled"; const wchar_t JSON_KEY_VALUE[] = L"value"; } @@ -255,6 +257,21 @@ class AdvancedPaste : public PowertoyModuleIface }; m_additional_actions.push_back(additionalAction); + + // Register coaching shortcut as a separate hotkey with a "-coaching" suffix ID + if (action.HasKey(JSON_KEY_COACHING_SHORTCUT) && action.GetNamedBoolean(JSON_KEY_COACHING_ENABLED, false)) + { + auto coachingHotkey = parse_single_hotkey(action.GetNamedObject(JSON_KEY_COACHING_SHORTCUT), actionIsShown); + if (coachingHotkey.key != 0) + { + const AdditionalAction coachingAction + { + std::wstring(actionName.c_str()) + L"-coaching", + coachingHotkey + }; + m_additional_actions.push_back(coachingAction); + } + } } else { @@ -407,6 +424,7 @@ class AdvancedPaste : public PowertoyModuleIface // Define the expected order to ensure consistent hotkey ID assignment const std::vector expectedOrder = { L"image-to-text", + L"fix-spelling-and-grammar", L"paste-as-file", L"transcode" }; @@ -982,6 +1000,12 @@ class AdvancedPaste : public PowertoyModuleIface m_triggerEventWaiter.start(CommonSharedConstants::ADVANCED_PASTE_SHOW_UI_EVENT, [this](DWORD) { // Same logic as hotkeyId == 1 (m_advanced_paste_ui_hotkey) Logger::trace(L"AdvancedPaste ShowUI event triggered"); + + if (m_auto_copy_selection_custom_action) + { + send_copy_selection(); // best-effort; ignore failure + } + m_process_manager.start(); m_process_manager.bring_to_front(); m_process_manager.send_message(CommonSharedConstants::ADVANCED_PASTE_SHOW_UI_MESSAGE); @@ -1032,13 +1056,11 @@ class AdvancedPaste : public PowertoyModuleIface } } - if (is_custom_action_hotkey && m_auto_copy_selection_custom_action) + // Try to capture selected text for all hotkey actions when the setting is enabled. + // If nothing is selected (clipboard unchanged), fall through to use existing clipboard content. + if (m_auto_copy_selection_custom_action) { - if (!send_copy_selection()) - { - Logger::warn(L"Auto-copy: failed to copy selection for custom action index {} — aborting action", custom_action_index); - return false; - } + send_copy_selection(); // best-effort; ignore failure } m_process_manager.start(); diff --git a/src/modules/AdvancedPaste/custom.props b/src/modules/AdvancedPaste/custom.props new file mode 100644 index 000000000000..3b4b52e42a8d --- /dev/null +++ b/src/modules/AdvancedPaste/custom.props @@ -0,0 +1,11 @@ + + + + + true + 2025 + 0 + 9 + PowerToys Advanced Paste + + diff --git a/src/settings-ui/Settings.UI.Library/AIServiceType.cs b/src/settings-ui/Settings.UI.Library/AIServiceType.cs index 27eccff1cf6e..e30ebbb5d2a7 100644 --- a/src/settings-ui/Settings.UI.Library/AIServiceType.cs +++ b/src/settings-ui/Settings.UI.Library/AIServiceType.cs @@ -19,5 +19,6 @@ public enum AIServiceType Google, AzureAIInference, Ollama, + PhiSilica, } } diff --git a/src/settings-ui/Settings.UI.Library/AIServiceTypeExtensions.cs b/src/settings-ui/Settings.UI.Library/AIServiceTypeExtensions.cs index 5b19212ebaf2..887c0dc78e50 100644 --- a/src/settings-ui/Settings.UI.Library/AIServiceTypeExtensions.cs +++ b/src/settings-ui/Settings.UI.Library/AIServiceTypeExtensions.cs @@ -31,6 +31,7 @@ public static AIServiceType ToAIServiceType(this string serviceType) "google" or "googleai" or "googlegemini" => AIServiceType.Google, "azureaiinference" or "azureinference" => AIServiceType.AzureAIInference, "ollama" => AIServiceType.Ollama, + "phisilica" or "phi" or "philm" => AIServiceType.PhiSilica, _ => AIServiceType.Unknown, }; } @@ -51,6 +52,7 @@ public static string ToConfigurationString(this AIServiceType serviceType) AIServiceType.Google => "Google", AIServiceType.AzureAIInference => "AzureAIInference", AIServiceType.Ollama => "Ollama", + AIServiceType.PhiSilica => "PhiSilica", AIServiceType.Unknown => string.Empty, _ => throw new ArgumentOutOfRangeException(nameof(serviceType), serviceType, "Unsupported AI service type."), }; @@ -72,6 +74,7 @@ public static string ToNormalizedKey(this AIServiceType serviceType) AIServiceType.Google => "google", AIServiceType.AzureAIInference => "azureaiinference", AIServiceType.Ollama => "ollama", + AIServiceType.PhiSilica => "phisilica", _ => string.Empty, }; } diff --git a/src/settings-ui/Settings.UI.Library/AIServiceTypeRegistry.cs b/src/settings-ui/Settings.UI.Library/AIServiceTypeRegistry.cs index 653b85553e39..2c45d9bae60a 100644 --- a/src/settings-ui/Settings.UI.Library/AIServiceTypeRegistry.cs +++ b/src/settings-ui/Settings.UI.Library/AIServiceTypeRegistry.cs @@ -118,6 +118,15 @@ public static class AIServiceTypeRegistry PrivacyLabel = "AdvancedPaste_OpenAI_PrivacyLabel", PrivacyUri = new Uri("https://openai.com/privacy"), }, + [AIServiceType.PhiSilica] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.PhiSilica, + DisplayName = "Phi Silica", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/WindowsML.svg", + IsOnlineService = false, + IsLocalModel = true, + LegalDescription = "AdvancedPaste_LocalModel_LegalDescription", + }, [AIServiceType.Unknown] = new AIServiceTypeMetadata { ServiceType = AIServiceType.Unknown, diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteAdditionalAction.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteAdditionalAction.cs index 1642ecf9c421..6e46e54e4b64 100644 --- a/src/settings-ui/Settings.UI.Library/AdvancedPasteAdditionalAction.cs +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteAdditionalAction.cs @@ -12,7 +12,15 @@ namespace Microsoft.PowerToys.Settings.UI.Library; public sealed partial class AdvancedPasteAdditionalAction : Observable, IAdvancedPasteAction { private HotkeySettings _shortcut = new(); + private HotkeySettings _coachingShortcut = new(); private bool _isShown; + private string _prompt = string.Empty; + private string _systemPrompt = string.Empty; + private string _coachingPrompt = string.Empty; + private string _coachingSystemPrompt = string.Empty; + private string _providerId = string.Empty; + private string _coachingProviderId = string.Empty; + private bool _coachingEnabled; private bool _hasConflict; private string _tooltip; @@ -33,6 +41,20 @@ public HotkeySettings Shortcut } } + [JsonPropertyName("coaching-shortcut")] + public HotkeySettings CoachingShortcut + { + get => _coachingShortcut; + set + { + if (_coachingShortcut != value) + { + _coachingShortcut = value ?? new(); + OnPropertyChanged(); + } + } + } + [JsonPropertyName("isShown")] public bool IsShown { @@ -40,6 +62,55 @@ public bool IsShown set => Set(ref _isShown, value); } + [JsonPropertyName("prompt")] + public string Prompt + { + get => _prompt; + set => Set(ref _prompt, value ?? string.Empty); + } + + [JsonPropertyName("system-prompt")] + public string SystemPrompt + { + get => _systemPrompt; + set => Set(ref _systemPrompt, value ?? string.Empty); + } + + [JsonPropertyName("coaching-prompt")] + public string CoachingPrompt + { + get => _coachingPrompt; + set => Set(ref _coachingPrompt, value ?? string.Empty); + } + + [JsonPropertyName("coaching-system-prompt")] + public string CoachingSystemPrompt + { + get => _coachingSystemPrompt; + set => Set(ref _coachingSystemPrompt, value ?? string.Empty); + } + + [JsonPropertyName("provider-id")] + public string ProviderId + { + get => _providerId; + set => Set(ref _providerId, value ?? string.Empty); + } + + [JsonPropertyName("coaching-provider-id")] + public string CoachingProviderId + { + get => _coachingProviderId; + set => Set(ref _coachingProviderId, value ?? string.Empty); + } + + [JsonPropertyName("coaching-enabled")] + public bool CoachingEnabled + { + get => _coachingEnabled; + set => Set(ref _coachingEnabled, value); + } + [JsonIgnore] public bool HasConflict { diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteAdditionalActions.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteAdditionalActions.cs index b193c01c74ee..b476f50f674a 100644 --- a/src/settings-ui/Settings.UI.Library/AdvancedPasteAdditionalActions.cs +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteAdditionalActions.cs @@ -11,12 +11,14 @@ namespace Microsoft.PowerToys.Settings.UI.Library; public sealed class AdvancedPasteAdditionalActions { private AdvancedPasteAdditionalAction _imageToText = new(); + private AdvancedPasteAdditionalAction _fixSpellingAndGrammar = new(); private AdvancedPastePasteAsFileAction _pasteAsFile = new(); private AdvancedPasteTranscodeAction _transcode = new(); public static class PropertyNames { public const string ImageToText = "image-to-text"; + public const string FixSpellingAndGrammar = "fix-spelling-and-grammar"; public const string PasteAsFile = "paste-as-file"; public const string Transcode = "transcode"; } @@ -28,6 +30,13 @@ public AdvancedPasteAdditionalAction ImageToText init => _imageToText = value ?? new(); } + [JsonPropertyName(PropertyNames.FixSpellingAndGrammar)] + public AdvancedPasteAdditionalAction FixSpellingAndGrammar + { + get => _fixSpellingAndGrammar; + init => _fixSpellingAndGrammar = value ?? new(); + } + [JsonPropertyName(PropertyNames.PasteAsFile)] public AdvancedPastePasteAsFileAction PasteAsFile { @@ -44,7 +53,7 @@ public AdvancedPasteTranscodeAction Transcode public IEnumerable GetAllActions() { - return GetAllActionsRecursive([ImageToText, PasteAsFile, Transcode]); + return GetAllActionsRecursive([ImageToText, FixSpellingAndGrammar, PasteAsFile, Transcode]); } /// diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteCustomAction.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteCustomAction.cs index c98129590698..4a3a763c4c94 100644 --- a/src/settings-ui/Settings.UI.Library/AdvancedPasteCustomAction.cs +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteCustomAction.cs @@ -16,6 +16,7 @@ public sealed class AdvancedPasteCustomAction : Observable, IAdvancedPasteAction private string _name = string.Empty; private string _description = string.Empty; private string _prompt = string.Empty; + private string _providerId = string.Empty; private HotkeySettings _shortcut = new(); private bool _isShown; private bool _canMoveUp; @@ -64,6 +65,13 @@ public string Prompt } } + [JsonPropertyName("provider-id")] + public string ProviderId + { + get => _providerId; + set => Set(ref _providerId, value ?? string.Empty); + } + [JsonPropertyName("shortcut")] public HotkeySettings Shortcut { @@ -138,6 +146,7 @@ public void Update(AdvancedPasteCustomAction other) Name = other.Name; Description = other.Description; Prompt = other.Prompt; + ProviderId = other.ProviderId; Shortcut = other.GetShortcutClone(); IsShown = other.IsShown; CanMoveUp = other.CanMoveUp; diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteDefaultPrompts.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteDefaultPrompts.cs new file mode 100644 index 000000000000..c58170f7320f --- /dev/null +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteDefaultPrompts.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.PowerToys.Settings.UI.Library; + +/// +/// Shared default prompts for built-in AI actions. Referenced by both the AdvancedPaste module +/// and the Settings UI to ensure consistent defaults and enable "reset to default" functionality. +/// +public static class AdvancedPasteDefaultPrompts +{ + public const string FixSpellingAndGrammar = "Fix all spelling and grammar errors in the following text. Return only the corrected text without any additional explanation or commentary."; + + public const string FixSpellingAndGrammarSystem = "You are a professional proofreader. You fix spelling and grammar errors in text. You return only the corrected text with no commentary."; + + public const string FixSpellingAndGrammarCoaching = "Briefly explain what was changed and why in terms of language rules. Be concise as reviewer."; + + public const string FixSpellingAndGrammarCoachingSystem = "You are a writing coach and language teacher. You will be given an original sentence and a corrected version."; +} diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteSettings.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteSettings.cs index be001fd9d6a1..e1a5eb64ed95 100644 --- a/src/settings-ui/Settings.UI.Library/AdvancedPasteSettings.cs +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteSettings.cs @@ -68,6 +68,7 @@ public HotkeyAccessor[] GetAllHotkeyAccessors() string[] additionalActionHeaderKeys = [ "ImageToText", + "FixSpellingAndGrammar", "PasteAsTxtFile", "PasteAsPngFile", "PasteAsHtmlFile", @@ -79,11 +80,25 @@ public HotkeyAccessor[] GetAllHotkeyAccessors() { if (action is AdvancedPasteAdditionalAction additionalAction) { + var headerKey = additionalActionHeaderKeys[Math.Min(index, additionalActionHeaderKeys.Length - 1)]; hotkeyAccessors.Add(new HotkeyAccessor( () => additionalAction.Shortcut, value => additionalAction.Shortcut = value ?? new HotkeySettings(), - additionalActionHeaderKeys[index])); + headerKey)); index++; + + // The coaching shortcut is registered by the runner as a separate hotkey + // immediately after Fix Spelling and Grammar (and only when it's active), so it + // must appear in the same position here to keep hotkey IDs aligned with conflicts. + if (ReferenceEquals(additionalAction, Properties.AdditionalActions.FixSpellingAndGrammar) + && additionalAction.CoachingEnabled + && additionalAction.CoachingShortcut is { Code: not 0 }) + { + hotkeyAccessors.Add(new HotkeyAccessor( + () => additionalAction.CoachingShortcut, + value => additionalAction.CoachingShortcut = value ?? new HotkeySettings(), + "FixSpellingAndGrammarCoaching")); + } } } diff --git a/src/settings-ui/Settings.UI/SettingsXAML/Views/AdvancedPastePage.xaml b/src/settings-ui/Settings.UI/SettingsXAML/Views/AdvancedPastePage.xaml index 5d1fa671ac7b..fc04cd2ed45d 100644 --- a/src/settings-ui/Settings.UI/SettingsXAML/Views/AdvancedPastePage.xaml +++ b/src/settings-ui/Settings.UI/SettingsXAML/Views/AdvancedPastePage.xaml @@ -116,7 +116,19 @@ Header="{x:Bind ModelName, Mode=OneWay}" HeaderIcon="{x:Bind ServiceType, Mode=OneWay, Converter={StaticResource ServiceTypeToIconConverter}}"> + + +