diff --git a/plugins/power-pages/.claude-plugin/plugin.json b/plugins/power-pages/.claude-plugin/plugin.json index ba73e795f..6e91e7418 100644 --- a/plugins/power-pages/.claude-plugin/plugin.json +++ b/plugins/power-pages/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "power-pages", - "version": "2.6.4", + "version": "2.6.5", "description": "Create and deploy Power Pages sites using modern development approaches. Supports code sites (SPAs) with React, Angular, Vue, or Astro. Includes ALM orchestration (plan-alm) with a solution-splitting decision tree, per-solution pipelines, Azure Blob asset advisory, manifest schema v2 for multi-solution deployments, and force-link remediation for cross-host pipeline migrations.", "author": { "name": "Microsoft", diff --git a/plugins/power-pages/.plugin/plugin.json b/plugins/power-pages/.plugin/plugin.json index ba73e795f..6e91e7418 100644 --- a/plugins/power-pages/.plugin/plugin.json +++ b/plugins/power-pages/.plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "power-pages", - "version": "2.6.4", + "version": "2.6.5", "description": "Create and deploy Power Pages sites using modern development approaches. Supports code sites (SPAs) with React, Angular, Vue, or Astro. Includes ALM orchestration (plan-alm) with a solution-splitting decision tree, per-solution pipelines, Azure Blob asset advisory, manifest schema v2 for multi-solution deployments, and force-link remediation for cross-host pipeline migrations.", "author": { "name": "Microsoft", diff --git a/plugins/power-pages/README.md b/plugins/power-pages/README.md index b111f9af9..1a1682f5a 100644 --- a/plugins/power-pages/README.md +++ b/plugins/power-pages/README.md @@ -38,7 +38,7 @@ This keeps hook behavior in one place and avoids relying on skill-frontmatter ho ## Skills -The plugin provides 33 skills that cover the full lifecycle of a Power Pages site — scaffolding, deployment, data modeling, backend integration, authentication, ALM and CI/CD, security review, testing, auditing, and Bootstrap 3→5 migration. Each skill is invoked conversationally — just describe what you want to do. +The plugin provides 34 skills that cover the full lifecycle of a Power Pages site — scaffolding, deployment, data modeling, backend integration, authentication, ALM and CI/CD, security review, testing, auditing, and platform migrations. Each skill is invoked conversationally — just describe what you want to do. ### Site scaffolding and deployment @@ -400,6 +400,17 @@ Migrates a traditional Power Pages site (Liquid web templates, not code sites) f - AI-assisted per-category fixes for grid, navbar, panel/card, and page-header changes - Uploads (auto-enabling the Bootstrap 5 runtime flag) and verifies the flip via `pac-log.txt` +#### `/migrate-webapi-selectall` + +> "Replace every wildcard Web API fields setting with the columns my site actually uses" + +Reviews every authored Power Pages Web API source call and response consumer, maps entity sets through Dataverse metadata, and replaces deprecated `Webapi//fields = *` values with evidence-backed explicit columns. Compiled and generated output is excluded. Works with both traditional/Liquid sites and React, Vue, Angular, or Astro SPA sites. + +- Reports every wildcard with its exact proposed fix and every already-explicit configuration +- Adds missing `$select` projections where normal record reads relied on implicit selection +- Traces every call site reaching a table, including duplicated wrappers and differing query shapes +- Verifies all configuration scopes and deployment profiles contain zero wildcards + ### Support #### `/report-issue` diff --git a/plugins/power-pages/references/approval-gates.md b/plugins/power-pages/references/approval-gates.md index dcd663860..91e721fc3 100644 --- a/plugins/power-pages/references/approval-gates.md +++ b/plugins/power-pages/references/approval-gates.md @@ -697,6 +697,21 @@ New skill (migrates a traditional Power Pages site from Bootstrap 3 to Bootstrap | `migrate-bootstrap:6.residual-fixes` | gate | progress | 6 | Per-category consent — *"Apply the `` fixes to `` file(s)?"* — repeated per residual category; changes are local to the V5 copy. | nothing | | `migrate-bootstrap:7.2.upload` | gate | final | 7.2 | First outward-facing change — *"Upload `` to ``?"* — publishes the Bootstrap 5 site and auto-enables the runtime flag. | nothing | +--- + +### 6.32 `migrate-webapi-selectall` (6 calls / 5 gates + 1 sub-prompt) + +Reviews traditional and SPA sites for deprecated Web API wildcard fields settings, derives least-privilege columns from every call chain, and applies only a complete approved migration. + +| ID | Kind | Category | Phase | Trigger / question | Cancel leaves | +|---|---|---|---|---|---| +| `migrate-webapi-selectall:1.download-site` | gate | consent | 1 | Approves an optional site download after confirming environment, website name and ID, site type, data model, and target path. | nothing | +| `migrate-webapi-selectall:2.confirm-scope` | gate | plan | 2 | Confirms all configuration scopes, wildcard and explicit counts, and the source call inventory before schema retrieval. | draft migration report | +| Phase 3 environment URL | sub-prompt | — | 3.1 | Collects the environment URL only when project and PAC context cannot resolve it. This is read-only metadata input and grants no write consent. | draft migration report | +| `migrate-webapi-selectall:4.apply-plan` | gate | consent | 4 | Approves every wildcard replacement, required source projection, selected explicit hardening, and local edits. No partial wildcard option is offered. | reviewed migration report | +| `migrate-webapi-selectall:7.deploy` | gate | final | 7 | Approves one independently verified deployment after re-confirming environment, website, site type, data model, and profile. Repeat for another target. | local migration | +| `migrate-webapi-selectall:7.smoke-test` | gate | progress | 7 | Approves the listed read-path smoke test against the deployed site. Write, file, and image paths are never issued. | deployed migration unverified | + --- ### Cross-plugin shared skills — out of catalog scope diff --git a/plugins/power-pages/references/skill-tracking-reference.md b/plugins/power-pages/references/skill-tracking-reference.md index 223b50746..8c7bc1e84 100644 --- a/plugins/power-pages/references/skill-tracking-reference.md +++ b/plugins/power-pages/references/skill-tracking-reference.md @@ -57,6 +57,7 @@ If the tracking script creates or updates site setting YAML files, include those | ensure-pipelines-host | EnsurePipelinesHost | Site/AI/Skills/EnsurePipelinesHost | | force-link-environment | ForceLinkEnvironment | Site/AI/Skills/ForceLinkEnvironment | | migrate-bootstrap | MigrateBootstrap | Site/AI/Skills/MigrateBootstrap | +| migrate-webapi-selectall | MigrateWebapiSelectall | Site/AI/Skills/MigrateWebapiSelectall | ## YAML Format diff --git a/plugins/power-pages/scripts/tests/query-webapi-selectall-schema.test.js b/plugins/power-pages/scripts/tests/query-webapi-selectall-schema.test.js new file mode 100644 index 000000000..a1e554ad8 --- /dev/null +++ b/plugins/power-pages/scripts/tests/query-webapi-selectall-schema.test.js @@ -0,0 +1,236 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { spawnSync } = require('node:child_process'); + +const schema = require( + '../../skills/migrate-webapi-selectall/scripts/query-table-schema' +); + +const ENVIRONMENT_URL_1 = 'https://placeholder.crm.dynamics.com'; +const TABLE_LOGICAL_NAME_1 = 'table_1'; +const TABLE_ENTITY_SET_NAME_1 = 'table_1_set'; +const TABLE_PRIMARY_ID_1 = 'column_name_1_id'; +const TABLE_LOGICAL_NAME_2 = 'table_2'; +const TABLE_ENTITY_SET_NAME_2 = 'table_2_set'; +const TABLE_PRIMARY_ID_2 = 'column_name_2_id'; +const COLUMN_NAME_1 = 'column_name_1'; +const LOOKUP_COLUMN_NAME_1 = 'lookup_column_name_1'; +const RELATIONSHIP_NAME_1 = 'relationship_1'; + +test('builds bounded metadata URLs for one table', () => { + const urls = schema.buildMetadataUrls( + ENVIRONMENT_URL_1, + TABLE_LOGICAL_NAME_1 + ); + + assert.ok(urls.attributes.includes( + `EntityDefinitions(LogicalName='${TABLE_LOGICAL_NAME_1}')/Attributes` + )); + assert.match(urls.attributes, /%24select=/); + assert.match(urls.manyToMany, /ManyToManyRelationships/); +}); + +test('resolves both logical names and entity sets', () => { + const definitions = [{ + LogicalName: TABLE_LOGICAL_NAME_1, + EntitySetName: TABLE_ENTITY_SET_NAME_1, + PrimaryIdAttribute: TABLE_PRIMARY_ID_1, + }]; + + assert.deepEqual( + schema.resolveRequestedTables( + definitions, + [TABLE_LOGICAL_NAME_1, TABLE_ENTITY_SET_NAME_1] + ), + definitions + ); +}); + +test('rejects project root as schema output', () => { + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'webapi-schema-root-')); + try { + assert.throws( + () => schema.validateOptions({ + environmentUrl: ENVIRONMENT_URL_1, + projectRoot, + output: projectRoot, + tables: [TABLE_LOGICAL_NAME_1], + }), + /file inside the project root/ + ); + } finally { + fs.rmSync(projectRoot, { recursive: true, force: true }); + } +}); + +test('rejects output through an escaping junction', (t) => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'webapi-schema-link-')); + const projectRoot = path.join(workspace, 'project'); + const outside = path.join(workspace, 'outside'); + t.after(() => fs.rmSync(workspace, { recursive: true, force: true })); + + fs.mkdirSync(projectRoot); + fs.mkdirSync(outside); + fs.writeFileSync( + path.join(outside, 'tables.txt'), + `${TABLE_LOGICAL_NAME_1}\n`, + 'utf8' + ); + + try { + fs.symlinkSync( + outside, + path.join(projectRoot, 'linked'), + process.platform === 'win32' ? 'junction' : 'dir' + ); + } catch (error) { + if (error.code === 'EPERM' || error.code === 'EACCES') { + t.skip(`symlinks are unavailable: ${error.code}`); + return; + } + throw error; + } + + assert.throws( + () => schema.validateOptions({ + environmentUrl: ENVIRONMENT_URL_1, + projectRoot, + output: path.join(projectRoot, 'linked', 'schema.json'), + tables: [TABLE_LOGICAL_NAME_1], + }), + /inside the project root/ + ); + assert.throws( + () => schema.validateOptions({ + environmentUrl: ENVIRONMENT_URL_1, + projectRoot, + output: path.join(projectRoot, 'schema.json'), + tables: [], + tablesFile: path.join(projectRoot, 'linked', 'tables.txt'), + }), + /Tables file must exist inside the project root/ + ); +}); + +test('normalizes attributes and navigation metadata', () => { + const normalized = schema.normalizeTableMetadata( + { + LogicalName: TABLE_LOGICAL_NAME_1, + EntitySetName: TABLE_ENTITY_SET_NAME_1, + PrimaryIdAttribute: TABLE_PRIMARY_ID_1, + }, + [{ + LogicalName: LOOKUP_COLUMN_NAME_1, + AttributeType: 'Lookup', + IsValidForRead: true, + IsValidForCreate: { Value: true }, + IsValidForUpdate: false, + }], + [{ + SchemaName: RELATIONSHIP_NAME_1, + ReferencingEntity: TABLE_LOGICAL_NAME_1, + ReferencedEntity: TABLE_LOGICAL_NAME_2, + ReferencingAttribute: LOOKUP_COLUMN_NAME_1, + ReferencingEntityNavigationPropertyName: LOOKUP_COLUMN_NAME_1, + ReferencedEntityNavigationPropertyName: RELATIONSHIP_NAME_1, + }], + [] + ); + + assert.deepEqual( + normalized.lookupReadProperties, + [`_${LOOKUP_COLUMN_NAME_1}_value`] + ); + assert.deepEqual(normalized.navigationProperties, [{ + name: LOOKUP_COLUMN_NAME_1, + targetLogicalName: TABLE_LOGICAL_NAME_2, + lookupAttribute: LOOKUP_COLUMN_NAME_1, + relationship: RELATIONSHIP_NAME_1, + }]); + assert.equal(normalized.attributes[0].isValidForCreate, true); +}); + +test('queries only resolved requested tables', async () => { + const calls = []; + const getAll = async (url) => { + calls.push(url); + if (/EntityDefinitions\?/.test(url)) { + return [{ + LogicalName: TABLE_LOGICAL_NAME_1, + EntitySetName: TABLE_ENTITY_SET_NAME_1, + PrimaryIdAttribute: TABLE_PRIMARY_ID_1, + }, { + LogicalName: TABLE_LOGICAL_NAME_2, + EntitySetName: TABLE_ENTITY_SET_NAME_2, + PrimaryIdAttribute: TABLE_PRIMARY_ID_2, + }]; + } + if (/\/Attributes\?/.test(url)) { + return [{ + LogicalName: COLUMN_NAME_1, + AttributeType: 'String', + IsValidForRead: true, + IsValidForCreate: true, + IsValidForUpdate: true, + }]; + } + return []; + }; + + const result = await schema.queryTableSchemas( + ENVIRONMENT_URL_1, + [TABLE_ENTITY_SET_NAME_1], + { + getAuthToken: () => 'token', + odataGetAll: getAll, + } + ); + + assert.equal(result.tables.length, 1); + assert.equal(result.tables[0].logicalName, TABLE_LOGICAL_NAME_1); + assert.equal(calls.filter(url => /\/Attributes\?/.test(url)).length, 1); +}); + +test('retries transient metadata throttling sequentially', async () => { + let attempts = 0; + const delays = []; + const getAll = async (url) => { + attempts += 1; + if (attempts === 1) throw new Error('HTTP 429 throttled'); + if (/EntityDefinitions\?/.test(url)) { + return [{ + LogicalName: TABLE_LOGICAL_NAME_1, + EntitySetName: TABLE_ENTITY_SET_NAME_1, + PrimaryIdAttribute: TABLE_PRIMARY_ID_1, + }]; + } + return []; + }; + + const result = await schema.queryTableSchemas( + ENVIRONMENT_URL_1, + [TABLE_LOGICAL_NAME_1], + { + getAuthToken: () => 'token', + odataGetAll: getAll, + sleep: async delay => delays.push(delay), + } + ); + + assert.equal(result.tables.length, 1); + assert.deepEqual(delays, [1000]); +}); + +test('query-table-schema answers --help before parsing arguments', () => { + const scriptPath = require.resolve( + '../../skills/migrate-webapi-selectall/scripts/query-table-schema' + ); + const result = spawnSync(process.execPath, [scriptPath, '--help'], { encoding: 'utf8' }); + assert.equal(result.status, 0); + assert.ok(result.stdout); +}); diff --git a/plugins/power-pages/scripts/tests/render-migration-report.test.js b/plugins/power-pages/scripts/tests/render-migration-report.test.js new file mode 100644 index 000000000..2b7b3a70e --- /dev/null +++ b/plugins/power-pages/scripts/tests/render-migration-report.test.js @@ -0,0 +1,195 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const vm = require('node:vm'); +const { spawnSync } = require('node:child_process'); + +const scriptPath = path.join( + __dirname, '..', '..', 'skills', 'migrate-webapi-selectall', 'scripts', 'render-migration-report.js' +); + +const attack = ' \' "'; + +function baseData(overrides = {}) { + return Object.assign({ + REPORT_STATUS: 'Complete', + SCOPE_NOTE: 'Reviewed 1 configuration scope and 4 source call sites.', + WILDCARD_DATA: [], + EXPLICIT_DATA: [], + }, overrides); +} + +function render(data) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'migration-report-')); + const dataPath = path.join(tempDir, 'data.json'); + const outputPath = path.join(tempDir, 'migration-report.html'); + fs.writeFileSync(dataPath, JSON.stringify(data), 'utf8'); + + const result = spawnSync(process.execPath, [scriptPath, '--output', outputPath, '--data', dataPath], { + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr || result.stdout); + return { html: fs.readFileSync(outputPath, 'utf8'), outputPath, dataPath }; +} + +class FakeElement { + constructor() { + this.innerHTML = ''; + this.textContent = ''; + } +} + +// Runs the inline renderer against a stub DOM. +function executeInlineRenderer(html) { + const inline = html.match(/ + + + diff --git a/plugins/power-pages/skills/migrate-webapi-selectall/references/column-analysis.md b/plugins/power-pages/skills/migrate-webapi-selectall/references/column-analysis.md new file mode 100644 index 000000000..6ecb43876 --- /dev/null +++ b/plugins/power-pages/skills/migrate-webapi-selectall/references/column-analysis.md @@ -0,0 +1,119 @@ +# Column Analysis Rules + +Use these rules to determine the smallest explicit field list that preserves +each Power Pages Web API operation. Read every call chain and consumer; static +search results are an inventory, not the final answer. + +## Contents + +- [Evidence priority](#evidence-priority) +- [Endpoint mapping](#endpoint-mapping) +- [Read operations](#read-operations) +- [Write operations](#write-operations) +- [File and image operations](#file-and-image-operations) +- [Dynamic code](#dynamic-code) +- [Configuration proposal](#configuration-proposal) + +## Evidence priority + +Prefer evidence in this order: + +1. Request syntax: `$select`, `$expand`, property paths, FetchXML, `$filter`, + `$orderby`, and `$apply`. +2. Write payloads: POST, PATCH, PUT, deep inserts, and `@odata.bind`. +3. Response consumers: property access, destructuring, mappers, templates, + framework bindings, and TypeScript interfaces. +4. Shared builders: callers, defaults, conditional branches, and runtime + configuration. +5. Dataverse schema: canonical attributes, entity sets, lookups, and + navigation properties. +6. User-confirmed integration contracts when repository evidence is + incomplete. + +Do not mark a proposal ready while a source wrapper, branch, or response +consumer remains unread. Never substitute compiled or generated code for +missing source. + +## Endpoint mapping + +Power Pages request paths use `EntitySetName`. +`Webapi/
/fields` settings use the table logical name. These names are +not reliably related by pluralization, so map them through the schema snapshot. + +Classify search, summarization, cloud-flow, and server-logic routes under +`/_api/` as non-table endpoints only after confirming their public API shape. +Do not silently omit an unfamiliar endpoint. + +## Read operations + +For each table-returning GET: + +- include root `$select` columns; +- include columns used by `$filter` and `$orderby`; +- include lookup read properties such as `__value` when + consumed; +- include the source lookup attribute for single-valued `$expand` navigation; +- assign nested `$select`, filter, and order columns to the expanded table; +- include FetchXML attributes, conditions, ordering, grouping, and aggregate + inputs on their owning entity or link-entity; +- include `$apply` group, filter, and aggregate inputs, excluding output + aliases; +- add `$select` when a normal record GET omits it, deriving the projection from + every response consumer. + +Maintain two separate results: + +- the response projection contains only columns returned to consumers; +- the fields-setting allowlist contains projection, filter, order, aggregate, + lookup, property-route, and write requirements. + +Do not widen an existing `$select` with a filter-only, order-only, +grouping-only, or write-only column. + +`$top`, `$skip`, `$count`, paging cookies, aggregate aliases, formatted-value +annotations, and filter literals are not Dataverse columns. + +## Write operations + +For POST, PATCH, and PUT: + +- include every payload property mapped to a Dataverse attribute; +- resolve `navigation@odata.bind` to the underlying lookup attribute; +- trace variables passed to `JSON.stringify`, HTTP clients, and custom + wrappers; +- inspect conditional properties and every object spread source; +- assign deep-insert object fields to their related target table. + +For property PUT or DELETE routes, include the property named in the URL. A +record DELETE adds no field requirement by itself, but the table setting still +needs a nonempty, evidence-backed field list. + +## File and image operations + +Treat file and image endpoints as explicit-column operations: + +- include the file or image column in the route; +- include file name, size, MIME type, or image metadata only when separately + read; +- never retain `*` as a compatibility workaround. + +## Dynamic code + +Trace dynamic entity sets, column arrays, URL fragments, query strings, +request bodies, and FetchXML through every caller and runtime branch. + +When values come from external configuration, obtain the owning contract or +user confirmation. Do not infer fields from labels, singularization, naming +conventions, or likely primary keys. + +## Configuration proposal + +For each logical table and configuration scope, compute the sorted union of +proven columns across all reads and writes. Every proposed column needs: + +- a relative source path and line, or a user-confirmed contract; +- the operation requiring it; +- its exact metadata-canonical name. + +Do not add every metadata attribute, speculative identifiers, unrelated +server-side fields, or columns owned by expanded target tables. diff --git a/plugins/power-pages/skills/migrate-webapi-selectall/references/configuration-and-reporting.md b/plugins/power-pages/skills/migrate-webapi-selectall/references/configuration-and-reporting.md new file mode 100644 index 000000000..89abdaf1f --- /dev/null +++ b/plugins/power-pages/skills/migrate-webapi-selectall/references/configuration-and-reporting.md @@ -0,0 +1,163 @@ +# Configuration and Reporting Contract + +## Contents + +- [Configuration layouts](#configuration-layouts) +- [Configuration scopes](#configuration-scopes) +- [Report](#report) +- [Completion criteria](#completion-criteria) +- [Public references](#public-references) + +## Configuration layouts + +Scan every layout because framework choice does not determine serialization. + +Traditional and downloaded declarative sites commonly use an aggregate file: + +```text +sitesetting.yml +``` + +```yaml +- adx_name: Webapi/
/fields + adx_sitesettingid: + adx_value: , +``` + +SPA sites can contain the same aggregate shape at +`.powerpages-site/sitesetting.yml`, or one file per setting: + +```text +.powerpages-site/site-settings/*.sitesetting.yml +``` + +```yaml +id: +name: Webapi/
/fields +value: , +``` + +Modular files can use either `name` and `value` or `adx_name` and +`adx_value`. Inspect deployment-profile copies too. + +## Configuration scopes + +Treat each aggregate `sitesetting.yml` as its own scope. Treat modular setting +files in the same `site-settings` directory as one scope. Deployment profiles +are separate scopes even when they contain the same setting name. + +For each scope: + +- pair `Webapi/
/enabled` with `Webapi/
/fields`; +- identify missing, duplicate, wildcard, and already-explicit field settings; +- preserve record identifiers, key style, quoting, comments, and unrelated + values; +- use logical table names in settings and `EntitySetName` in requests. + +Table permissions and column permissions remain separate authorization +layers. Replacing `*` reduces the exposed Web API surface but does not repair +overbroad record permissions. + +## Report + +Write the report data, then render it. Never hand-write report HTML; the +renderer is the encoding boundary that keeps setting names, column names, and +paths as data. + +Write the data file to: + +```text +docs/webapi-selectall-migration/migration-report.json +``` + +```json +{ + "REPORT_STATUS": "Complete | Partial | Draft | Failed", + "SCOPE_NOTE": "", + "WILDCARD_DATA": [ + { + "setting": "Webapi/
/fields", + "status": "Pending | Migrated | Removed | Blocked", + "usages": [ + { + "location": ":", + "detail": "" + } + ], + "fields": [ + "", + "" + ], + "finding": "", + "fix": "" + } + ], + "EXPLICIT_DATA": [ + { + "setting": "Webapi/
/fields", + "status": "Least privilege | Incomplete | Overbroad", + "usages": [ + { + "location": ":", + "detail": "" + } + ], + "fields": [ + "" + ], + "finding": "", + "fix": "" + } + ] +} +``` + +Both arrays are required and may be empty. One setting gets one entry, however +many call sites it has: list every call site in `usages` so the report shows the +same table read through different wrappers or pages. `status` selects the badge +tone and the counters, so use the listed values. Use `Pending` in the draft +report before replacements are proposed, and `Blocked` for a wildcard whose +replacement cannot be proven, stating in its `fix` what the user has to decide. +Use an empty `fields` array when a setting is removed or blocked. + +Render with: + +```bash +node "${PLUGIN_ROOT}/skills/migrate-webapi-selectall/scripts/render-migration-report.js" --output "/docs/webapi-selectall-migration/migration-report.html" --data "/docs/webapi-selectall-migration/migration-report.json" +``` + +The renderer refuses to overwrite, so delete the previous +`migration-report.html` before re-rendering an updated report. It stamps the +generated time itself, and also writes `power-pages-icon.png` beside the +report; leave that file in place. + +Record only relative paths, line numbers, table names, and column names. Never +include absolute local paths, source snippets, request or response bodies, +tokens, environment URLs, hostnames, filter literals, record identifiers, or +user data. + +## Completion criteria + +The wildcard migration is complete only when: + +- every configuration file and deployment scope was inspected; +- every original wildcard has an approved explicit replacement; +- every candidate source `/_api/` occurrence has a resolved disposition; +- every proposed field has source or contract evidence and schema validation; +- every normal table GET has an explicit `$select`; +- a fresh independent pass finds zero wildcard field settings; +- the report includes every wildcard and already-explicit setting; +- only `migration-report.html` remains in the migration output directory. + +If explicit settings still have unresolved missing or overbroad fields, set +`REPORT_STATUS` to `Partial` rather than claiming full Web API hardening. + +## Public references + +- [Important upcoming changes and deprecations in Power Pages](https://learn.microsoft.com/en-us/power-pages/important-changes-deprecations#wildcard-value--in-web-api-field-configuration) +- [Portals Web API overview](https://learn.microsoft.com/en-us/power-pages/configure/web-api-overview) +- [Query data using portals Web API](https://learn.microsoft.com/en-us/power-pages/configure/read-operations) +- [Use portals Web API write, update, and delete operations](https://learn.microsoft.com/en-us/power-pages/configure/write-update-delete-operations) +- [Overview of developer capabilities](https://learn.microsoft.com/en-us/power-pages/configure/developer-overview) +- [Create a code site using AI coding agents](https://learn.microsoft.com/en-us/power-pages/configure/create-code-site-using-claude-code) +- [Power Platform CLI pages commands](https://learn.microsoft.com/en-us/power-platform/developer/cli/reference/pages) diff --git a/plugins/power-pages/skills/migrate-webapi-selectall/references/site-transfer.md b/plugins/power-pages/skills/migrate-webapi-selectall/references/site-transfer.md new file mode 100644 index 000000000..d8d6a2455 --- /dev/null +++ b/plugins/power-pages/skills/migrate-webapi-selectall/references/site-transfer.md @@ -0,0 +1,77 @@ +# Site Transfer Safety + +Download and upload move Power Pages configuration between Dataverse and the +local project. Neither operation has an undo, and both can destroy +configuration that this migration never touched. + +## Identity to confirm + +Establish every value below and have the user confirm it. Never infer one from +a folder name, a previous session, or the active default. + +| Detail | Confirm with | Needed for | +|---|---|---| +| Environment | `pac auth who`, `pac env who` | download, upload | +| Website name and `WebSiteId` | `pac pages list`, `website.yml` | download, upload | +| Site type: traditional or SPA | `powerpages.config.json` presence | download, upload | +| Data model: Standard or Enhanced | user | traditional only | +| Deployment profile | user, and profile files in the project | upload | +| Target path | user | download | + +Re-confirm the whole set whenever the environment, website, or data model +changes. One approval covers one target. + +## Command matrix + +| Site type | Download | Upload | +|---|---|---| +| Traditional | `pac pages download` | `pac pages upload` | +| SPA | `pac pages download-code-site` | `pac pages upload-code-site` | + +```bash +pac pages list --environment "" + +pac pages download --path "" --webSiteId "" --environment "" --modelVersion "" + +pac pages upload --path "" --environment "" --modelVersion "" --deploymentProfile "" + +pac pages download-code-site --path "" --webSiteId "" --environment "" + +pac pages upload-code-site --rootPath "" --siteName "" +``` + +Pass `--modelVersion` explicitly on every traditional download and upload; +`Standard` and `Enhanced` may also be written as `1` and `2`. + +## Failure modes + +Each of these corrupts configuration and cannot be reverted. + +- **Wrong upload command.** `pac pages upload` corrupts an SPA site's + metadata, and `pac pages upload-code-site` corrupts a traditional site. A + project holding `powerpages.config.json` is an SPA site even when it also + contains `.powerpages-site/` artifacts. +- **Wrong data model.** `pac pages download` uses `Standard` when + `--modelVersion` is omitted, so an Enhanced site downloads incomplete. + Editing that copy and uploading it overwrites live configuration. +- **Unpinned environment.** `pac pages upload-code-site` accepts no + `--environment` and targets whatever `pac auth who` reports. Re-run + `pac auth who` immediately before it and stop on any mismatch. +- **Wrong website.** Sites in one environment differ only by name and + `--webSiteId`. A wrong ID downloads another site over the target path. + Before uploading, confirm the project's `website.yml` identity still matches + the approved site. +- **Overwritten local work.** `--overwrite` replaces existing local content. + Never pass it while the project holds unreviewed changes. +- **Stale local copy.** Uploading configuration downloaded before another + maker's change reverts that change. Download fresh before editing when the + local copy's age is unknown. +- **Unintended deployment profile.** `pac pages upload` uses `default` when + `--deploymentProfile` is omitted, which can target the wrong environment's + values. + +## Public references + +- [Power Platform CLI pages commands](https://learn.microsoft.com/en-us/power-platform/developer/cli/reference/pages) +- [Tutorial: Use Power Platform CLI with Power Pages](https://learn.microsoft.com/en-us/power-pages/configure/power-platform-cli-tutorial) +- [Use deployment profiles](https://learn.microsoft.com/en-us/power-apps/maker/portals/power-apps-cli#use-deployment-profile) diff --git a/plugins/power-pages/skills/migrate-webapi-selectall/scripts/query-table-schema.js b/plugins/power-pages/skills/migrate-webapi-selectall/scripts/query-table-schema.js new file mode 100644 index 000000000..51dfedbca --- /dev/null +++ b/plugins/power-pages/skills/migrate-webapi-selectall/scripts/query-table-schema.js @@ -0,0 +1,444 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { + getAuthToken, + odataGetAll, + validateDataverseEnvironmentUrl, +} = require('../../../scripts/lib/validation-helpers'); + +// Refresh the token periodically so long runs never expire. +const TABLES_PER_TOKEN_REFRESH = 4; +// Retry ceilings bound transient Dataverse throttling delays safely. +const MAX_RETRY_ATTEMPTS = 6; +const BASE_RETRY_DELAY_MS = 1000; +const MAX_RETRY_DELAY_MS = 30000; + +const HELP = ` +Retrieves Dataverse table schema for Web API wildcard migration analysis. + +Usage: + node query-table-schema.js --project-root --environment-url + (--table | --tables-file ) --output + +Options: + --project-root Power Pages project root; input and output stay inside it + --environment-url Dataverse environment URL + --table Logical name or entity set name; repeatable + --tables-file File of identifiers, one per line or a JSON array + --output JSON schema snapshot to write, inside the project root + --help Show this help message + +Exit codes: + 0 Snapshot written; prints { output, tableCount } to stdout + 1 Invalid arguments, unresolved table, auth failure, or query failure + +Example: + node query-table-schema.js --project-root . --environment-url https://contoso.crm.dynamics.com --table account --output docs/webapi-selectall-migration/table-schema.json +`; + +function parseArgs(argv) { + const options = { tables: [] }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (!argument.startsWith('--')) { + throw new Error(`Unexpected argument: ${argument}`); + } + const value = argv[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error(`Missing value for ${argument}.`); + } + index += 1; + if (argument === '--table') { + options.tables.push(value); + } else if (argument === '--tables-file') { + options.tablesFile = value; + } else if (argument === '--environment-url') { + options.environmentUrl = value; + } else if (argument === '--project-root') { + options.projectRoot = value; + } else if (argument === '--output') { + options.output = value; + } else { + throw new Error(`Unknown option: ${argument}`); + } + } + return options; +} + +function sleep(milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)); +} + +function isTransientError(error) { + return /\b(?:408|429|500|502|503|504)\b|ECONNRESET|ETIMEDOUT|socket hang up/i + .test(String(error?.message || error)); +} + +async function getAllWithRetry(getAll, url, token, wait = sleep) { + let lastError; + for (let attempt = 0; attempt < MAX_RETRY_ATTEMPTS; attempt += 1) { + try { + return await getAll(url, token); + } catch (error) { + lastError = error; + if (!isTransientError(error) || attempt === MAX_RETRY_ATTEMPTS - 1) { + throw error; + } + await wait(Math.min(MAX_RETRY_DELAY_MS, BASE_RETRY_DELAY_MS * (2 ** attempt))); + } + } + throw lastError; +} + +function realPath(candidate) { + return fs.realpathSync.native + ? fs.realpathSync.native(candidate) + : fs.realpathSync(candidate); +} + +function resolveThroughExistingAncestor(candidate) { + const unresolved = []; + let current = candidate; + while (!fs.existsSync(current)) { + const parent = path.dirname(current); + if (parent === current) { + throw new Error('Unable to resolve an existing path ancestor.'); + } + unresolved.unshift(path.basename(current)); + current = parent; + } + return path.resolve(realPath(current), ...unresolved); +} + +function isStrictlyInside(root, candidate) { + const relative = path.relative(root, candidate); + return relative !== '' && + !relative.startsWith(`..${path.sep}`) && + relative !== '..' && + !path.isAbsolute(relative); +} + +function resolveOutputPath(projectRoot, candidate) { + // Canonical paths prevent symlink and junction escapes. + const resolved = resolveThroughExistingAncestor(candidate); + if (resolved === projectRoot || + (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory())) { + throw new Error('Output must be a file inside the project root.'); + } + if (!isStrictlyInside(projectRoot, resolved)) { + throw new Error('Output must remain inside the project root.'); + } + return resolved; +} + +function metadataUrl(environmentUrl, resource, select) { + const url = new URL(`${environmentUrl}/api/data/v9.2/${resource}`); + url.searchParams.set('$select', select); + return url.href; +} + +function buildMetadataUrls(environmentUrl, logicalName) { + const escaped = logicalName.replace(/'/g, "''"); + const resource = `EntityDefinitions(LogicalName='${escaped}')`; + return { + attributes: metadataUrl( + environmentUrl, + `${resource}/Attributes`, + 'LogicalName,AttributeType,IsValidForRead,IsValidForCreate,IsValidForUpdate' + ), + oneToMany: metadataUrl( + environmentUrl, + `${resource}/OneToManyRelationships`, + 'SchemaName,ReferencingEntity,ReferencedEntity,ReferencingAttribute,ReferencingEntityNavigationPropertyName,ReferencedEntityNavigationPropertyName' + ), + manyToOne: metadataUrl( + environmentUrl, + `${resource}/ManyToOneRelationships`, + 'SchemaName,ReferencingEntity,ReferencedEntity,ReferencingAttribute,ReferencingEntityNavigationPropertyName,ReferencedEntityNavigationPropertyName' + ), + manyToMany: metadataUrl( + environmentUrl, + `${resource}/ManyToManyRelationships`, + 'SchemaName,Entity1LogicalName,Entity2LogicalName,Entity1NavigationPropertyName,Entity2NavigationPropertyName' + ), + }; +} + +function booleanValue(value) { + if (typeof value === 'boolean') return value; + if (value && typeof value.Value === 'boolean') return value.Value; + return null; +} + +function addNavigation(output, seen, navigation) { + if (!navigation.name || !navigation.targetLogicalName) return; + const key = `${navigation.name}:${navigation.targetLogicalName}`; + if (seen.has(key)) return; + seen.add(key); + output.push(navigation); +} + +function normalizeTableMetadata(definition, attributes, relationships, manyToMany) { + const navigationProperties = []; + const seenNavigation = new Set(); + + // Navigation names require relationship metadata for expansions. + for (const relationship of relationships) { + if (relationship.ReferencingEntity === definition.LogicalName) { + addNavigation(navigationProperties, seenNavigation, { + name: relationship.ReferencingEntityNavigationPropertyName, + targetLogicalName: relationship.ReferencedEntity, + lookupAttribute: relationship.ReferencingAttribute, + relationship: relationship.SchemaName, + }); + } + if (relationship.ReferencedEntity === definition.LogicalName) { + addNavigation(navigationProperties, seenNavigation, { + name: relationship.ReferencedEntityNavigationPropertyName, + targetLogicalName: relationship.ReferencingEntity, + lookupAttribute: null, + relationship: relationship.SchemaName, + }); + } + } + + for (const relationship of manyToMany) { + if (relationship.Entity1LogicalName === definition.LogicalName) { + addNavigation(navigationProperties, seenNavigation, { + name: relationship.Entity1NavigationPropertyName, + targetLogicalName: relationship.Entity2LogicalName, + lookupAttribute: null, + relationship: relationship.SchemaName, + }); + } + if (relationship.Entity2LogicalName === definition.LogicalName) { + addNavigation(navigationProperties, seenNavigation, { + name: relationship.Entity2NavigationPropertyName, + targetLogicalName: relationship.Entity1LogicalName, + lookupAttribute: null, + relationship: relationship.SchemaName, + }); + } + } + + const normalizedAttributes = attributes + .filter(attribute => attribute.LogicalName) + .map(attribute => ({ + logicalName: attribute.LogicalName, + attributeType: attribute.AttributeType || null, + isValidForRead: booleanValue(attribute.IsValidForRead), + isValidForCreate: booleanValue(attribute.IsValidForCreate), + isValidForUpdate: booleanValue(attribute.IsValidForUpdate), + })) + .sort((left, right) => left.logicalName.localeCompare(right.logicalName)); + const lookupTypes = new Set(['Customer', 'Lookup', 'Owner']); + const lookupReadProperties = normalizedAttributes + .filter(attribute => lookupTypes.has(attribute.attributeType)) + .map(attribute => `_${attribute.logicalName}_value`); + + return { + logicalName: definition.LogicalName, + entitySetName: definition.EntitySetName, + primaryIdAttribute: definition.PrimaryIdAttribute, + attributes: normalizedAttributes, + lookupReadProperties, + navigationProperties: navigationProperties.sort((left, right) => + left.name.localeCompare(right.name) + ), + }; +} + +function resolveRequestedTables(definitions, requested) { + const byIdentifier = new Map(); + for (const definition of definitions) { + if (definition.LogicalName) { + byIdentifier.set(definition.LogicalName.toLowerCase(), definition); + } + if (definition.EntitySetName) { + byIdentifier.set(definition.EntitySetName.toLowerCase(), definition); + } + } + const resolved = []; + const missing = []; + for (const identifier of requested) { + const definition = byIdentifier.get(identifier.toLowerCase()); + if (!definition) { + missing.push(identifier); + } else if (!resolved.some(item => item.LogicalName === definition.LogicalName)) { + resolved.push(definition); + } + } + if (missing.length > 0) { + throw new Error(`Unknown table logical names or entity sets: ${missing.join(', ')}`); + } + return resolved; +} + +async function queryTableSchemas(environmentUrl, requested, dependencies = {}) { + const getToken = dependencies.getAuthToken || getAuthToken; + const getAll = dependencies.odataGetAll || odataGetAll; + const wait = dependencies.sleep || sleep; + let token = getToken(environmentUrl); + if (!token) { + throw new Error('Authentication unavailable. Run az login --allow-no-subscriptions.'); + } + const definitions = await getAllWithRetry( + getAll, + metadataUrl( + environmentUrl, + 'EntityDefinitions', + 'LogicalName,EntitySetName,PrimaryIdAttribute' + ), + token, + wait + ); + const selected = resolveRequestedTables(definitions, requested); + const tables = []; + let queried = 0; + + for (let index = 0; index < selected.length; index += 1) { + const definition = selected[index]; + if (queried > 0 && queried % TABLES_PER_TOKEN_REFRESH === 0) { + token = getToken(environmentUrl); + if (!token) throw new Error('Authentication expired while querying table schemas.'); + } + const urls = buildMetadataUrls(environmentUrl, definition.LogicalName); + const attributes = await getAllWithRetry(getAll, urls.attributes, token, wait); + const oneToMany = await getAllWithRetry(getAll, urls.oneToMany, token, wait); + const manyToOne = await getAllWithRetry(getAll, urls.manyToOne, token, wait); + const manyToMany = await getAllWithRetry(getAll, urls.manyToMany, token, wait); + const table = normalizeTableMetadata( + definition, + attributes, + [...oneToMany, ...manyToOne], + manyToMany + ); + tables.push(table); + queried += 1; + } + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + tables: tables.sort((left, right) => left.logicalName.localeCompare(right.logicalName)), + }; +} + +function validateOptions(options) { + if (!options.environmentUrl || !options.projectRoot || !options.output) { + throw new Error( + 'Usage: query-table-schema.js --project-root --environment-url ' + + '--table [--table ] ' + + '[--tables-file ] --output ' + ); + } + const requestedProjectRoot = path.resolve(options.projectRoot); + if (!fs.existsSync(requestedProjectRoot) || + !fs.statSync(requestedProjectRoot).isDirectory()) { + throw new Error('Project root must be an existing directory.'); + } + const projectRoot = realPath(requestedProjectRoot); + const output = resolveOutputPath(projectRoot, path.resolve(options.output)); + let tablesFile = null; + if (options.tablesFile) { + const requestedTablesFile = path.resolve(options.tablesFile); + if (!fs.existsSync(requestedTablesFile)) { + throw new Error('Tables file must exist inside the project root.'); + } + tablesFile = realPath(requestedTablesFile); + if (!isStrictlyInside(projectRoot, tablesFile) || + !fs.statSync(tablesFile).isFile()) { + throw new Error('Tables file must exist inside the project root.'); + } + } + return { + environmentUrl: validateDataverseEnvironmentUrl(options.environmentUrl), + projectRoot, + output, + tables: options.tables, + tablesFile, + }; +} + +function loadTableIdentifiers(options) { + const values = [...options.tables]; + if (options.tablesFile) { + const content = fs.readFileSync(options.tablesFile, 'utf8').trim(); + if (content.startsWith('[')) { + const parsed = JSON.parse(content); + if (!Array.isArray(parsed)) { + throw new Error('Tables file JSON must contain an array.'); + } + for (const value of parsed) values.push(value); + } else if (content) { + values.push(...content.split(/\r?\n/)); + } + } + const normalized = [...new Set(values.map(value => String(value).trim().toLowerCase()) + .filter(Boolean))]; + if (normalized.length === 0) { + throw new Error('At least one table identifier is required.'); + } + for (const table of normalized) { + if (!/^[a-z_][a-z0-9_]*$/.test(table)) { + throw new Error(`Invalid table identifier: ${table}`); + } + } + return normalized; +} + +function writeAtomicJson(projectRoot, filePath, value) { + const output = resolveOutputPath(projectRoot, filePath); + fs.mkdirSync(path.dirname(output), { recursive: true }); + const temporary = `${output}.${process.pid}.tmp`; + try { + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); + fs.renameSync(temporary, output); + } catch (error) { + // Never strand a partial .tmp file behind. + fs.rmSync(temporary, { force: true }); + throw error; + } +} + +async function main() { + if (process.argv.includes('--help')) { + process.stdout.write(HELP); + return; + } + + let validated; + try { + validated = validateOptions(parseArgs(process.argv.slice(2))); + const tables = loadTableIdentifiers(validated); + const result = await queryTableSchemas(validated.environmentUrl, tables); + writeAtomicJson(validated.projectRoot, validated.output, result); + process.stdout.write(`${JSON.stringify({ + output: path.relative(validated.projectRoot, validated.output).split(path.sep).join('/'), + tableCount: result.tables.length, + })}\n`); + } catch (error) { + const message = validated?.environmentUrl + ? error.message.replaceAll(validated.environmentUrl, '') + : error.message; + process.stderr.write(`${message}\n`); + process.exitCode = 1; + } +} + +if (require.main === module) { + main(); +} + +module.exports = { + buildMetadataUrls, + normalizeTableMetadata, + parseArgs, + queryTableSchemas, + resolveRequestedTables, + loadTableIdentifiers, + validateOptions, +}; diff --git a/plugins/power-pages/skills/migrate-webapi-selectall/scripts/render-migration-report.js b/plugins/power-pages/skills/migrate-webapi-selectall/scripts/render-migration-report.js new file mode 100644 index 000000000..eb4b01d5d --- /dev/null +++ b/plugins/power-pages/skills/migrate-webapi-selectall/scripts/render-migration-report.js @@ -0,0 +1,76 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { renderTemplate, parseArgs } = require('../../../scripts/lib/render-template'); + +const HELP = ` +Renders the Web API wildcard migration report from a data file. + +Usage: + node render-migration-report.js --output --data + +Options: + --output HTML report to write; must not already exist + --data JSON data file describing the migration + --help Show this help message + +Required keys in the data file: + REPORT_STATUS, SCOPE_NOTE, WILDCARD_DATA, EXPLICIT_DATA + +WILDCARD_DATA and EXPLICIT_DATA are arrays of: + { setting, status, usages: [{ location, detail }], fields, finding, fix } + +Exit codes: + 0 Report written + 1 Missing flag, unreadable data file, missing key, or existing output + +Example: + node render-migration-report.js --output docs/webapi-selectall-migration/migration-report.html --data docs/webapi-selectall-migration/migration-report.json +`; + +// UTC keeps the stamp unambiguous across reader regions. +// dateStyle and timeStyle cannot combine with timeZoneName. +function formatGeneratedAt(now) { + return new Intl.DateTimeFormat('en-US', { + year: 'numeric', + month: 'long', + day: 'numeric', + hour: 'numeric', + minute: '2-digit', + timeZone: 'UTC', + timeZoneName: 'short', + }).format(now); +} + +function main() { + if (process.argv.includes('--help')) { + process.stdout.write(HELP); + return; + } + + const args = parseArgs(process.argv); + if (!args.output || !args.data) { + process.stderr.write('Usage: node render-migration-report.js --output --data \n'); + process.exit(1); + } + + let data; + try { + data = JSON.parse(fs.readFileSync(path.resolve(args.data), 'utf8')); + } catch (error) { + process.stderr.write(`Could not read data file ${args.data}: ${error.message}\n`); + process.exit(1); + } + + renderTemplate({ + templatePath: path.join(__dirname, '..', 'assets', 'migration-report-template.html'), + outputPath: path.resolve(args.output), + // Renderer owns the stamp reflecting render time. + dataObject: { ...data, GENERATED_AT: formatGeneratedAt(new Date()) }, + requiredKeys: ['REPORT_STATUS', 'SCOPE_NOTE', 'WILDCARD_DATA', 'EXPLICIT_DATA'], + }); +} + +main();