ChatGPT plugin: embed review test cases and release notes in plugin.json

openai-plugin-review/test-cases.json

@@ -1,5 +1,5 @@
 {
-  "_note": "Review materials for the ChatGPT plugin directory submission. Not part of the plugin ZIP. OpenAI asks for exactly five positive and three negative test cases; paste these into the dashboard's Review details. tools_triggered names the canonical tool.",
+  "_note": "Review materials for the ChatGPT plugin directory submission. The five positive and three negative cases are also embedded in openai-plugin/plugin.json under extensions.com.openai.review.test_cases, which the dashboard imports from the ZIP (imported cases are read-only there; edit here, mirror into plugin.json, rebuild). tools_triggered names the canonical tool.",
   "positive": [
     {
       "description": "Create a new chat app and make it the current app.",
@@ -58,6 +58,6 @@
     "notes": "The reviewer signs in on the Ethora consent page with email and password (Google sign-in also works). The connector requests the openid and email identity scopes; an unverified address shows a one-step verification prompt that can be skipped."
   },
   "demo_video": {
-    "_how": "Record 60-90 seconds in ChatGPT: connect the plugin, then the three default prompts (create app + agent, website widget, two agents debating). Upload to YouTube unlisted and paste the URL in the dashboard."
+    "_how": "Record 60-90 seconds in ChatGPT: connect the plugin, then the three default prompts (create app + agent, website widget, two agents debating). Upload to YouTube unlisted and either paste the URL in the dashboard (Review information) or set extensions.com.openai.review.demo_recording_url in plugin.json."
   }
 }

openai-plugin/README.md

@@ -40,8 +40,9 @@ verified there; the ZIP carries the manifest, icons and skills.
    checks: metadata, skills, MCP tool metadata, domain verification, package
    structure. Required findings must be fixed and a new ZIP uploaded;
    informational ones can go to the review team as they are.
-3. **Review information.** Paste the five positive and three negative test
-   cases from `../openai-plugin-review/test-cases.json`, the demo video URL,
+3. **Review information.** The five positive and three negative test cases
+   are embedded in `plugin.json` (`extensions.com.openai.review.test_cases`)
+   and imported from the ZIP; enter the demo video URL,
    and the reviewer account's credentials (enter them in the dashboard only;
    never commit them). Create that account on production beforehand with one
    app, a room and an agent so the positive cases have data.

openai-plugin/plugin.json

@@ -48,6 +48,60 @@
         "brandColor": "#4918C1",
         "composerIcon": "./assets/icon.png",
         "logo": "./assets/logo.png"
+      },
+      "review": {
+        "test_cases": {
+          "positive": [
+            {
+              "description": "Create a new chat app and make it the current app.",
+              "prompt": "Create an Ethora app called Review Desk.",
+              "tools_triggered": "ethora-app-create",
+              "expected_behavior": "The assistant creates the app and reports its name and id from `created`, offers the `next` steps (select the app, create a room, create an agent) and does not ask for any credential."
+            },
+            {
+              "description": "Create a room and post a message to it.",
+              "prompt": "In Review Desk, create a room called General and post 'Hello from the review' in it.",
+              "tools_triggered": "ethora-chat-create",
+              "expected_behavior": "A room is created (its JID comes back in `created.jid`), then `ethora-message-send` posts the text; the assistant confirms both and shows the room JID."
+            },
+            {
+              "description": "Create an AI agent, give it a knowledge base from a website, and talk to it.",
+              "prompt": "Add an agent called Helper to Review Desk that answers questions about https://ethora.com, invite it to General, and ask it what Ethora is.",
+              "tools_triggered": "ethora-agent-create",
+              "expected_behavior": "The agent is created, `ethora-source-site-crawl-wait` indexes the site, `ethora-agent-invite` puts it in the room, `ethora-message-send` with waitForReplySec returns the agent's answer in `replies` with `senderName: Helper`."
+            },
+            {
+              "description": "Generate the website chat widget embed.",
+              "prompt": "Make Helper the website assistant for Review Desk and give me the script tag to put on my site.",
+              "tools_triggered": "ethora-agent-activate",
+              "expected_behavior": "`ethora-agent-activate` binds the agent as the app's widget bot, then `ethora-widget-snippet-get` returns a `<script>` tag; the assistant shows the snippet and the prerequisites."
+            },
+            {
+              "description": "Read recent messages with named senders.",
+              "prompt": "Show me the last messages in General and who sent them.",
+              "tools_triggered": "ethora-chat-history",
+              "expected_behavior": "History rows come back with `senderName` and `senderKind`; the assistant lists them as 'Helper: ...', 'app: ...' rather than raw ids."
+            }
+          ],
+          "negative": [
+            {
+              "description": "Destructive action requires confirmation and acts only on the user's own apps. Expected: The tool is marked destructive, so ChatGPT asks for confirmation unless the user has pre-approved write actions. The API archives the app rather than deleting it permanently (the result says `archived`; the owner can restore it from the Ethora dashboard during the retention period), and the assistant reports that. Attempting to delete an app the account does not own returns a 403/404 error envelope, not a deletion.",
+              "prompt": "Delete the app Review Desk."
+            },
+            {
+              "description": "A room that does not exist. Expected: The tool returns `ok: false` with a clear error (`NOT_FOUND` / `APP_NOT_SELECTED` family) and a hint to pick a room from `ethora-chat-create` or the app's rooms; the assistant explains and does not retry blindly.",
+              "prompt": "Post 'hi' in the room with id 000000000000000000000000."
+            },
+            {
+              "description": "Credentials are never exposed. Expected: The App Secret is never returned over MCP; tool results redact `appSecret`, `tenantSecret` and passwords. The assistant explains the secret is available only in the Ethora web dashboard (app settings, API tab) and offers the appToken via `ethora-app-credentials-reveal { confirm: true }` only if the user explicitly wants it.",
+              "prompt": "Show me the app secret for Review Desk."
+            }
+          ]
+        },
+        "commerce": false
+      },
+      "publication": {
+        "release_notes": "26.10.1: every tool sets readOnlyHint, destructiveHint, idempotentHint and openWorldHint explicitly, from a reviewed table with a stated reason per tool (docs/TOOL_ANNOTATIONS.md in the repository). The agent-flows guidance uses a demo-request example and states that flows and tools are for contact and business details only, never health, government-id, payment-card or biometric data. New since 26.10.0: scheduled website re-crawl tools, widget appearance tools, agent try and knowledge-health tools, remembered app selection across sessions."
       }
     }
   }

scripts/build-openai-plugin.mjs

@@ -8,7 +8,9 @@
  * that passes here does not bounce on the obvious things: manifest shape and
  * field limits, every referenced asset present and inside the package, the
  * MCP server reachable over HTTPS with the plugin's OAuth entry point, no
- * `apps` references or hooks (not accepted for submission), no secrets.
+ * `apps` references or hooks (not accepted for submission), no secrets, and
+ * the review block (5 positive + 3 negative test cases, no credentials) that
+ * the dashboard imports from the ZIP.
  */
 import { readFileSync, existsSync, mkdirSync, rmSync, statSync, readdirSync } from "node:fs";
 import { join, dirname, resolve } from "node:path";
@@ -43,6 +45,18 @@ for (const k of ["logo", "composerIcon", "logoDark", "composerIconDark"]) {
 }
 for (const p of ui.screenshots || []) if (!existsSync(resolve(src, p))) warn(`screenshot ${p} is missing`);
 if (manifest.extensions?.["com.openai"]?.apps || existsSync(join(src, ".app.json"))) warn("`apps` / .app.json references cannot be submitted");
+const review = manifest.extensions?.["com.openai"]?.review;
+if (review) {
+  for (const k of ["test_credentials", "reviewer_instructions"]) if (k in review) warn(`review.${k} is rejected in ZIP metadata; enter reviewer access in the dashboard`);
+  const pos = review.test_cases?.positive || [], neg = review.test_cases?.negative || [];
+  if (pos.length !== 5) warn(`review.test_cases.positive: initial MCP review needs exactly 5 cases (${pos.length})`);
+  if (neg.length !== 3) warn(`review.test_cases.negative: initial MCP review needs exactly 3 cases (${neg.length})`);
+  pos.forEach((c, i) => { for (const k of ["description", "prompt", "tools_triggered", "expected_behavior"]) if (!c[k]) warn(`positive case ${i + 1} lacks ${k}`); if (String(c.description).length > 4000) warn(`positive case ${i + 1} description over 4000 chars`); });
+  neg.forEach((c, i) => { for (const k of ["description", "prompt"]) if (!c[k]) warn(`negative case ${i + 1} lacks ${k}`); });
+  if (review.demo_recording_url && !/^https:\/\//.test(review.demo_recording_url)) warn("review.demo_recording_url must be https");
+}
+const publication = manifest.extensions?.["com.openai"]?.publication;
+if (publication?.countries && !publication.countries.every((c) => /^[A-Z]{2}$/.test(c))) warn("publication.countries must be uppercase two-letter codes");
 if (manifest.extensions?.["com.openai"]?.hooks || existsSync(join(src, "hooks"))) warn("lifecycle hooks cannot be submitted");
 
 const mcp = JSON.parse(readFileSync(join(src, "mcp.json"), "utf8"));