diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index 301b75ab12..d0633f8896 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -44,6 +44,7 @@ import type * as httpApiV1_packagePublishRecoveryV1 from "../httpApiV1/packagePu import type * as httpApiV1_packagesV1 from "../httpApiV1/packagesV1.js"; import type * as httpApiV1_promotionsV1 from "../httpApiV1/promotionsV1.js"; import type * as httpApiV1_publishersV1 from "../httpApiV1/publishersV1.js"; +import type * as httpApiV1_searchInsightsV1 from "../httpApiV1/searchInsightsV1.js"; import type * as httpApiV1_shared from "../httpApiV1/shared.js"; import type * as httpApiV1_skillsShCatalogV1 from "../httpApiV1/skillsShCatalogV1.js"; import type * as httpApiV1_skillsV1 from "../httpApiV1/skillsV1.js"; @@ -132,6 +133,7 @@ import type * as lib_reservedHandles from "../lib/reservedHandles.js"; import type * as lib_reservedSlugs from "../lib/reservedSlugs.js"; import type * as lib_retentionPolicy from "../lib/retentionPolicy.js"; import type * as lib_rolloutCapabilities from "../lib/rolloutCapabilities.js"; +import type * as lib_searchInsights from "../lib/searchInsights.js"; import type * as lib_searchRanking from "../lib/searchRanking.js"; import type * as lib_searchText from "../lib/searchText.js"; import type * as lib_securityPrompt from "../lib/securityPrompt.js"; @@ -195,6 +197,8 @@ import type * as rateLimits from "../rateLimits.js"; import type * as retention from "../retention.js"; import type * as rolloutCapabilities from "../rolloutCapabilities.js"; import type * as search from "../search.js"; +import type * as searchInsights from "../searchInsights.js"; +import type * as searchInsightsFixtures from "../searchInsightsFixtures.js"; import type * as searchTestFixtures from "../searchTestFixtures.js"; import type * as securityDataset from "../securityDataset.js"; import type * as securityDatasetNode from "../securityDatasetNode.js"; @@ -271,6 +275,7 @@ declare const fullApi: ApiFromModules<{ "httpApiV1/packagesV1": typeof httpApiV1_packagesV1; "httpApiV1/promotionsV1": typeof httpApiV1_promotionsV1; "httpApiV1/publishersV1": typeof httpApiV1_publishersV1; + "httpApiV1/searchInsightsV1": typeof httpApiV1_searchInsightsV1; "httpApiV1/shared": typeof httpApiV1_shared; "httpApiV1/skillsShCatalogV1": typeof httpApiV1_skillsShCatalogV1; "httpApiV1/skillsV1": typeof httpApiV1_skillsV1; @@ -359,6 +364,7 @@ declare const fullApi: ApiFromModules<{ "lib/reservedSlugs": typeof lib_reservedSlugs; "lib/retentionPolicy": typeof lib_retentionPolicy; "lib/rolloutCapabilities": typeof lib_rolloutCapabilities; + "lib/searchInsights": typeof lib_searchInsights; "lib/searchRanking": typeof lib_searchRanking; "lib/searchText": typeof lib_searchText; "lib/securityPrompt": typeof lib_securityPrompt; @@ -422,6 +428,8 @@ declare const fullApi: ApiFromModules<{ retention: typeof retention; rolloutCapabilities: typeof rolloutCapabilities; search: typeof search; + searchInsights: typeof searchInsights; + searchInsightsFixtures: typeof searchInsightsFixtures; searchTestFixtures: typeof searchTestFixtures; securityDataset: typeof securityDataset; securityDatasetNode: typeof securityDatasetNode; diff --git a/convex/crons.test.ts b/convex/crons.test.ts index de94d7db53..70ab4f3fd9 100644 --- a/convex/crons.test.ts +++ b/convex/crons.test.ts @@ -60,6 +60,10 @@ vi.mock("convex/server", () => ({ vi.mock("./_generated/api", () => ({ internal: { + searchInsights: { + aggregateInternal: Symbol("search-insights-aggregate"), + pruneExpiredInternal: Symbol("search-insights-retention"), + }, canonicalTrending: { materializeInternal: mocks.canonicalTrendingMaterializeRef, pruneExpiredActionInternal: mocks.canonicalTrendingPruneRef, diff --git a/convex/crons.ts b/convex/crons.ts index e170b43884..0d67f1f092 100644 --- a/convex/crons.ts +++ b/convex/crons.ts @@ -5,6 +5,18 @@ import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy"; const crons = cronJobs(); if (process.env.CLAWHUB_DISABLE_CRONS !== "1" && process.env.CLAWHUB_PREVIEW !== "1") { + crons.interval( + "search-insights-aggregate", + { hours: 1 }, + internal.searchInsights.aggregateInternal, + {}, + ); + crons.interval( + "search-insights-retention", + { hours: 24 }, + internal.searchInsights.pruneExpiredInternal, + {}, + ); crons.interval( "github-skill-source-sync", { minutes: 15 }, diff --git a/convex/http.ts b/convex/http.ts index b3b9480c3a..a1c7f1233b 100644 --- a/convex/http.ts +++ b/convex/http.ts @@ -17,6 +17,7 @@ import { searchSkillsHttp, } from "./httpApi"; import { + searchInsightsV1Http, exportSkillsV1Http, exportPluginsV1Http, listBundlePluginsV1Http, @@ -82,6 +83,7 @@ import { skillPresentationAssetHttp } from "./skillPresentationAssetsHttp"; const http = installRateLimitedRoutes(httpRouter()); auth.addHttpRoutes(http); +http.route({ path: "/api/v1/search-insights", method: "GET", handler: searchInsightsV1Http }); http.route({ pathPrefix: "/api/v1/skill-icons/", diff --git a/convex/httpApiV1.ts b/convex/httpApiV1.ts index ad34c3507c..44e7ac014e 100644 --- a/convex/httpApiV1.ts +++ b/convex/httpApiV1.ts @@ -31,6 +31,7 @@ import { promotionsPostRouterV1Handler, } from "./httpApiV1/promotionsV1"; import { createPublisherV1Handler } from "./httpApiV1/publishersV1"; +import { searchInsightsV1Handler } from "./httpApiV1/searchInsightsV1"; import { skillsShCatalogPublicV1Handler, skillsShCatalogTestV1Handler, @@ -61,6 +62,8 @@ import { } from "./httpApiV1/usersV1"; import { whoamiV1Handler } from "./httpApiV1/whoamiV1"; +export const searchInsightsV1Http = httpAction(searchInsightsV1Handler); + export const listPackagesV1Http = httpAction(listPackagesV1Handler); export const listPluginsV1Http = httpAction(listPluginsV1Handler); export const listPluginCategoriesV1Http = httpAction(listPluginCategoriesV1Handler); diff --git a/convex/httpApiV1/packagesV1.ts b/convex/httpApiV1/packagesV1.ts index 2893e766a9..aab6d915df 100644 --- a/convex/httpApiV1/packagesV1.ts +++ b/convex/httpApiV1/packagesV1.ts @@ -1186,7 +1186,7 @@ function compareCatalogItemsForSort( return compareCatalogItems(a, b); } -function compareCatalogSearchEntries(a: CatalogSearchEntry, b: CatalogSearchEntry) { +export function compareCatalogSearchEntries(a: CatalogSearchEntry, b: CatalogSearchEntry) { return ( Number( isCuratedSearchResult({ diff --git a/convex/httpApiV1/searchInsightsV1.ts b/convex/httpApiV1/searchInsightsV1.ts new file mode 100644 index 0000000000..4b198e5b3d --- /dev/null +++ b/convex/httpApiV1/searchInsightsV1.ts @@ -0,0 +1,48 @@ +import { internal } from "../_generated/api"; +import type { ActionCtx } from "../_generated/server"; +import type { SearchInsightArgs } from "../lib/searchInsights"; +import { json, requireApiTokenUserOrResponse, requireModeratorOrResponse, text } from "./shared"; + +export async function searchInsightsV1Handler(ctx: ActionCtx, request: Request) { + const headers = { "Cache-Control": "private, no-store" }; + const auth = await requireApiTokenUserOrResponse(ctx, request, headers); + if (!auth.ok) return auth.response; + const staff = requireModeratorOrResponse(auth.user, headers); + if (!staff.ok) return staff.response; + const params = new URL(request.url).searchParams; + const args: SearchInsightArgs = {}; + const source = params.get("source"); + if (source !== null) { + if (source !== "clawhub-web" && source !== "openclaw-control-ui") + return text("Invalid source", 400, headers); + args.source = source; + } + const intent = params.get("intentKind"); + if (intent !== null) { + if (intent !== "company_product" && intent !== "generic_capability" && intent !== "ambiguous") + return text("Invalid intentKind", 400, headers); + args.intentKind = intent; + } + for (const key of ["endDay", "limit", "window"] as const) { + const value = params.get(key); + if (value === null) continue; + if (!/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) + return text(`Invalid ${key}`, 400, headers); + const number = Number(value); + if (key === "window") { + if (number !== 7 && number !== 30) return text("window must be 7 or 30", 400, headers); + args.window = number; + } else args[key] = number; + } + const officialGap = params.get("officialGap"); + if (officialGap !== null) { + if (officialGap !== "true" && officialGap !== "false") + return text("Invalid officialGap", 400, headers); + args.officialGap = officialGap === "true"; + } + if (args.limit !== undefined && (args.limit < 1 || args.limit > 100)) + return text("limit must be between 1 and 100", 400, headers); + if (args.endDay !== undefined && args.endDay % 86_400_000 !== 0) + return text("endDay must be a UTC day boundary", 400, headers); + return json(await ctx.runAction(internal.searchInsights.getInternal, args), 200, headers); +} diff --git a/convex/lib/retentionPolicy.ts b/convex/lib/retentionPolicy.ts index 87739166ba..a5f71934d9 100644 --- a/convex/lib/retentionPolicy.ts +++ b/convex/lib/retentionPolicy.ts @@ -64,6 +64,33 @@ const ephemeral = ( }); export const RETENTION_POLICIES = { + searchAggregateStates: permanent( + "One ingestion cursor and query-free coverage bounds; no identities.", + ), + searchDailyAggregates: ephemeral("Daily anonymous search facts; no historical log backfill.", { + expirationField: "expirationTime", + expirationIndex: "by_expirationTime", + prune: "searchInsights.pruneExpiredInternal", + retention: "13 calendar months after the UTC day.", + }), + searchClassificationRuns: ephemeral( + "Query-free weekly classification completion/failure status.", + { + expirationField: "expirationTime", + expirationIndex: "by_expirationTime", + prune: "searchInsights.pruneExpiredInternal", + retention: "13 calendar months after the week.", + }, + ), + searchWeeklyClassifications: ephemeral( + "Advisory weekly intent only; never official provenance.", + { + expirationField: "expirationTime", + expirationIndex: "by_expirationTime", + prune: "searchInsights.pruneExpiredInternal", + retention: "13 calendar months after the week.", + }, + ), users: permanent("Canonical user profiles and account state."), authSessions: ephemeral("Convex Auth sessions expire after their total session duration.", { expirationField: "expirationTime", diff --git a/convex/lib/searchInsights.ts b/convex/lib/searchInsights.ts new file mode 100644 index 0000000000..e8c6e43165 --- /dev/null +++ b/convex/lib/searchInsights.ts @@ -0,0 +1,120 @@ +import { v, type Infer } from "convex/values"; + +export const SEARCH_DAY_MS = 86_400_000; +export const SEARCH_INTENT_CONFIDENCE = 0.8; +export const searchInsightSource = v.union( + v.literal("clawhub-web"), + v.literal("openclaw-control-ui"), +); +export const searchIntentKind = v.union( + v.literal("company_product"), + v.literal("generic_capability"), + v.literal("ambiguous"), +); +export const searchInsightArgs = { + endDay: v.optional(v.number()), + includeCurrentResults: v.optional(v.boolean()), + order: v.optional( + v.union(v.literal("searches"), v.literal("change"), v.literal("official-gaps")), + ), + source: v.optional(searchInsightSource), + window: v.optional(v.union(v.literal(7), v.literal(30))), + officialGap: v.optional(v.boolean()), + intentKind: v.optional(searchIntentKind), + limit: v.optional(v.number()), +}; +export const searchClassification = v.object({ + weekStart: v.number(), + weekEnd: v.number(), + query: v.string(), + intentKind: searchIntentKind, + companyProductName: v.optional(v.string()), + confidence: v.number(), + model: v.string(), + modelVersion: v.string(), + processedAt: v.number(), +}); +export type SearchClassification = Infer; +export type SearchInsightSource = Infer; +export type SearchInsightArgs = Infer>; +function argsValidator() { + return v.object(searchInsightArgs); +} +export type SearchInsightRow = { + query: string; + searches7d: number; + searchesPrevious7d: number; + searches30d: number; + officialGaps7d: number; + officialGaps30d: number; + zeroResults7d: number; + change7d: number; + changePercent: number | null; + sources7d: Record; + classification: SearchClassification | null; + companyOpportunity: boolean; + currentResults: SearchCurrentResult[]; + featuredCandidate: SearchCurrentResult | null; + searchUrl: string; +}; +export type SearchInsightReport = { + window: { + endDay: number; + start7d: number; + startPrevious7d: number; + start30d: number; + days: 7 | 30; + }; + source: SearchInsightSource | null; + generatedAt: number; + metadataCheckedAt: number | null; + currentMetadataStatus: "available" | "unavailable"; + coverage: { + dataThrough: number | null; + collectionStartedAt: number | null; + gapStart: number | null; + gapEnd: number | null; + }; + totalQueries: number; + totalSearches7d: number; + sources7d: Record; + truncated: boolean; + classificationStatus: "available" | "partial" | "unavailable"; + classificationRun: { + weekStart: number; + weekEnd: number; + processedAt: number; + expectedQualified: number; + classifiedCount: number; + truncated: boolean; + model: string; + modelVersion: string; + failureCode?: string; + } | null; + rows: SearchInsightRow[]; +}; + +// Calendar months, clamped at month-end (not a fixed 390-day approximation). +export function searchAggregateExpiration(day: number) { + const date = new Date(day); + const target = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 13, 1)); + const lastDay = new Date( + Date.UTC(target.getUTCFullYear(), target.getUTCMonth() + 1, 0), + ).getUTCDate(); + target.setUTCDate(Math.min(date.getUTCDate(), lastDay)); + return target.getTime(); +} +export type SearchCurrentResult = { + name: string; + displayName: string; + summary: string | null; + version: string | null; + url: string; + isOfficial: boolean; + isFeatured: boolean; + eligibleForFeatured: boolean; +}; +export type SearchCurrentResults = { + metadataCheckedAt: number; + rows: Array<{ query: string; results: SearchCurrentResult[] }>; +}; diff --git a/convex/schema.ts b/convex/schema.ts index 5ff9792de2..047fe73fe8 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -7,6 +7,7 @@ import { canonicalTrendingSourceRefValidator, } from "./lib/canonicalTrending"; import { EMBEDDING_DIMENSIONS } from "./lib/embeddings"; +import { searchClassification, searchInsightSource } from "./lib/searchInsights"; const PLATFORM_SKILL_LICENSE = "MIT-0" as const; @@ -4484,7 +4485,63 @@ const skillOwnershipTransfers = defineTable({ .index("by_from_user_status", ["fromUserId", "status"]) .index("by_skill_status", ["skillId", "status"]); +const searchAggregateStates = defineTable({ + key: v.literal("plugin"), + cursor: v.union(v.string(), v.null()), + processedThrough: v.number(), + revision: v.number(), + coverageStart: v.number(), + coverageGapStart: v.optional(v.number()), + coverageGapEnd: v.optional(v.number()), +}).index("by_key", ["key"]); +const searchDailyAggregates = defineTable({ + dayStart: v.number(), + query: v.string(), + source: searchInsightSource, + artifactKind: v.literal("plugin"), + category: v.string(), + intent: v.string(), + searches: v.number(), + officialGaps: v.number(), + zeroResults: v.number(), + expirationTime: v.number(), +}) + .index("by_dayStart_and_source_and_query_and_category_and_intent", [ + "dayStart", + "source", + "query", + "category", + "intent", + ]) + .index("by_source_and_dayStart", ["source", "dayStart"]) + .index("by_expirationTime", ["expirationTime"]); +const searchClassificationRuns = defineTable({ + weekStart: v.number(), + weekEnd: v.number(), + processedAt: v.number(), + status: v.union(v.literal("available"), v.literal("unavailable")), + expectedQualified: v.number(), + classifiedCount: v.number(), + truncated: v.optional(v.boolean()), + model: v.string(), + modelVersion: v.string(), + failureCode: v.optional(v.string()), + expirationTime: v.number(), +}) + .index("by_weekEnd", ["weekEnd"]) + .index("by_expirationTime", ["expirationTime"]); +const searchWeeklyClassifications = defineTable( + searchClassification.extend({ expirationTime: v.number() }), +) + .index("by_query_and_weekEnd", ["query", "weekEnd"]) + .index("by_weekEnd", ["weekEnd"]) + .index("by_expirationTime", ["expirationTime"]); + export default defineSchema({ + searchAggregateStates, + searchDailyAggregates, + searchWeeklyClassifications, + searchClassificationRuns, ...authTables, authSessions, authRefreshTokens, diff --git a/convex/searchInsights.test.ts b/convex/searchInsights.test.ts new file mode 100644 index 0000000000..26b98ae883 --- /dev/null +++ b/convex/searchInsights.test.ts @@ -0,0 +1,415 @@ +/// +/* @vitest-environment edge-runtime */ +import { register as registerRateLimiter } from "@convex-dev/rate-limiter/test"; +import { convexTest } from "convex-test"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { api, internal } from "./_generated/api"; +import { hashToken } from "./lib/tokens"; +import schema from "./schema"; + +const modules = import.meta.glob("./**/*.ts"); +const END = Date.UTC(2026, 8, 8); +const DAY = 86_400_000; +afterEach(() => { + vi.unstubAllEnvs(); + vi.useRealTimers(); +}); + +describe("staff search intelligence report", () => { + it("serves deterministic 7-day, previous-week and 30-day aggregate facts without raw rows", async () => { + const t = convexTest(schema, modules); + const staffId = await t.run(async (ctx) => { + const id = await ctx.db.insert("users", { role: "moderator", handle: "staff" }); + for (const row of [ + { query: "notion", day: 1, source: "clawhub-web" as const, searches: 3, officialGaps: 3 }, + { + query: "notion", + day: 2, + source: "openclaw-control-ui" as const, + searches: 2, + officialGaps: 0, + }, + { query: "notion", day: 8, source: "clawhub-web" as const, searches: 2, officialGaps: 2 }, + { query: "notion", day: 29, source: "clawhub-web" as const, searches: 4, officialGaps: 4 }, + { query: "memory", day: 1, source: "clawhub-web" as const, searches: 5, officialGaps: 5 }, + { + query: "excluded", + day: 31, + source: "clawhub-web" as const, + searches: 10, + officialGaps: 10, + }, + ]) { + await ctx.db.insert("searchDailyAggregates", { + dayStart: END - row.day * DAY, + query: row.query, + source: row.source, + artifactKind: "plugin", + category: "", + intent: "", + searches: row.searches, + officialGaps: row.officialGaps, + zeroResults: 0, + expirationTime: END + 360 * DAY, + }); + } + return id; + }); + const report = await t + .withIdentity({ subject: staffId }) + .action(api.searchInsights.get, { endDay: END }); + expect(report.rows.map((row) => row.query)).toEqual(["memory", "notion"]); + expect(report.rows[1]).toMatchObject({ + query: "notion", + searches7d: 5, + searchesPrevious7d: 2, + searches30d: 11, + change7d: 3, + changePercent: 150, + officialGaps7d: 3, + sources7d: { "clawhub-web": 3, "openclaw-control-ui": 2 }, + classification: null, + }); + expect(report.window).toMatchObject({ endDay: END, start7d: Date.UTC(2026, 8, 1) }); + expect(JSON.stringify(report)).not.toMatch(/staff|userId|_id|_creationTime|observations/); + }); + it("keeps official gaps deterministic while filtering only fresh high-confidence company opportunities", async () => { + const t = convexTest(schema, modules); + const id = await t.run(async (ctx) => { + const userId = await ctx.db.insert("users", { role: "admin" }); + for (const query of ["notion", "memory", "uncertain", "stale"]) { + await ctx.db.insert("searchDailyAggregates", { + dayStart: END - DAY, + query, + source: "clawhub-web", + artifactKind: "plugin", + category: "", + intent: "", + searches: 5, + officialGaps: 4, + zeroResults: 0, + expirationTime: END + DAY * 360, + }); + } + return userId; + }); + await t.mutation(internal.searchInsights.storeClassificationsInternal, { + weekStart: END - 8 * DAY, + weekEnd: END - DAY, + model: "fixture-model", + modelVersion: "v1", + processedAt: END, + expectedQualified: 4, + rows: [ + { + query: "notion", + intentKind: "company_product", + companyProductName: "Notion", + confidence: 0.95, + }, + { query: "memory", intentKind: "generic_capability", confidence: 0.99 }, + { query: "uncertain", intentKind: "company_product", confidence: 0.5 }, + ], + }); + const staff = t.withIdentity({ subject: id }); + const filtered = await staff.action(api.searchInsights.get, { + endDay: END, + officialGap: true, + intentKind: "company_product", + }); + expect(filtered.rows.map((row) => row.query)).toEqual(["notion"]); + expect(filtered.rows[0]).toMatchObject({ + officialGaps7d: 4, + companyOpportunity: true, + classification: { model: "fixture-model", modelVersion: "v1", companyProductName: "Notion" }, + }); + const all = await staff.action(api.searchInsights.get, { endDay: END, officialGap: true }); + expect(all.rows).toHaveLength(4); + expect(all.classificationStatus).toBe("partial"); + }); + it("reports capped weekly classification scope as partial even when the cohort is complete", async () => { + const t = convexTest(schema, modules); + await t.mutation(internal.searchInsights.storeClassificationsInternal, { + weekStart: END - 8 * DAY, + weekEnd: END - DAY, + processedAt: END, + model: "fixture-model", + modelVersion: "v1", + expectedQualified: 1, + truncated: true, + rows: [{ query: "notion", intentKind: "company_product", confidence: 0.95 }], + }); + const report = await t.action(internal.searchInsights.getInternal, { endDay: END }); + expect(report.classificationStatus).toBe("partial"); + expect(report.classificationRun).toMatchObject({ + expectedQualified: 1, + classifiedCount: 1, + truncated: true, + }); + }); + it("materializes each raw fact once across batch retries and advances after new arrivals", async () => { + const t = convexTest(schema, modules); + vi.useFakeTimers(); + vi.setSystemTime(END); + await t.run(async (ctx) => { + for (let i = 0; i < 3; i++) + await ctx.db.insert("pluginSearchObservations", { + normalizedQuery: "notion", + observedAt: END - DAY, + source: "clawhub-web", + artifactKind: "plugin", + resultCount: 2, + officialResultCount: i === 0 ? 1 : 0, + }); + }); + await t.mutation(internal.searchInsights.aggregateInternal, {}); + await t.mutation(internal.searchInsights.aggregateInternal, {}); + let report = await t.action(internal.searchInsights.getInternal, { endDay: END }); + expect(report.rows[0]).toMatchObject({ searches7d: 3, officialGaps7d: 2 }); + await t.run( + async (ctx) => + await ctx.db.insert("pluginSearchObservations", { + normalizedQuery: "notion", + observedAt: END - DAY, + source: "openclaw-control-ui", + artifactKind: "plugin", + resultCount: 0, + officialResultCount: 0, + }), + ); + await t.mutation(internal.searchInsights.aggregateInternal, {}); + report = await t.action(internal.searchInsights.getInternal, { endDay: END }); + expect(report.rows[0]).toMatchObject({ searches7d: 4, officialGaps7d: 3, zeroResults7d: 1 }); + vi.useRealTimers(); + }); + + it("denies anonymous and non-staff reports and serves the same facts over the authenticated HTTP boundary", async () => { + const t = convexTest(schema, modules); + registerRateLimiter(t); + const token = "search-insights-contract-fixture"; + const ids = await t.run(async (ctx) => { + const staff = await ctx.db.insert("users", { role: "moderator" }); + const user = await ctx.db.insert("users", { role: "user" }); + await ctx.db.insert("apiTokens", { + userId: staff, + label: "fixture", + prefix: "fixture", + tokenHash: await hashToken(token), + createdAt: END, + }); + return { staff, user }; + }); + await expect(t.action(api.searchInsights.get, { endDay: END })).rejects.toThrow("Unauthorized"); + await expect( + t.withIdentity({ subject: ids.user }).action(api.searchInsights.get, { endDay: END }), + ).rejects.toThrow("Forbidden"); + expect((await t.fetch("/api/v1/search-insights")).status).toBe(401); + const response = await t.fetch( + `/api/v1/search-insights?endDay=${END}&source=clawhub-web&window=30&officialGap=true`, + { + headers: { Authorization: `Bearer ${token}` }, + }, + ); + expect(response.status).toBe(200); + const report = await response.json(); + const dashboard = await t.withIdentity({ subject: ids.staff }).action(api.searchInsights.get, { + endDay: END, + source: "clawhub-web", + window: 30, + officialGap: true, + }); + expect(report).toEqual({ ...dashboard, generatedAt: report.generatedAt }); + expect(response.headers.get("cache-control")).toContain("no-store"); + }); + + it("keeps visible suspicious metadata for intent but recommends only clean public installs", async () => { + const t = convexTest(schema, modules); + await t.run(async (ctx) => { + const ownerUserId = await ctx.db.insert("users", { handle: "fixture" }); + for (const [name, channel, scanStatus] of [ + ["notion-clean", "community", "clean"], + ["notion-private", "private", "clean"], + ["notion-unsafe", "community", "suspicious"], + ] as const) { + const packageId = await ctx.db.insert("packages", { + name, + normalizedName: name, + displayName: name, + ownerUserId, + family: "code-plugin", + channel, + isOfficial: false, + tags: {}, + scanStatus, + stats: { downloads: 0, installs: 0, stars: 0, versions: 1 }, + createdAt: END, + updatedAt: END, + }); + const releaseId = await ctx.db.insert("packageReleases", { + packageId, + version: "1.0.0", + changelog: "fixture", + distTags: ["latest"], + files: [ + { + path: "index.js", + size: 1, + storageId: await ctx.storage.store(new Blob(["x"])), + sha256: "a".repeat(64), + }, + ], + integritySha256: "a".repeat(64), + verification: { scanStatus, tier: "structural", scope: "artifact-only" }, + createdBy: ownerUserId, + createdAt: END, + }); + await ctx.db.patch(packageId, { latestReleaseId: releaseId, tags: { latest: releaseId } }); + await ctx.db.insert("packageSearchDigest", { + packageId, + name, + normalizedName: name, + displayName: name, + ownerUserId, + family: "code-plugin", + channel, + isOfficial: false, + latestVersion: "1.0.0", + scanStatus, + stats: { downloads: 0, installs: 0, stars: 0, versions: 1 }, + createdAt: END, + updatedAt: END, + }); + } + }); + const results = await t.action(internal.searchInsights.readCurrentResultsInternal, { + queries: ["notion"], + }); + expect(results.rows[0].results.map((item) => item.name)).toEqual([ + "notion-clean", + "notion-unsafe", + ]); + expect(results.rows[0].results[1]).toMatchObject({ + name: "notion-unsafe", + eligibleForFeatured: false, + }); + expect(results.rows[0].results[0]).toMatchObject({ + eligibleForFeatured: true, + isOfficial: false, + version: "1.0.0", + }); + expect(JSON.stringify(results)).not.toMatch(/ownerUserId|storageId|_id|integritySha256/); + }); + + it("expires daily totals after exactly 13 calendar months at clamped month-end", async () => { + const t = convexTest(schema, modules); + vi.useFakeTimers(); + vi.setSystemTime(Date.UTC(2026, 8, 30)); + await t.run(async (ctx) => { + for (const [normalizedQuery, observedAt] of [ + ["expired", Date.UTC(2025, 7, 31)], + ["retained", Date.UTC(2025, 8, 1)], + ] as const) { + await ctx.db.insert("pluginSearchObservations", { + normalizedQuery, + observedAt, + source: "clawhub-web", + artifactKind: "plugin", + resultCount: 0, + officialResultCount: 0, + }); + } + }); + await t.mutation(internal.searchInsights.aggregateInternal, {}); + await t.mutation(internal.searchInsights.pruneExpiredInternal, {}); + const report = await t.action(internal.searchInsights.getInternal, { + endDay: Date.UTC(2025, 8, 2), + includeCurrentResults: false, + }); + expect(report.rows.map((row) => row.query)).toEqual(["retained"]); + }); + + it("reports coverage loss after a 30-day aggregation outage without retaining raw searches", async () => { + const t = convexTest(schema, modules); + vi.useFakeTimers(); + vi.setSystemTime(END); + await t.run( + async (ctx) => + await ctx.db.insert("searchAggregateStates", { + key: "plugin", + cursor: null, + processedThrough: END - 40 * DAY, + revision: 1, + coverageStart: END - 50 * DAY, + }), + ); + await t.mutation(internal.searchInsights.aggregateInternal, {}); + const report = await t.action(internal.searchInsights.getInternal, { endDay: END }); + expect(report.coverage).toMatchObject({ + gapStart: END - 40 * DAY, + gapEnd: END - 30 * DAY, + dataThrough: END, + }); + }); + + it("hides previous successful intent classifications when this week's provider fails", async () => { + const t = convexTest(schema, modules); + await t.mutation(internal.searchInsights.storeClassificationsInternal, { + weekStart: END - 7 * DAY, + weekEnd: END, + processedAt: END, + model: "fixture", + modelVersion: "v1", + rows: [{ query: "notion", intentKind: "company_product", confidence: 0.99 }], + }); + await t.mutation(internal.searchInsights.storeClassificationsInternal, { + weekStart: END - 7 * DAY, + weekEnd: END, + processedAt: END + 1, + model: "fixture", + modelVersion: "v1", + status: "unavailable", + expectedQualified: 1, + failureCode: "provider_unavailable", + rows: [], + }); + const report = await t.action(internal.searchInsights.getInternal, { endDay: END }); + expect(report.classificationStatus).toBe("unavailable"); + expect(report.classificationRun).toMatchObject({ + failureCode: "provider_unavailable", + classifiedCount: 0, + }); + }); + it("includes demand that fell to zero when ranking the largest week-over-week movers", async () => { + const t = convexTest(schema, modules); + await t.run(async (ctx) => { + for (const [query, day, searches] of [ + ["vanished", 8, 20], + ["active", 1, 5], + ] as const) { + await ctx.db.insert("searchDailyAggregates", { + query, + dayStart: END - day * DAY, + source: "clawhub-web", + artifactKind: "plugin", + category: "", + intent: "", + searches, + officialGaps: searches, + zeroResults: 0, + expirationTime: END + DAY, + }); + } + }); + const report = await t.action(internal.searchInsights.getInternal, { + endDay: END, + order: "change", + includeCurrentResults: false, + }); + expect(report.rows[0]).toMatchObject({ + query: "vanished", + searches7d: 0, + change7d: -20, + changePercent: -100, + }); + }); +}); diff --git a/convex/searchInsights.ts b/convex/searchInsights.ts new file mode 100644 index 0000000000..a3111c1b32 --- /dev/null +++ b/convex/searchInsights.ts @@ -0,0 +1,535 @@ +import { getPage, type IndexKey } from "convex-helpers/server/pagination"; +import { paginationOptsValidator, type FunctionReturnType } from "convex/server"; +import { v } from "convex/values"; +import { internal } from "./_generated/api"; +import type { Doc } from "./_generated/dataModel"; +import type { ActionCtx } from "./_generated/server"; +import { action, internalAction, internalQuery, internalMutation } from "./functions"; +import { compareCatalogSearchEntries } from "./httpApiV1/packagesV1"; +import { assertModerator, requireUserFromAction } from "./lib/access"; +import { + getPackageDownloadSecurityBlock, + resolvePackageReleaseScanStatus, +} from "./lib/packageSecurity"; +import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy"; +import { + SEARCH_DAY_MS, + SEARCH_INTENT_CONFIDENCE, + searchAggregateExpiration, + searchIntentKind, + searchInsightArgs, + searchInsightSource, + type SearchInsightArgs, + type SearchInsightReport, + type SearchInsightRow, + type SearchCurrentResults, + type SearchCurrentResult, +} from "./lib/searchInsights"; + +export const get = action({ + args: searchInsightArgs, + handler: async (ctx, args): Promise => { + const { user } = await requireUserFromAction(ctx); + assertModerator(user); + return await readReport(ctx, args); + }, +}); +export const getInternal = internalAction({ + args: searchInsightArgs, + handler: readReport, +}); +export const listDailyInternal = internalQuery({ + args: { + start: v.number(), + end: v.number(), + source: v.optional(searchInsightSource), + paginationOpts: paginationOptsValidator, + }, + handler: async (ctx, args) => { + const rows = ctx.db.query("searchDailyAggregates"); + return await ( + args.source + ? rows.withIndex("by_source_and_dayStart", (q) => + q.eq("source", args.source!).gte("dayStart", args.start).lt("dayStart", args.end), + ) + : rows.withIndex("by_dayStart_and_source_and_query_and_category_and_intent", (q) => + q.gte("dayStart", args.start).lt("dayStart", args.end), + ) + ).paginate(args.paginationOpts); + }, +}); +async function readReport(ctx: ActionCtx, args: SearchInsightArgs): Promise { + const coverageState: Doc<"searchAggregateStates"> | null = await ctx.runQuery( + internal.searchInsights.getAggregateStateInternal, + {}, + ); + const endDay = args.endDay ?? Math.floor(Date.now() / SEARCH_DAY_MS) * SEARCH_DAY_MS; + if (!Number.isSafeInteger(endDay) || endDay % SEARCH_DAY_MS !== 0) + throw new Error("endDay must be a UTC day boundary"); + const days = args.window ?? 7; + const limit = args.limit ?? 50; + if (!Number.isInteger(limit) || limit < 1 || limit > 100) + throw new Error("limit must be between 1 and 100"); + const window = { + endDay, + start7d: endDay - 7 * SEARCH_DAY_MS, + startPrevious7d: endDay - 14 * SEARCH_DAY_MS, + start30d: endDay - 30 * SEARCH_DAY_MS, + days, + }; + const facts = new Map(); + let cursor: string | null = null; + for (let batch = 0; ; batch++) { + if (batch >= 200) + throw new Error("Search aggregate scan budget exceeded; narrow the source filter"); + const page: { page: Doc<"searchDailyAggregates">[]; isDone: boolean; continueCursor: string } = + await ctx.runQuery(internal.searchInsights.listDailyInternal, { + start: window.start30d, + end: endDay, + source: args.source, + paginationOpts: { + cursor, + numItems: 500, + maximumRowsRead: 500, + maximumBytesRead: 1_000_000, + }, + }); + for (const daily of page.page) { + const row = facts.get(daily.query) ?? { + query: daily.query, + searches7d: 0, + searchesPrevious7d: 0, + searches30d: 0, + officialGaps7d: 0, + officialGaps30d: 0, + zeroResults7d: 0, + change7d: 0, + changePercent: null, + sources7d: { "clawhub-web": 0, "openclaw-control-ui": 0 }, + classification: null, + companyOpportunity: false, + currentResults: [], + featuredCandidate: null, + searchUrl: `/plugins?q=${encodeURIComponent(daily.query)}`, + }; + row.searches30d += daily.searches; + row.officialGaps30d += daily.officialGaps; + if (daily.dayStart >= window.start7d) { + row.searches7d += daily.searches; + row.officialGaps7d += daily.officialGaps; + row.zeroResults7d += daily.zeroResults; + row.sources7d[daily.source] += daily.searches; + } else if (daily.dayStart >= window.startPrevious7d) row.searchesPrevious7d += daily.searches; + facts.set(daily.query, row); + } + if (page.isDone) break; + cursor = page.continueCursor; + } + let rows = [...facts.values()].filter( + (row) => + (days === 7 ? row.searches7d : row.searches30d) > 0 || + (args.order === "change" && row.searchesPrevious7d > 0), + ); + const classificationRun: Doc<"searchClassificationRuns"> | null = await ctx.runQuery( + internal.searchInsights.getClassificationRunInternal, + { endDay }, + ); + const classifications = new Map>(); + if (classificationRun?.status === "available") { + const batch: Doc<"searchWeeklyClassifications">[] = await ctx.runQuery( + internal.searchInsights.getClassificationsInternal, + { weekEnd: classificationRun.weekEnd }, + ); + for (const entry of batch) classifications.set(entry.query, entry); + } + for (const row of rows) { + const entry = classifications.get(row.query); + if (entry) { + const { _id, _creationTime, expirationTime: _expirationTime, ...classification } = entry; + row.classification = classification; + } + row.companyOpportunity = + (days === 7 ? row.officialGaps7d : row.officialGaps30d) >= 3 && + row.classification?.intentKind === "company_product" && + row.classification.confidence >= SEARCH_INTENT_CONFIDENCE; + row.change7d = row.searches7d - row.searchesPrevious7d; + row.changePercent = row.searchesPrevious7d + ? (100 * row.change7d) / row.searchesPrevious7d + : null; + } + rows = rows.filter( + (row) => + (!args.officialGap || (days === 7 ? row.officialGaps7d : row.officialGaps30d) > 0) && + (!args.intentKind || + (args.intentKind === "company_product" + ? row.companyOpportunity + : row.classification?.intentKind === args.intentKind)), + ); + const rank = (row: SearchInsightRow) => + args.order === "change" + ? Math.abs(row.change7d) + : args.order === "official-gaps" + ? days === 7 + ? row.officialGaps7d + : row.officialGaps30d + : days === 7 + ? row.searches7d + : row.searches30d; + rows.sort((a, b) => rank(b) - rank(a) || (a.query < b.query ? -1 : a.query > b.query ? 1 : 0)); + const sources7d = { "clawhub-web": 0, "openclaw-control-ui": 0 }; + let totalSearches7d = 0; + for (const row of rows) { + totalSearches7d += row.searches7d; + sources7d["clawhub-web"] += row.sources7d["clawhub-web"]; + sources7d["openclaw-control-ui"] += row.sources7d["openclaw-control-ui"]; + } + const run = classificationRun + ? { + weekStart: classificationRun.weekStart, + weekEnd: classificationRun.weekEnd, + processedAt: classificationRun.processedAt, + expectedQualified: classificationRun.expectedQualified, + classifiedCount: classificationRun.classifiedCount, + truncated: classificationRun.truncated ?? false, + model: classificationRun.model, + modelVersion: classificationRun.modelVersion, + ...(classificationRun.failureCode ? { failureCode: classificationRun.failureCode } : {}), + } + : null; + const classificationStatus = + !classificationRun || classificationRun.status === "unavailable" + ? "unavailable" + : classificationRun.truncated || + classificationRun.classifiedCount < classificationRun.expectedQualified + ? "partial" + : "available"; + const selected = rows.slice(0, limit); + let metadataCheckedAt: number | null = null; + if (args.includeCurrentResults !== false && selected.length) { + try { + const current = await readCurrentResults(ctx, { queries: selected.map((row) => row.query) }); + metadataCheckedAt = current.metadataCheckedAt; + for (const row of selected) { + row.currentResults = current.rows.find((entry) => entry.query === row.query)?.results ?? []; + row.featuredCandidate = + row.currentResults.find((entry) => entry.eligibleForFeatured) ?? null; + } + } catch { + // Catalog lookup failure must not discard deterministic search counts. + } + } + const latestCoverage: Doc<"searchAggregateStates"> | null = await ctx.runQuery( + internal.searchInsights.getAggregateStateInternal, + {}, + ); + if (coverageState?.revision !== latestCoverage?.revision) + throw new Error("Search aggregates changed during report; refresh to retry"); + const coverage = { + dataThrough: coverageState?.processedThrough ?? null, + collectionStartedAt: coverageState?.coverageStart ?? null, + gapStart: coverageState?.coverageGapStart ?? null, + gapEnd: coverageState?.coverageGapEnd ?? null, + }; + return { + coverage, + metadataCheckedAt, + currentMetadataStatus: metadataCheckedAt === null ? "unavailable" : "available", + totalSearches7d, + sources7d, + truncated: rows.length > limit, + classificationStatus, + classificationRun: run, + window, + source: args.source ?? null, + generatedAt: Date.now(), + totalQueries: rows.length, + rows: selected, + }; +} + +export const storeClassificationsInternal = internalMutation({ + args: { + weekStart: v.number(), + weekEnd: v.number(), + processedAt: v.number(), + model: v.string(), + modelVersion: v.string(), + status: v.optional(v.union(v.literal("available"), v.literal("unavailable"))), + expectedQualified: v.optional(v.number()), + truncated: v.optional(v.boolean()), + failureCode: v.optional(v.string()), + rows: v.array( + v.object({ + query: v.string(), + intentKind: searchIntentKind, + companyProductName: v.optional(v.string()), + confidence: v.number(), + }), + ), + }, + handler: async (ctx, args) => { + if ( + args.rows.length > 100 || + args.weekEnd - args.weekStart !== 7 * SEARCH_DAY_MS || + args.weekEnd % SEARCH_DAY_MS !== 0 || + !Number.isSafeInteger(args.weekEnd) || + !Number.isSafeInteger(args.processedAt) || + args.model.length > 120 || + args.modelVersion.length > 120 || + (args.failureCode?.length ?? 0) > 80 + ) + throw new Error("Invalid bounded classification batch"); + const expectedQualified = args.expectedQualified ?? args.rows.length; + if ( + !Number.isInteger(expectedQualified) || + expectedQualified < args.rows.length || + expectedQualified > 100 + ) + throw new Error("Invalid qualified query count"); + const expirationTime = searchAggregateExpiration(args.weekEnd); + const status = args.status ?? "available"; + if (status === "unavailable" && args.rows.length) + throw new Error("Unavailable classification cannot include rows"); + const previous = await ctx.db + .query("searchWeeklyClassifications") + .withIndex("by_weekEnd", (q) => q.eq("weekEnd", args.weekEnd)) + .take(101); + if (previous.length > 100) throw new Error("Classification week exceeds bounded batch"); + for (const row of previous) await ctx.db.delete(row._id); + const queries = new Set(); + for (const row of args.rows) { + if ( + !row.query || + row.query.length > 256 || + row.query !== row.query.trim().toLowerCase().replace(/\s+/g, " ") || + queries.has(row.query) || + !Number.isFinite(row.confidence) || + row.confidence < 0 || + row.confidence > 1 || + (row.companyProductName?.length ?? 0) > 120 + ) + throw new Error("Invalid classification row"); + queries.add(row.query); + const existing = await ctx.db + .query("searchWeeklyClassifications") + .withIndex("by_query_and_weekEnd", (q) => + q.eq("query", row.query).eq("weekEnd", args.weekEnd), + ) + .unique(); + const doc = { + ...row, + weekStart: args.weekStart, + weekEnd: args.weekEnd, + processedAt: args.processedAt, + model: args.model, + modelVersion: args.modelVersion, + expirationTime, + }; + if (existing) await ctx.db.replace(existing._id, doc); + else await ctx.db.insert("searchWeeklyClassifications", doc); + } + const existing = await ctx.db + .query("searchClassificationRuns") + .withIndex("by_weekEnd", (q) => q.eq("weekEnd", args.weekEnd)) + .unique(); + const run = { + weekStart: args.weekStart, + weekEnd: args.weekEnd, + processedAt: args.processedAt, + model: args.model, + modelVersion: args.modelVersion, + expectedQualified, + classifiedCount: args.rows.length, + truncated: args.truncated ?? false, + status, + expirationTime, + ...(args.failureCode ? { failureCode: args.failureCode } : {}), + }; + if (existing) await ctx.db.replace(existing._id, run); + else await ctx.db.insert("searchClassificationRuns", run); + return { status, classifiedCount: args.rows.length }; + }, +}); +export const getClassificationRunInternal = internalQuery({ + args: { endDay: v.number() }, + handler: async (ctx, args) => + await ctx.db + .query("searchClassificationRuns") + .withIndex("by_weekEnd", (q) => + q.gt("weekEnd", args.endDay - 7 * SEARCH_DAY_MS).lte("weekEnd", args.endDay), + ) + .order("desc") + .first(), +}); +export const getClassificationsInternal = internalQuery({ + args: { weekEnd: v.number() }, + handler: async (ctx, args) => + await ctx.db + .query("searchWeeklyClassifications") + .withIndex("by_weekEnd", (q) => q.eq("weekEnd", args.weekEnd)) + .take(100), +}); + +export const aggregateInternal = internalMutation({ + args: {}, + handler: async (ctx) => { + const now = Date.now(); + const state = await ctx.db + .query("searchAggregateStates") + .withIndex("by_key", (q) => q.eq("key", "plugin")) + .unique(); + // The cursor and every bucket delta commit together. Retrying a batch cannot double-count it. + const page = await getPage(ctx, { + table: "pluginSearchObservations", + targetMaxRows: 200, + absoluteMaxRows: 200, + startIndexKey: state?.cursor ? (JSON.parse(state.cursor) as IndexKey) : undefined, + }); + for (const raw of page.page) { + const dayStart = Math.floor(raw.observedAt / SEARCH_DAY_MS) * SEARCH_DAY_MS; + const category = raw.category ?? ""; + const intent = raw.topic ?? ""; + const existing = await ctx.db + .query("searchDailyAggregates") + .withIndex("by_dayStart_and_source_and_query_and_category_and_intent", (q) => + q + .eq("dayStart", dayStart) + .eq("source", raw.source) + .eq("query", raw.normalizedQuery) + .eq("category", category) + .eq("intent", intent), + ) + .unique(); + const counts = { + searches: (existing?.searches ?? 0) + 1, + officialGaps: (existing?.officialGaps ?? 0) + (raw.officialResultCount === 0 ? 1 : 0), + zeroResults: (existing?.zeroResults ?? 0) + (raw.resultCount === 0 ? 1 : 0), + }; + if (existing) await ctx.db.patch(existing._id, counts); + else + await ctx.db.insert("searchDailyAggregates", { + dayStart, + source: raw.source, + query: raw.normalizedQuery, + category, + intent, + artifactKind: "plugin", + ...counts, + expirationTime: searchAggregateExpiration(dayStart), + }); + } + const missedBefore = now - 30 * SEARCH_DAY_MS; + const coverageGap = + state && state.processedThrough < missedBefore + ? { + coverageGapStart: state.coverageGapStart ?? state.processedThrough, + coverageGapEnd: missedBefore, + } + : {}; + const next = { + key: "plugin" as const, + cursor: page.page.length ? JSON.stringify(page.indexKeys.at(-1)) : (state?.cursor ?? null), + processedThrough: !page.hasMore + ? now + : (state?.processedThrough ?? page.page[0]?.observedAt ?? now), + revision: (state?.revision ?? 0) + 1, + coverageStart: state?.coverageStart ?? page.page[0]?.observedAt ?? now, + ...coverageGap, + }; + if (state) await ctx.db.patch(state._id, next); + else await ctx.db.insert("searchAggregateStates", next); + if (page.hasMore) + await ctx.scheduler.runAfter(0, internal.searchInsights.aggregateInternal, {}); + return { processed: page.page.length, hasMore: page.hasMore }; + }, +}); +export const pruneExpiredInternal = internalMutation({ + args: {}, + handler: async (ctx) => { + const now = Date.now(); + let deleted = 0; + let hasMore = false; + for (const table of [ + "searchDailyAggregates", + "searchWeeklyClassifications", + "searchClassificationRuns", + ] as const) { + const rows = await ctx.db + .query(table) + .withIndex("by_expirationTime", (q) => q.lte("expirationTime", now)) + .take(RETENTION_STANDARD_BATCH_SIZE); + for (const row of rows) await ctx.db.delete(row._id); + deleted += rows.length; + hasMore ||= rows.length === RETENTION_STANDARD_BATCH_SIZE; + } + if (hasMore) await ctx.scheduler.runAfter(0, internal.searchInsights.pruneExpiredInternal, {}); + return { deleted, hasMore }; + }, +}); + +export const readCurrentResultsInternal = internalAction({ + args: { queries: v.array(v.string()) }, + handler: readCurrentResults, +}); +async function readCurrentResults( + ctx: ActionCtx, + args: { queries: string[] }, +): Promise { + if (args.queries.length > 100 || args.queries.some((query) => !query || query.length > 256)) + throw new Error("Maximum 100 bounded queries"); + const rows: SearchCurrentResults["rows"] = []; + for (const query of args.queries) { + // Current public metadata is separate from historical visible-result facts. No attribution marker. + const groups: FunctionReturnType[] = + await Promise.all( + (["code-plugin", "bundle-plugin"] as const).map((family) => + ctx.runQuery(internal.packages.searchForViewerInternal, { + query, + family, + limit: 3, + }), + ), + ); + const candidates = groups.flat().sort(compareCatalogSearchEntries).slice(0, 3); + const results: SearchCurrentResult[] = []; + for (const { package: pkg } of candidates) { + if (pkg.channel === "private") continue; + const detail: FunctionReturnType = + pkg.latestVersion + ? await ctx.runQuery(internal.packages.getVersionByNameForViewerInternal, { + name: pkg.name, + version: pkg.latestVersion, + }) + : null; + const release = detail?.version; + const installableAndClean = Boolean( + release && + resolvePackageReleaseScanStatus(release) === "clean" && + !getPackageDownloadSecurityBlock(release) && + (release.files.length || release.clawpackStorageId), + ); + const isFeatured = pkg.featuredAt !== undefined; + results.push({ + name: pkg.name, + displayName: pkg.displayName.slice(0, 120), + summary: pkg.summary?.slice(0, 500) ?? null, + version: pkg.latestVersion, + url: `/plugins/${encodeURIComponent(pkg.name)}`, + isOfficial: pkg.isOfficial === true, + isFeatured, + eligibleForFeatured: installableAndClean && !isFeatured, + }); + } + rows.push({ query, results }); + } + return { metadataCheckedAt: Date.now(), rows }; +} + +export const getAggregateStateInternal = internalQuery({ + args: {}, + handler: async (ctx) => + await ctx.db + .query("searchAggregateStates") + .withIndex("by_key", (q) => q.eq("key", "plugin")) + .unique(), +}); diff --git a/convex/searchInsightsFixtures.ts b/convex/searchInsightsFixtures.ts new file mode 100644 index 0000000000..1cb9f873ac --- /dev/null +++ b/convex/searchInsightsFixtures.ts @@ -0,0 +1,181 @@ +import { v } from "convex/values"; +import { internal } from "./_generated/api"; +import type { Doc, Id } from "./_generated/dataModel"; +import { internalAction, internalMutation } from "./functions"; +import { SEARCH_DAY_MS } from "./lib/searchInsights"; + +function assertLocal() { + const origin = process.env.CONVEX_SITE_URL ?? ""; + if (!/^http:\/\/(127\.0\.0\.1|localhost):\d+$/.test(origin)) + throw new Error("Search fixtures require a disposable local backend"); +} + +export const seed = internalAction({ + args: { + state: v.union(v.literal("empty"), v.literal("typical"), v.literal("dense")), + apiTokenHash: v.optional(v.string()), + }, + handler: async (ctx, args): Promise<{ state: string; endDay: number; seededRaw: number }> => { + assertLocal(); + const storageId = await ctx.storage.store( + new Blob(["export default {};"], { type: "text/javascript" }), + ); + const result: { state: string; endDay: number; seededRaw: number } = await ctx.runMutation( + internal.searchInsightsFixtures.seedInternal, + { ...args, storageId }, + ); + await ctx.runMutation(internal.searchInsights.aggregateInternal, {}); + if (args.state !== "empty") + await ctx.runMutation(internal.searchInsights.storeClassificationsInternal, { + weekStart: result.endDay - 7 * SEARCH_DAY_MS, + weekEnd: result.endDay, + processedAt: Date.now(), + model: "local-fixture", + modelVersion: "v1", + expectedQualified: args.state === "dense" ? 24 : 4, + rows: [ + { + query: "notion", + intentKind: "company_product", + companyProductName: "Notion", + confidence: 0.96, + }, + { + query: "google drive", + intentKind: "company_product", + companyProductName: "Google Drive", + confidence: 0.97, + }, + { query: "memory", intentKind: "generic_capability", confidence: 0.99 }, + { query: "atlas", intentKind: "ambiguous", confidence: 0.5 }, + ], + }); + return result; + }, +}); +export const seedInternal = internalMutation({ + args: { + state: v.union(v.literal("empty"), v.literal("typical"), v.literal("dense")), + storageId: v.id("_storage"), + apiTokenHash: v.optional(v.string()), + }, + handler: async (ctx, args) => { + assertLocal(); + const endDay = Math.floor(Date.now() / SEARCH_DAY_MS) * SEARCH_DAY_MS; + for (const table of [ + "pluginSearchObservations", + "searchDailyAggregates", + "searchAggregateStates", + "searchWeeklyClassifications", + "searchClassificationRuns", + ] as const) { + const rows = await ctx.db.query(table).take(1000); + if (rows.length === 1000) throw new Error("Fixture reset exceeds bounded disposable dataset"); + for (const row of rows) await ctx.db.delete(row._id); + } + let user = await ctx.db + .query("users") + .withIndex("handle", (q) => q.eq("handle", "local")) + .unique(); + let userId: Id<"users">; + if (user) { + userId = user._id; + await ctx.db.patch(userId, { role: "admin" }); + } else { + userId = await ctx.db.insert("users", { + handle: "local", + displayName: "Local staff fixture", + role: "admin", + }); + } + if (args.apiTokenHash) { + const existing = await ctx.db + .query("apiTokens") + .withIndex("by_hash", (q) => q.eq("tokenHash", args.apiTokenHash!)) + .unique(); + if (!existing) + await ctx.db.insert("apiTokens", { + userId, + label: "Local search proof", + prefix: "fixture", + tokenHash: args.apiTokenHash, + createdAt: Date.now(), + }); + } + let seededRaw = 0; + if (args.state !== "empty") { + const packageName = "notion-search-proof"; + let pkg = await ctx.db + .query("packages") + .withIndex("by_name", (q) => q.eq("normalizedName", packageName)) + .unique(); + if (!pkg) { + const packageId = await ctx.db.insert("packages", { + name: packageName, + normalizedName: packageName, + displayName: "Notion workspace connector", + ownerUserId: userId, + family: "code-plugin", + channel: "community", + isOfficial: false, + tags: {}, + scanStatus: "clean", + stats: { downloads: 0, installs: 0, stars: 0, versions: 1 }, + createdAt: endDay, + updatedAt: endDay, + }); + const releaseId = await ctx.db.insert("packageReleases", { + packageId, + version: "1.0.0", + changelog: "Local fixture", + distTags: ["latest"], + files: [ + { path: "index.js", size: 18, storageId: args.storageId, sha256: "a".repeat(64) }, + ], + integritySha256: "a".repeat(64), + verification: { tier: "structural", scope: "artifact-only", scanStatus: "clean" }, + createdBy: userId, + createdAt: endDay, + }); + await ctx.db.patch(packageId, { latestReleaseId: releaseId, tags: { latest: releaseId } }); + } + const rows = [ + { query: "notion", count: 12, previous: 4, older: 5 }, + { query: "google drive", count: 9, previous: 12, older: 7 }, + { query: "memory", count: 7, previous: 3, older: 9 }, + { query: "atlas", count: 4, previous: 0, older: 0 }, + ]; + if (args.state === "dense") + for (let i = 0; i < 20; i++) + rows.push({ + query: `workspace automation for cross-functional research and product planning ${i + 1}`, + count: 3, + previous: 1, + older: 2, + }); + const rawRows: Array, "_id" | "_creationTime">> = []; + for (const row of rows) + for (const [count, age] of [ + [row.count, 1], + [row.previous, 8], + [row.older, 20], + ]) { + for (let i = 0; i < count; i++) { + rawRows.push({ + normalizedQuery: row.query, + observedAt: endDay - age * SEARCH_DAY_MS, + source: i % 3 ? "clawhub-web" : "openclaw-control-ui", + artifactKind: "plugin", + resultCount: row.query === "notion" ? 1 : 0, + officialResultCount: 0, + }); + } + } + for (const raw of rawRows.sort((a, b) => a.observedAt - b.observedAt)) { + await ctx.db.insert("pluginSearchObservations", raw); + seededRaw++; + } + } + return { state: args.state, endDay, seededRaw }; + }, +}); diff --git a/packages/clawhub-admin/README.md b/packages/clawhub-admin/README.md index d46b8d5e56..b8aa8b6564 100644 --- a/packages/clawhub-admin/README.md +++ b/packages/clawhub-admin/README.md @@ -147,3 +147,24 @@ All skill and plugin commands accept `--json` where the underlying endpoint supp `packages validation-report --json` exhaustively fetches the current validation state for every plugin and writes exactly one JSON document to stdout. Redirect stdout to archive the report; authentication, registry, and request failures are written to stderr by the CLI error handler. + +### Search intelligence + +Admins and moderators can read the same aggregate report as Management → Search +intelligence. This is read-only and does not change Featured or Trending. + +```sh +clawhub-admin search-insights +clawhub-admin search-insights --source openclaw-control-ui --window 30 --official-gap +clawhub-admin search-insights --intent-kind company_product --json +clawhub-admin search-insights --end-day 2026-09-07 --limit 100 --json +``` + +Windows contain complete UTC days before `--end-day` (exclusive; defaults to today). +Every response includes seven-day, previous-seven-day and 30-day counts. Source can +be `clawhub-web` or `openclaw-control-ui`; omit it to combine them. `--window 7|30` +selects ranking. `--intent-kind` accepts `company_product`, `generic_capability`, or +`ambiguous`; company opportunities require fresh weekly confidence of at least 80% +and three official-gap searches. Missing or failed classification is unavailable, +not inferred from package names. The data-through/coverage fields expose refresh +lag or lost coverage. Current package metadata is freshness-labeled separately. diff --git a/packages/clawhub-admin/src/cli.ts b/packages/clawhub-admin/src/cli.ts index 185df888d4..e920885443 100644 --- a/packages/clawhub-admin/src/cli.ts +++ b/packages/clawhub-admin/src/cli.ts @@ -72,6 +72,7 @@ import { cmdSetPromotionStatus, cmdUpdatePromotion, } from "./commands/promotions.js"; +import { cmdSearchInsights } from "./commands/searchInsights.js"; import { cmdHardDeleteSkill } from "./commands/skills.js"; const program = new Command() @@ -994,6 +995,20 @@ function registerFeaturedCommands(command: Command, kind: "plugin" | "skill") { } } +program + .command("search-insights") + .description("Read staff-only plugin search demand and advisory opportunities") + .option("--source ", "clawhub-web|openclaw-control-ui (default: both)") + .option("--window ", "Rank by 7 or 30 completed UTC days") + .option("--official-gap", "Only queries with zero-official-result searches") + .option("--intent-kind ", "company_product|generic_capability|ambiguous") + .option("--end-day ", "Exclusive UTC window end, YYYY-MM-DD") + .option("--limit ", "Maximum query rows, 1–100") + .option("--json", "Output canonical aggregate JSON") + .action(async (options) => { + await cmdSearchInsights(await resolveGlobalOpts(), options); + }); + program.action(() => { program.outputHelp(); process.exitCode = 0; diff --git a/packages/clawhub-admin/src/commands/searchInsights.test.ts b/packages/clawhub-admin/src/commands/searchInsights.test.ts new file mode 100644 index 0000000000..644d75276f --- /dev/null +++ b/packages/clawhub-admin/src/commands/searchInsights.test.ts @@ -0,0 +1,112 @@ +/* @vitest-environment node */ +import { execFile } from "node:child_process"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +it("parses real admin CLI filters and emits the canonical JSON plus readable demand facts", async () => { + const fixture = { + window: { + endDay: 1788825600000, + start7d: 1788220800000, + startPrevious7d: 1787616000000, + start30d: 1786233600000, + days: 7, + }, + source: "clawhub-web", + generatedAt: 1788825600000, + totalQueries: 1, + totalSearches7d: 5, + sources7d: { "clawhub-web": 5, "openclaw-control-ui": 0 }, + truncated: false, + classificationStatus: "partial", + classificationRun: { + weekStart: 1788220800000, + weekEnd: 1788825600000, + processedAt: 1788825600000, + expectedQualified: 1, + classifiedCount: 1, + truncated: true, + model: "fixture-model", + modelVersion: "v1", + }, + metadataCheckedAt: null, + currentMetadataStatus: "unavailable", + coverage: { + dataThrough: 1788825600000, + collectionStartedAt: 1788220800000, + gapStart: null, + gapEnd: null, + }, + rows: [ + { + query: "notion", + searches7d: 5, + searchesPrevious7d: 2, + searches30d: 11, + officialGaps7d: 5, + officialGaps30d: 9, + zeroResults7d: 0, + change7d: 3, + changePercent: 150, + sources7d: { "clawhub-web": 5, "openclaw-control-ui": 0 }, + classification: null, + companyOpportunity: false, + searchUrl: "/plugins?q=notion", + currentResults: [], + featuredCandidate: null, + }, + ], + }; + const requests: string[] = []; + const server = createServer((request, response) => { + requests.push(request.url ?? ""); + expect(request.headers.authorization).toBe("Bearer fixture-token"); + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify(fixture)); + }); + await new Promise((done) => server.listen(0, "127.0.0.1", done)); + try { + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture port"); + const dir = await mkdtemp(join(tmpdir(), "search-insights-cli-")); + const configPath = join(dir, "config.json"); + await writeFile( + configPath, + JSON.stringify({ token: "fixture-token", registry: `http://127.0.0.1:${address.port}` }), + ); + const cli = resolve(import.meta.dirname, "../cli.ts"); + const args = [ + cli, + "--registry", + `http://127.0.0.1:${address.port}`, + "search-insights", + "--source", + "clawhub-web", + "--window", + "7", + "--official-gap", + "--intent-kind", + "company_product", + ]; + const env = { ...process.env, CLAWHUB_CONFIG_PATH: configPath, NO_COLOR: "1" }; + const json = await promisify(execFile)("bun", [...args, "--json"], { env }); + expect(JSON.parse(json.stdout)).toEqual(fixture); + const human = await promisify(execFile)("bun", args, { env }); + expect(human.stdout).toContain("notion"); + expect(human.stdout).toContain("5 searches"); + expect(human.stdout).toContain("+3"); + expect(human.stdout).toContain("11 in 30d"); + expect(human.stdout).toContain("Classification: partial"); + expect(human.stdout).toContain("capped shortlist"); + expect(human.stdout).toContain("Collection started: 2026-09-01T00:00:00.000Z"); + expect(new URL(requests[0], "http://fixture").searchParams.toString()).toBe( + "source=clawhub-web&window=7&officialGap=true&intentKind=company_product", + ); + } finally { + await new Promise((done) => server.close(() => done())); + } +}); diff --git a/packages/clawhub-admin/src/commands/searchInsights.ts b/packages/clawhub-admin/src/commands/searchInsights.ts new file mode 100644 index 0000000000..e5640c1928 --- /dev/null +++ b/packages/clawhub-admin/src/commands/searchInsights.ts @@ -0,0 +1,104 @@ +import { requireAuthToken } from "../../../clawhub/src/cli/authToken.js"; +import { getRegistry } from "../../../clawhub/src/cli/registry.js"; +import type { GlobalOpts } from "../../../clawhub/src/cli/types.js"; +import { fail } from "../../../clawhub/src/cli/ui.js"; +import { apiRequest } from "../../../clawhub/src/http.js"; +import { SearchInsightsReportSchema } from "../../../clawhub/src/schema/searchInsights.js"; + +export async function cmdSearchInsights( + opts: GlobalOpts, + options: { + source?: string; + window?: string; + officialGap?: boolean; + intentKind?: string; + endDay?: string; + limit?: string; + json?: boolean; + }, +) { + const params = new URLSearchParams(); + if (options.source) { + if (!["clawhub-web", "openclaw-control-ui"].includes(options.source)) + fail("source must be clawhub-web or openclaw-control-ui"); + params.set("source", options.source); + } + if (options.window) { + if (!["7", "30"].includes(options.window)) fail("window must be 7 or 30"); + params.set("window", options.window); + } + if (options.officialGap) params.set("officialGap", "true"); + if (options.intentKind) { + if (!["company_product", "generic_capability", "ambiguous"].includes(options.intentKind)) + fail("intent-kind must be company_product, generic_capability, or ambiguous"); + params.set("intentKind", options.intentKind); + } + if (options.endDay) { + const time = Date.parse(`${options.endDay}T00:00:00.000Z`); + if ( + !/^\d{4}-\d{2}-\d{2}$/.test(options.endDay) || + !Number.isFinite(time) || + new Date(time).toISOString().slice(0, 10) !== options.endDay + ) + fail("end-day must be a UTC date (YYYY-MM-DD)"); + params.set("endDay", String(time)); + } + if (options.limit) { + if (!/^\d+$/.test(options.limit) || Number(options.limit) < 1 || Number(options.limit) > 100) + fail("limit must be between 1 and 100"); + params.set("limit", options.limit); + } + const token = await requireAuthToken(); + const registry = await getRegistry(opts, { cache: true }); + const report = await apiRequest( + registry, + { method: "GET", path: `/api/v1/search-insights?${params}`, token }, + SearchInsightsReportSchema, + ); + if (options.json) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + return report; + } + console.log( + `Search demand · completed UTC days before ${new Date(report.window.endDay).toISOString()} · ${report.source ?? "both sources"}`, + ); + console.log( + `Classification: ${report.classificationStatus}. Featured candidates are advisory; Trending is unchanged.`, + ); + if (report.classificationRun?.truncated) + console.log( + "Classification covers a capped shortlist; additional queries were not classified.", + ); + console.log( + `Collection started: ${report.coverage.collectionStartedAt === null ? "not available yet" : new Date(report.coverage.collectionStartedAt).toISOString()}. Earlier days have no collected history.`, + ); + console.log( + `Aggregated through: ${report.coverage.dataThrough === null ? "not yet refreshed" : new Date(report.coverage.dataThrough).toISOString()}`, + ); + if (report.coverage.gapStart !== null) + console.log( + `Coverage gap: ${new Date(report.coverage.gapStart).toISOString()} — ${new Date(report.coverage.gapEnd!).toISOString()}`, + ); + console.log( + `Current catalog: ${report.metadataCheckedAt === null ? "unavailable" : new Date(report.metadataCheckedAt).toISOString()}`, + ); + if (!report.rows.length) console.log("No matching search demand yet."); + for (const row of report.rows) { + const count = report.window.days === 7 ? row.searches7d : row.searches30d; + console.log( + `${row.query} ${count} searches (${report.window.days}d) ${row.change7d >= 0 ? "+" : ""}${row.change7d} vs previous 7d ${row.searches30d} in 30d ${row.officialGaps7d} official gaps (7d)`, + ); + if (row.classification) + console.log( + ` ${row.classification.intentKind} · ${(row.classification.confidence * 100).toFixed(0)}% · ${row.classification.model}/${row.classification.modelVersion}`, + ); + if (row.featuredCandidate) + console.log( + ` Featured consideration: ${row.featuredCandidate.name} ${row.featuredCandidate.version}`, + ); + console.log(` ${new URL(row.searchUrl, opts.site).toString()}`); + } + if (report.truncated) + console.log(`Showing ${report.rows.length} of ${report.totalQueries} queries.`); + return report; +} diff --git a/packages/clawhub/src/schema/searchInsights.ts b/packages/clawhub/src/schema/searchInsights.ts new file mode 100644 index 0000000000..0ece825e56 --- /dev/null +++ b/packages/clawhub/src/schema/searchInsights.ts @@ -0,0 +1,77 @@ +import { type } from "arktype"; + +const sourceCounts = type({ "clawhub-web": "number", "openclaw-control-ui": "number" }); +const intentKind = '"company_product" | "generic_capability" | "ambiguous"'; +const classification = type({ + weekStart: "number", + weekEnd: "number", + query: "string", + intentKind, + "companyProductName?": "string", + confidence: "number", + model: "string", + modelVersion: "string", + processedAt: "number", +}); +const currentResult = type({ + name: "string", + displayName: "string", + summary: "string | null", + version: "string | null", + url: "string", + isOfficial: "boolean", + isFeatured: "boolean", + eligibleForFeatured: "boolean", +}); +export const SearchInsightsReportSchema = type({ + window: { + endDay: "number", + start7d: "number", + startPrevious7d: "number", + start30d: "number", + days: "7 | 30", + }, + source: '"clawhub-web" | "openclaw-control-ui" | null', + generatedAt: "number", + totalQueries: "number", + totalSearches7d: "number", + sources7d: sourceCounts, + truncated: "boolean", + classificationStatus: '"available" | "partial" | "unavailable"', + classificationRun: type({ + weekStart: "number", + weekEnd: "number", + processedAt: "number", + expectedQualified: "number", + classifiedCount: "number", + truncated: "boolean", + model: "string", + modelVersion: "string", + "failureCode?": "string", + }).or("null"), + metadataCheckedAt: "number | null", + currentMetadataStatus: '"available" | "unavailable"', + coverage: { + dataThrough: "number | null", + collectionStartedAt: "number | null", + gapStart: "number | null", + gapEnd: "number | null", + }, + rows: type({ + query: "string", + searches7d: "number", + searchesPrevious7d: "number", + searches30d: "number", + officialGaps7d: "number", + officialGaps30d: "number", + zeroResults7d: "number", + change7d: "number", + changePercent: "number | null", + sources7d: sourceCounts, + classification: classification.or("null"), + companyOpportunity: "boolean", + searchUrl: "string", + currentResults: currentResult.array(), + featuredCandidate: currentResult.or("null"), + }).array(), +}); diff --git a/specs/search-insights.md b/specs/search-insights.md new file mode 100644 index 0000000000..e863c65953 --- /dev/null +++ b/specs/search-insights.md @@ -0,0 +1,77 @@ +# Staff search intelligence + +ClawHub owns anonymous plugin search observations, daily aggregates, and the canonical +staff report used by Management, HTTP, the admin CLI, and the weekly digest. + +## Trust and meaning + +- An official gap counts a completed visible response with `officialResultCount === 0`. + Capture computes that field only from returned authoritative `isOfficial === true`. +- Daily facts retain normalized query, UTC date, source, artifact kind, category/topic + dimensions, search count, zero-result count, and official-gap count. No identity, + device, session, request, result snapshot, IP, or User-Agent data enters them. +- A query can have some official-gap searches and some official-result searches. The + Official gaps filter includes it when its selected window has at least one gap. +- Weekly company intent is advisory, not provenance. Company opportunities require + company_product confidence >= 0.8 and at least three official-gap searches. +- Current catalog enrichment uses at most three bounded public visible + results per returned query. `metadataCheckedAt` labels its freshness. These are + not the historical results underlying official-gap counts. Suspicious, unpublished, and already Featured + results remain classifier context but cannot be Featured candidates. Featured + candidates alone require clean/installable gates. Candidate + order follows demand, never public Trending. Nothing is automatically featured. + +## Aggregation and retention + +`searchInsights.aggregateInternal` runs hourly. The bounded 200-row ingestion batch +uses the existing convex-helpers index-key paginator. Bucket deltas and the last +index key commit together. Native pagination's terminal cursor must not be stored +as an ingestion cursor: it would miss subsequent arrivals after an empty stream. + +Raw observations always expire after 30 days. A missed aggregation window older +than that becomes an explicit query-free coverage gap, not extended raw retention. +The singleton aggregation state stores only database position, revision, and +coverage bounds. Daily facts and weekly derived classifications expire after 13 +calendar months (clamped at month-end), via indexed 500-row-per-table prune batches. +There is no historical-log backfill. + +## Canonical boundary + +`searchInsights.get` authenticates an active admin/moderator. The internal equivalent +is for trusted HTTP/digest callers only. `GET /api/v1/search-insights` authenticates +an API token and verifies the same staff role, returning private/no-store responses. +No client-supplied user ID is accepted. Management uses an explicit-refresh action, +not a subscription to raw or high-churn data. + +Report bounds are complete UTC days ending at exclusive `endDay` (default today's +UTC midnight): [endDay-7d,endDay), [endDay-14d,endDay-7d), and [endDay-30d,endDay). +The data-through and coverage bounds are separate from requested window bounds. +Seven days means seven complete UTC days, not a partial-day real-time window. + +Sources can be combined or selected. Output order is selected-window searches then +query code-point order. Internal digest callers can select absolute seven-day change +or official-gap count ordering. Counts/source totals cover all filtered rows before +the 1–100 output limit; `truncated` explicitly marks a shortlist. The paginated action +reads at most 100,000 daily rows and fails explicitly rather than silently truncating +facts. An aggregation revision change during the read requires a refresh. + +Weekly classification storage is an atomic <=100-query replacement per week. +A query-free run record persists success/failure, expected/classified counts, +model/version, processing time, and capped-scope flag, including failed and empty runs. +A capped cohort is partial even when every query in that cohort was classified. The newest run +ending no later than the report window and within seven days owns classification; +failed or stale runs never fall back to older successful advice. The report still +serves deterministic counts when classification or current catalog enrichment fails. + +## Verification and local fixtures + +`convex/searchInsights.test.ts` exercises staff/public denial, HTTP parity, daily +idempotency/new arrivals, calendar-month expiry, outage coverage, classification +failure and high-confidence filtering, and public-visible metadata with separate Featured eligibility through +real Convex function boundaries. The admin CLI test launches the actual parser +against a local HTTP fixture in human and JSON modes. + +`searchInsightsFixtures.seed` is internal and refuses non-loopback Convex origins. +It seeds empty, typical, or dense synthetic datasets, a local staff persona, and an +optional hashed API-token fixture for real local browser/API/CLI proof. It never +runs from production crons. These fixtures are not historical demand. diff --git a/src/routes/-management/SearchInsightsPage.tsx b/src/routes/-management/SearchInsightsPage.tsx new file mode 100644 index 0000000000..a414f334fe --- /dev/null +++ b/src/routes/-management/SearchInsightsPage.tsx @@ -0,0 +1,238 @@ +import { useAction } from "convex/react"; +import { useEffect, useState } from "react"; +import { api } from "../../../convex/_generated/api"; +import type { SearchInsightArgs, SearchInsightReport } from "../../../convex/lib/searchInsights"; +import { Button } from "../../components/ui/button"; + +function date(value: number | null) { + return value === null + ? "Not available yet" + : new Date(value).toISOString().replace("T", " ").replace(/Z$/, " UTC"); +} + +export function SearchInsightsPage({ endDay }: { endDay?: number }) { + const getReport = useAction(api.searchInsights.get); + const [source, setSource] = useState(); + const [window, setWindow] = useState<7 | 30>(7); + const [view, setView] = useState("all"); + const [refresh, setRefresh] = useState(0); + const [report, setReport] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + useEffect(() => { + let active = true; + setLoading(true); + setError(null); + void getReport({ + endDay, + source, + window, + officialGap: view === "gaps" || view === "company", + ...(view === "company" ? { intentKind: "company_product" as const } : {}), + }) + .then((value) => { + if (active) setReport(value); + }) + .catch(() => { + if (active) setError("Search insights could not be loaded. Refresh to retry."); + }) + .finally(() => { + if (active) setLoading(false); + }); + return () => { + active = false; + }; + }, [getReport, source, window, view, refresh, endDay]); + const rows = + view === "featured" ? report?.rows.filter((row) => row.featuredCandidate) : report?.rows; + return ( +
+
+
+

Search intelligence

+

+ Manual plugin search demand, official gaps, and curation leads. +

+
+ +
+
+ + + +
+ {error ?

{error}

: null} + {loading ?

Loading search demand…

: null} + {!loading && report ? ( + <> +
+
+ {report.totalSearches7d.toLocaleString()} + searches in 7 days +
+
+ {report.sources7d["clawhub-web"].toLocaleString()} + ClawHub web +
+
+ {report.sources7d["openclaw-control-ui"].toLocaleString()} + Control UI +
+
+ {report.totalQueries.toLocaleString()} + matching queries +
+
+

+ Completed UTC days before {date(report.window.endDay)}. Aggregated through{" "} + {date(report.coverage.dataThrough)}. +

+

+ Collection started: {date(report.coverage.collectionStartedAt)}. Earlier days have no + collected history. +

+ {view === "featured" ? ( +

Featured candidates among the top {report.rows.length} demand queries.

+ ) : null} + {report.coverage.gapStart !== null ? ( +

+ Incomplete coverage: {date(report.coverage.gapStart)} to{" "} + {date(report.coverage.gapEnd)}. Expired raw searches cannot be recovered. +

+ ) : null} +

+ Company classification: {report.classificationStatus} + {report.classificationRun + ? ` · ${report.classificationRun.model}/${report.classificationRun.modelVersion} · processed ${date(report.classificationRun.processedAt)}` + : ". Available after a weekly rollup."} + {report.classificationRun?.truncated + ? " · capped shortlist; additional queries were not classified" + : null} +

+

+ Official gaps count searches with no returned official package. Company intent is + advisory (confidence ≥80%, at least 3 gap searches). Featured candidates require staff + quality and security review. Trending is unchanged. +

+
+ + + + + + + + + + + + + + {rows?.map((row) => ( + + + + + + + + + + ))} + +
Query7 daysChange30 daysOfficial gaps · 7dCompany intentFeatured consideration
+ {row.query} + {row.searches7d} + {row.change7d >= 0 ? "+" : ""} + {row.change7d} + + {row.changePercent === null ? "New" : `${row.changePercent.toFixed(0)}%`} vs + previous 7d + + {row.searches30d}{row.officialGaps7d} + {row.classification ? ( + <> + {row.classification.companyProductName ?? + row.classification.intentKind.replaceAll("_", " ")} + + {(row.classification.confidence * 100).toFixed(0)}% confidence + {row.companyOpportunity ? " · opportunity" : ""} + + + ) : ( + "Unavailable" + )} + + {row.featuredCandidate ? ( + + {row.featuredCandidate.displayName} + {row.featuredCandidate.version} + + ) : ( + "—" + )} +
+
+ {!rows?.length ? ( +

+ No matching search demand yet. Collection starts with manually entered searches; there + is no historical log backfill. +

+ ) : null} + {report.truncated ? ( +

+ Showing the top {report.rows.length} of {report.totalQueries} queries. Narrow the + filters to inspect more. +

+ ) : null} +

+ Current package metadata checked: {date(report.metadataCheckedAt)}. These are current + catalog results, not historical result snapshots. Raw searches expire after 30 days; + daily totals after 13 months. +

+ + ) : null} +
+ ); +} diff --git a/src/routes/-management/managementShared.ts b/src/routes/-management/managementShared.ts index 385eacefd2..832afd5ec7 100644 --- a/src/routes/-management/managementShared.ts +++ b/src/routes/-management/managementShared.ts @@ -34,6 +34,7 @@ export type PublisherAbuseTab = | "resolved" | "signals"; export type ManagementView = + | "search-insights" | "overview" | "abuse" | "reports" diff --git a/src/routes/management.tsx b/src/routes/management.tsx index 239bf72919..f1dbcc1cb8 100644 --- a/src/routes/management.tsx +++ b/src/routes/management.tsx @@ -64,10 +64,12 @@ import { PluginsPage } from "./-management/PluginsPage"; import { PromotionsPage } from "./-management/PromotionsPage"; import { RecentPushesPage } from "./-management/RecentPushesPage"; import { ReportsPage } from "./-management/ReportsPage"; +import { SearchInsightsPage } from "./-management/SearchInsightsPage"; import { SkillsPage } from "./-management/SkillsPage"; import { UsersPage } from "./-management/UsersPage"; const MANAGEMENT_VIEWS = new Set([ + "search-insights", "overview", "abuse", "reports", @@ -190,6 +192,7 @@ export const Route = createFileRoute("/management")({ skill?: string; plugin?: string; view?: ManagementView; + endDay?: number; tab?: PublisherAbuseTab; } = {}; if (typeof search.skill === "string" && search.skill.trim()) { @@ -198,6 +201,14 @@ export const Route = createFileRoute("/management")({ if (typeof search.plugin === "string" && search.plugin.trim()) { validated.plugin = search.plugin; } + if ( + typeof search.endDay === "number" && + Number.isSafeInteger(search.endDay) && + search.endDay >= 0 && + search.endDay % 86_400_000 === 0 + ) { + validated.endDay = search.endDay; + } if (isManagementView(search.view)) { validated.view = search.view; } @@ -755,6 +766,7 @@ export function Management() { {formatManagementViewLabel(activeView)} + {activeView === "search-insights" ? : null} {activeView === "abuse" ? (
Staff tools
+ } + label="Search intelligence" + view="search-insights" + />
{admin ? ( = { + "search-insights": "Search intelligence", overview: "Overview", abuse: "Publisher abuse", reports: "Content reports", diff --git a/src/styles.css b/src/styles.css index 655411ba5c..00e71af481 100644 --- a/src/styles.css +++ b/src/styles.css @@ -34157,3 +34157,111 @@ a[class*="rounded-full"] { margin-top: 8px; } } + +.search-insights { + display: grid; + gap: 20px; +} +.search-insights-header { + display: flex; + justify-content: space-between; + align-items: start; + gap: 16px; +} +.search-insights h1 { + font-size: 1.7rem; + font-weight: 650; + margin: 0 0 8px; +} +.search-insights p { + margin: 0; + font-size: 0.88rem; + line-height: 1.6; +} +.search-insights-controls { + display: flex; + flex-wrap: wrap; + gap: 16px; +} +.search-insights-controls label { + display: grid; + gap: 6px; + font-size: 0.8rem; + color: var(--ink-soft); +} +.search-insights-controls select { + border: 1px solid var(--line); + border-radius: 7px; + background: var(--surface); + color: var(--ink); + padding: 8px 12px; + min-width: 180px; +} +.search-insights-summary { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + border: 1px solid var(--line); + border-radius: 10px; + background: var(--surface); +} +.search-insights-summary > div { + display: grid; + gap: 5px; + padding: 20px; +} +.search-insights-summary strong { + font-size: 1.7rem; + font-variant-numeric: tabular-nums; +} +.search-insights-summary span { + color: var(--ink-soft); + font-size: 0.8rem; +} +.search-insights-table-wrap { + overflow-x: auto; + border: 1px solid var(--line); + border-radius: 10px; +} +.search-insights-table { + width: 100%; + min-width: 1000px; + border-collapse: collapse; + text-align: left; + font-size: 0.85rem; +} +.search-insights-table th, +.search-insights-table td { + padding: 14px 16px; + border-bottom: 1px solid var(--line); + vertical-align: top; +} +.search-insights-table th:first-child, +.search-insights-table td:first-child { + min-width: 180px; + max-width: 320px; +} +.search-insights-table th { + background: var(--surface-muted); + color: var(--ink-soft); + white-space: nowrap; + font-size: 0.75rem; + font-weight: 500; +} +.search-insights-table small { + display: block; + color: var(--ink-soft); + margin-top: 4px; + font-size: 0.7rem; +} +.search-insights-table a { + color: var(--accent); +} +@media (max-width: 700px) { + .search-insights-summary { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .search-insights-controls label, + .search-insights-controls select { + width: 100%; + } +}