diff --git a/.gitignore b/.gitignore index 0046d1087f..d79ab5525c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,9 @@ build doc/dependency_decisions.yml .serena/ .ralphex/progress/ + +# compiled cmd/ tool binaries, which land at the repo root when built with +# `go build ./cmd/` and have been committed by accident more than once +/anyblockroundtrip +/anyblockconvert +/anyblockvalidate diff --git a/cmd/anyblockconvert/batch.go b/cmd/anyblockconvert/batch.go new file mode 100644 index 0000000000..145b9ec189 --- /dev/null +++ b/cmd/anyblockconvert/batch.go @@ -0,0 +1,554 @@ +package main + +import ( + "crypto/sha1" + "encoding/hex" + "github.com/globalsign/mgo/bson" + "sort" + + "github.com/anyproto/anytype-heart/cmd/internal/anyblockbatch" + "github.com/anyproto/lexid" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/constant" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +// pendingSnapshot is a Relation/RelationOption object the batch mints the +// first time a custom property or option value is referenced. +type pendingSnapshot struct { + id string + sbType model.SmartBlockType + snapshot *model.SmartBlockSnapshotBase +} + +// batch is the cross-document wiring pkg/lib/anyblockjson deliberately +// leaves to its caller (SPEC.md §2a "the import wiring's job", §3 "creating +// options is the wiring's job"): it answers anyblockjson.Options' +// FormatResolver/OptionResolver/PropertyResolver callbacks from the +// pre-scanned typeProperties table, lazily minting Relation/RelationOption +// snapshots the same way core/block/import/csv and core/block/import/notion +// build them for their own generated relations and options. +type batch struct { + // formats is keyed by the STORED property key, which is what every + // reader below hands it: anyblockjson resolves a `properties` key or a + // `type_settings.property_definitions[].property` through the document's own property_internal_keys legend + // and the bundled table (§3, importer.propertyKey) before it calls + // ResolveFormat or builds a PropertyDefinition. anyblockbatch.ScanFormats + // runs the same chain when it builds the table — keying it by the raw + // spelling instead made every legend-backed property a silent miss here. + formats map[string]anyblockbatch.FormatInfo + + relIDs map[string]string // stored property key -> minted relation object id + // minted is the batch's key vocabulary: the SPELLING a document used -> + // the internal key minted for it. A property an author declared by name + // or spelling gets a fresh bson id here, the way the app mints one when + // a user creates a property, and every document's detail keys resolve + // through this map so they all land on that one key (§2e). + minted map[string]string + optIDs map[string]string // "stored key\x00name" -> minted option object id + + // optNames is optIDs read backwards — "stored key\x00option id" -> name — + // and it is what makes OptionName answerable here. The batch is the space + // this conversion imports into, so the set of options it has minted IS the + // set of live options, and an id outside it names nothing the archive + // carries (see OptionName). + optNames map[string]string + + // optOrder is the last order id handed out per property key. Every option + // needs one: options sort on orderId+name concatenated + // (database.OrderMap.BuildOrder), so an option without an order id is + // compared by name against everyone else's order id and lands + // arbitrarily — before the declared vocabulary when its name sorts below + // the lexid alphabet, after it otherwise. + optOrder map[string]string + + // optColor is the next palette position per property key, and optClaimed + // the colors that property's vocabulary names explicitly (§2a) so the + // cycle never hands one of them out a second time. Every option needs a + // color too: the app assigns one on creation (pkg/lib/schema.Relation + // CreateOptionDetails, core/block/import/markdown), so an option minted + // without one is not "default-colored", it is the only kind of option in + // the space missing the detail entirely. + optColor map[string]int + optClaimed map[string]map[string]bool + + // typeIDs maps a type key this bundle defines to the id its document + // carries, so a property targeting it references the same id every other + // reference in the batch uses. + typeIDs map[string]string + + pending []pendingSnapshot +} + +// newBatch pre-declares every select vocabulary the batch knows about before +// any document converts. Order matters: an option first seen as a value on +// some object is minted without an orderId, and the directory walk reaches +// objects/ before types/, so declaring lazily would leave the used values +// unordered and the unused ones ordered. +// +// The map key IS the relation key the options are declared under — it has to +// be the stored key, since that is what OptionId is later called with. That +// holds because ScanFormats resolves the term (§3) before keying; when it +// did not, a legend-backed vocabulary was pre-minted under the SPELLING, so +// the declared options sat on a relation nothing referenced while the values +// that actually arrived minted a second, order-less set under the real key. +func newBatch(formats map[string]anyblockbatch.FormatInfo, typeIds map[string]string) (b *batch) { + b = &batch{ + formats: formats, + relIDs: map[string]string{}, + minted: map[string]string{}, + optIDs: map[string]string{}, + optNames: map[string]string{}, + optOrder: map[string]string{}, + optColor: map[string]int{}, + optClaimed: map[string]map[string]bool{}, + typeIDs: typeIds, + } + keys := make([]string, 0, len(formats)) + for k := range formats { + keys = append(keys, k) + } + sort.Strings(keys) // deterministic option ids across runs + for _, k := range keys { + b.declareOptions(domain.RelationKey(k), formats[k].Options) + } + return b +} + +func (b *batch) relationCount() int { return len(b.relIDs) } +func (b *batch) optionCount() int { return len(b.optIDs) } + +// resolveFormat implements anyblockjson.FormatResolver. `key` arrives already +// resolved (importer.propertyKey), which is why formats is keyed the same way. +func (b *batch) resolveFormat(key domain.RelationKey) (model.RelationFormat, bool) { + if fi, ok := b.formats[string(key)]; ok { + return fi.Format, true + } + return 0, false +} + +// OptionId implements anyblockjson.OptionResolver: allocates a stable +// RelationOption object the first time (key, name) is seen in the batch. +func (b *batch) OptionId(key domain.RelationKey, name string) (string, bool) { + mapKey := string(key) + "\x00" + name + if id, ok := b.optIDs[mapKey]; ok { + return id, true + } + id := b.mintOption(key, name, b.nextOptionOrder(key), b.nextOptionColor(key)) + b.optIDs[mapKey] = id + return id, true +} + +// OptionName implements anyblockjson.OptionResolver. This tool only imports, +// but the call is NOT export-only: it is also the liveness question a +// document's `option_ids` entry is checked against (§3, §9a — an id is +// honoured only where the resolver answers for it as an option of that +// relation). A resolver that stubs this out disables the legend for +// everything it converts, which is what this one used to do, silently and on +// the strength of a stale doc comment. +// +// The archive being built is the space that answers here, and its options are +// exactly the ones this batch has minted, so optNames IS the liveness table. +// That keeps the safety property the legend leans on: an id this method +// confirms always has a RelationOption object in `pending`, so honouring one +// can never write a reference the archive does not carry. +// +// What it means in practice, said plainly rather than left to be inferred: a +// bundle exported from another space carries THAT space's option ids, and +// none of them can be live here, since every id in this archive is derived +// from (property key, option name) by optionLocalKey. Those entries fail +// liveness and their values resolve by name, which is the fallback §3 +// prescribes and the only thing a fresh-space converter could do with a +// foreign id anyway. Where the legend does bite is an id this batch itself +// minted, which is any id naming an option the archive already carries: the +// value lands on that option even when the name beside it has moved on, so a +// renamed vocabulary re-points its old values instead of minting a second +// option under the stale name (the resurrection §3 describes). +func (b *batch) OptionName(key domain.RelationKey, id string) (string, bool) { + name, ok := b.optNames[string(key)+"\x00"+id] + return name, ok +} + +// PropertyId implements anyblockjson.PropertyResolver: allocates a stable +// Relation object for a typeProperties entry (§2a) the first time its key is +// seen in the batch. Bundled (system) properties resolve to their bundled +// url instead of minting a new object — installBundledRelationsAndTypes +// (core/block/import/common/objectcreator) installs those automatically. +func (b *batch) PropertyId(def anyblockjson.PropertyDefinition) (string, bool) { + if bundle.HasRelation(def.Key) { + // a bundled select still needs its options to exist in this space + b.declareOptions(def.Key, def.Options) + return def.Key.BundledURL(), true + } + key := string(def.Key) + b.declareOptions(def.Key, def.Options) + if id, ok := b.relIDs[key]; ok { + return id, true + } + id := b.mintRelation(def) + b.relIDs[key] = id + return id, true +} + +// PropertySlug and PropertyKey implement anyblockjson.KeyVocabulary: the +// batch's own spelling layer, laid over the bundled one. +// +// This is what makes minting safe across a bundle. A property declared once +// in properties.json by the spelling `cooking_time` is minted a bson key +// here; a recipe document that carries `"cooking_time": 90` then resolves +// that detail key through this vocabulary and lands on the SAME minted key, +// instead of writing a detail no relation object describes. +func (b *batch) PropertySlug(key string) string { + for spelling, minted := range b.minted { + if minted == key { + return spelling + } + } + return anyblockjson.BundledKeyVocabulary{}.PropertySlug(key) +} + +func (b *batch) PropertyKey(slug string) (string, bool) { + if minted, ok := b.minted[slug]; ok { + return minted, true + } + return anyblockjson.BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (b *batch) TypeSlug(key string) string { + return anyblockjson.BundledKeyVocabulary{}.TypeSlug(key) +} + +func (b *batch) TypeKey(slug string) (string, bool) { + return anyblockjson.BundledKeyVocabulary{}.TypeKey(slug) +} + +// PropertyById implements anyblockjson.PropertyResolver. Only used on +// export; this tool only imports, so it's never called. +func (b *batch) PropertyById(id string) (anyblockjson.PropertyDefinition, bool) { + return anyblockjson.PropertyDefinition{}, false +} + +// declareOptions pre-mints the vocabulary a typeProperties entry declares +// (§2a), in declaration order, so every value exists whether or not any +// record happens to use it and the order is the author's rather than +// alphabetical. Names already minted from usage are adopted, not duplicated. +// Options that declare no color take the next palette entry the vocabulary +// has not claimed, which is why the explicit colors are collected first. +func (b *batch) declareOptions(key domain.RelationKey, opts []anyblockjson.OptionDefinition) { + for _, o := range opts { + if o.Color == "" { + continue + } + if b.optClaimed[string(key)] == nil { + b.optClaimed[string(key)] = map[string]bool{} + } + b.optClaimed[string(key)][o.Color] = true + } + for _, o := range opts { + mapKey := string(key) + "\x00" + o.Name + if _, done := b.optIDs[mapKey]; done { + continue + } + color := o.Color + if color == "" { + color = b.nextOptionColor(key) + } + b.optIDs[mapKey] = b.mintOption(key, o.Name, b.nextOptionOrder(key), color) + } +} + +// objectTypeIds turns the type keys a property targets into the ids a +// snapshot references them by, the same split PropertyId makes for +// relations: a type this batch defines is referenced by the id its own +// document carries, so the importer relinks it along with everything else, +// and a bundled type is referenced by its bundled url (_ot), the form +// recommendedRelations already uses for bundled properties (_br). +func (b *batch) objectTypeIds(def anyblockjson.PropertyDefinition) []string { + if len(def.ObjectTypes) == 0 { + return nil + } + out := make([]string, 0, len(def.ObjectTypes)) + for _, key := range def.ObjectTypes { + if id, local := b.typeIDs[key]; local { + out = append(out, id) + continue + } + out = append(out, domain.TypeKey(key).BundledURL()) + } + return out +} + +// targetTypeId resolves a template's target type key to the id that type's own +// document carries — the value targetObjectType has to hold for the pb importer +// to relink it along with every other reference in the batch. A type the bundle +// does not define, or defines without an id, has no usable value: unlike a +// property's objectTypes, a bundled url will not do (see +// anyblockbatch.CheckTemplateTargets, which rejects that bundle up front). +func (b *batch) targetTypeId(key string) (string, bool) { + id, defined := b.typeIDs[key] + return id, defined && id != "" +} + +// optionLexId mirrors core/block/editor/order.LexId. It is duplicated rather +// than imported because that package pulls in the whole smartblock editor; +// the two must stay in step or ids minted here will not interleave with ones +// the app generates later. +var optionLexId = lexid.Must(lexid.CharsBase64, 4, 4000) + +// nextOptionOrder hands out the next order id for a property, continuing +// after whatever was assigned last. Declared vocabulary is laid down first +// (newBatch), so an option discovered later from a value nobody declared +// lands after it rather than in the middle of it. +func (b *batch) nextOptionOrder(key domain.RelationKey) string { + last, seen := b.optOrder[string(key)] + if !seen { + last = optionLexId.Middle() + } else { + last = optionLexId.Next(last) + } + b.optOrder[string(key)] = last + return last +} + +// nextOptionColor hands out the next palette color for a property, skipping +// the ones its vocabulary claims explicitly. Cycling rather than picking at +// random (constant.RandomOptionColor, what the app does) gives a vocabulary +// that names no colors ten distinct ones instead of the same color three +// times, and keeps a converted bundle byte-identical across runs. +func (b *batch) nextOptionColor(key domain.RelationKey) string { + palette := constant.OptionColors() + claimed := b.optClaimed[string(key)] + take := func() string { + c := palette[b.optColor[string(key)]%len(palette)] + b.optColor[string(key)]++ + return c.String() + } + for range palette { + if c := take(); !claimed[c] { + return c + } + } + return take() // a vocabulary claiming all ten: reuse, in cycle order +} + +// looksLikeMintedKey reports whether key has the shape of an app-minted +// internal key (a bson object id: 24 hex characters) — the population that +// owes no mint because it already is one. +func looksLikeMintedKey(key string) bool { + if len(key) != 24 { + return false + } + for _, r := range key { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} + +// mintRelation builds a Relation object snapshot matching the shape +// core/block/import/csv and core/block/import/notion build for their own +// generated relations: Details{name, relationKey, relationFormat, layout}, +// ObjectTypes = [ot-relation], Key = the property key (so the id/dedup +// pipeline in core/block/import/processor.go can recognize it across +// re-imports the same way it does for CSV/Notion relations). +func (b *batch) mintRelation(def anyblockjson.PropertyDefinition) string { + // A property the document declared by SPELLING gets a fresh internal key + // here, the way the app mints one when a user creates a property + // (objectcreator/relation.go: bson.NewObjectId().Hex()). A property that + // stated its `internal_key` keeps it exactly, which is the whole point of + // stating one: a bundle re-imported elsewhere reproduces the same stored + // key (§2e). + // + // The document's spelling is remembered as the api key, and bound in this + // batch's vocabulary so every OTHER document that names the property by + // the same spelling resolves to this one minted key. + storedKey := string(def.Key) + apiKey := "" + if !def.KeyIsInternal { + apiKey = storedKey + if existing, ok := b.minted[apiKey]; ok { + storedKey = existing + } else { + storedKey = bson.NewObjectId().Hex() + b.minted[apiKey] = storedKey + } + } + format := def.Format + name := def.Name + if fi, ok := b.formats[string(def.Key)]; ok { + format = fi.Format + if name == "" { + name = fi.Name + } + } + if name == "" { + name = string(def.Key) + } + + uk, err := domain.NewUniqueKey(coresb.SmartBlockTypeRelation, storedKey) + if err != nil { + // def.Key already passed through the schema's property-key charset + // (§2a); NewUniqueKey only fails for an unsupported smartblock type. + panic(err) + } + id := uk.Marshal() + + details := &types.Struct{Fields: map[string]*types.Value{ + detailID: strVal(id), + detailName: strVal(name), + detailRelationKey: strVal(storedKey), + detailRelationFormat: numVal(float64(format)), + detailLayout: numVal(float64(model.ObjectType_relation)), + }} + if apiKey != "" { + // the spelling the document used, kept the way the app keeps it for a + // property created through the API: the internal key is opaque, and + // this is the name anything addressing the property by spelling asks + // for (objectcreator/relation.go) + details.Fields[detailApiObjectKey] = strVal(apiKey) + } + if format == model.RelationFormat_status { + details.Fields[detailRelationMaxCount] = numVal(1) + } + if ids := b.objectTypeIds(def); len(ids) > 0 { + details.Fields[detailRelationFormatObjectTypes] = strListVal(ids) + } + // the rest of the shared propertyDefinition shape (§2a): a definition may + // state these, and a minted relation that shed them would make listing a + // member in the document weaker than not listing it — the exact trap the + // absent-format rule documents. Each writes only when declared, so a + // definition that says nothing changes nothing. + if def.Description != "" { + details.Fields[detailDescription] = strVal(def.Description) + } + if def.IncludeTime != nil { + details.Fields[detailRelationFormatIncludeTime] = boolVal(*def.IncludeTime) + } + if def.MaxCount > 0 { + details.Fields[detailRelationMaxCount] = numVal(float64(def.MaxCount)) + } + if def.Readonly { + details.Fields[detailRelationReadonlyValue] = boolVal(true) + } + if def.DefaultValue != nil { + details.Fields[detailRelationDefaultValue] = pbtypes.InterfaceToValue(def.DefaultValue) + } + + snap := &model.SmartBlockSnapshotBase{ + Blocks: rootOnlyBlocks(id), + Details: details, + ObjectTypes: []string{bundle.TypeKeyRelation.URL()}, + Key: string(def.Key), + } + b.pending = append(b.pending, pendingSnapshot{id: id, sbType: model.SmartBlockType_STRelation, snapshot: snap}) + return id +} + +// mintOption builds a RelationOption object snapshot, matching the shape +// core/block/import/notion builds for select/multiSelect/status options: +// Details{name, relationKey, layout}, ObjectTypes = [ot-relationOption]. +func (b *batch) mintOption(key domain.RelationKey, name, orderId, color string) string { + localKey := optionLocalKey(key, name) + uk, err := domain.NewUniqueKey(coresb.SmartBlockTypeRelationOption, localKey) + if err != nil { + panic(err) + } + id := uk.Marshal() + // the reverse entry is recorded HERE, at the one place an option object + // comes into existence, so "this batch can name the id" and "this batch + // carries the object" are the same statement — OptionName's liveness + // answer is only safe while that holds. + b.optNames[string(key)+"\x00"+id] = name + + details := &types.Struct{Fields: map[string]*types.Value{ + detailID: strVal(id), + detailName: strVal(name), + detailRelationKey: strVal(string(key)), + detailLayout: numVal(float64(model.ObjectType_relationOption)), + }} + // options sort on orderId+name concatenated (database.OrderMap.BuildOrder), + // so every option needs an order id: without one it is compared by name + // against everyone else's order id and lands arbitrarily. Declared + // vocabulary comes first, discovered names after it. + details.Fields[detailOrderId] = strVal(orderId) + // an option's color is a detail like any other, and every creation path in + // the app sets one (pkg/lib/schema.Relation.CreateOptionDetails); a + // declared vocabulary says which, rather than leaving it to chance (§2a). + details.Fields[detailRelationOptionColor] = strVal(color) + + snap := &model.SmartBlockSnapshotBase{ + Blocks: rootOnlyBlocks(id), + Details: details, + ObjectTypes: []string{bundle.TypeKeyRelationOption.URL()}, + Key: localKey, + } + b.pending = append(b.pending, pendingSnapshot{id: id, sbType: model.SmartBlockType_STRelationOption, snapshot: snap}) + return id +} + +// optionLocalKey derives a short, stable, charset-safe UniqueKey component +// from (property key, option name). Unlike relations, two different +// properties may share an option name ("Active"), so the property key has to +// be part of the hash, not just the name. +func optionLocalKey(key domain.RelationKey, name string) string { + sum := sha1.Sum([]byte(string(key) + "\x00" + name)) + return hex.EncodeToString(sum[:])[:12] +} + +func rootOnlyBlocks(id string) []*model.Block { + return []*model.Block{{ + Id: id, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }} +} + +const ( + detailID = "id" + detailName = "name" + // the relation a type's templates are queried by + // (core/block/template/templateimpl.queryTemplatesByType) + detailTargetObjectType = string(bundle.RelationKeyTargetObjectType) + detailRelationKey = "relationKey" + detailRelationFormat = "relationFormat" + detailOrderId = "orderId" + detailRelationOptionColor = string(bundle.RelationKeyRelationOptionColor) + detailRelationFormatObjectTypes = "relationFormatObjectTypes" + detailRelationMaxCount = "relationMaxCount" + detailLayout = "layout" + detailIsHidden = string(bundle.RelationKeyIsHidden) + detailDescription = string(bundle.RelationKeyDescription) + detailRelationFormatIncludeTime = string(bundle.RelationKeyRelationFormatIncludeTime) + detailRelationReadonlyValue = string(bundle.RelationKeyRelationReadonlyValue) + detailRelationDefaultValue = string(bundle.RelationKeyRelationDefaultValue) + // the spelling a property was created under, which is what an API caller + // addresses it by when its internal key is an opaque minted id + detailApiObjectKey = string(bundle.RelationKeyApiObjectKey) +) + +func strVal(s string) *types.Value { + return &types.Value{Kind: &types.Value_StringValue{StringValue: s}} +} + +func numVal(n float64) *types.Value { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: n}} +} + +func boolVal(b bool) *types.Value { + return &types.Value{Kind: &types.Value_BoolValue{BoolValue: b}} +} + +func strListVal(ss []string) *types.Value { + vals := make([]*types.Value, 0, len(ss)) + for _, s := range ss { + vals = append(vals, strVal(s)) + } + return &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: vals}}} +} diff --git a/cmd/anyblockconvert/batch_test.go b/cmd/anyblockconvert/batch_test.go new file mode 100644 index 0000000000..a82c413544 --- /dev/null +++ b/cmd/anyblockconvert/batch_test.go @@ -0,0 +1,327 @@ +package main + +// Options sort on orderId+name concatenated (database.OrderMap.BuildOrder), +// so an option with no order id is compared by *name* against everyone +// else's order id: "Abandoned" would sort ahead of the whole declared +// vocabulary because 'A' < the lexid alphabet, while "Zebra" would sort +// behind it. Every option therefore needs an order id. + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/cmd/internal/anyblockbatch" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/constant" +) + +// vocab is a declared vocabulary that names no colors. +func vocab(names ...string) []anyblockjson.OptionDefinition { + out := make([]anyblockjson.OptionDefinition, 0, len(names)) + for _, n := range names { + out = append(out, anyblockjson.OptionDefinition{Name: n}) + } + return out +} + +// mintedColors maps option name to the relationOptionColor minted for it, +// for one property. +func mintedColors(t *testing.T, b *batch, key string) map[string]string { + t.Helper() + out := map[string]string{} + for _, p := range b.pending { + if p.sbType != model.SmartBlockType_STRelationOption { + continue + } + d := p.snapshot.Details.Fields + if d[detailRelationKey].GetStringValue() != key { + continue + } + name := d[detailName].GetStringValue() + out[name] = d[detailRelationOptionColor].GetStringValue() + } + return out +} + +// mintedOptions returns option names in the order BuildOrder would compare +// them: the orderId and name concatenated. +func mintedOptions(t *testing.T, b *batch) []string { + t.Helper() + type opt struct{ sortKey, name string } + var opts []opt + for _, p := range b.pending { + if p.sbType != model.SmartBlockType_STRelationOption { + continue + } + d := p.snapshot.Details.Fields + name := d[detailName].GetStringValue() + order := d[detailOrderId].GetStringValue() + require.NotEmpty(t, order, "option %q has no order id", name) + opts = append(opts, opt{order + name, name}) + } + sort.Slice(opts, func(i, j int) bool { return opts[i].sortKey < opts[j].sortKey }) + names := make([]string, 0, len(opts)) + for _, o := range opts { + names = append(names, o.name) + } + return names +} + +func TestBatch_DeclaredVocabularyKeepsDeclarationOrder(t *testing.T) { + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": { + Format: model.RelationFormat_status, + Options: vocab("Backlog", "In progress", "In review", "Blocked", "Done"), + }, + }, nil) + assert.Equal(t, + []string{"Backlog", "In progress", "In review", "Blocked", "Done"}, + mintedOptions(t, b), + "declaration order, not alphabetical") +} + +// a value no vocabulary declares must land after the declared ones whatever +// its name — this is the case that used to get an empty order id +func TestBatch_UndeclaredValueSortsAfterVocabulary(t *testing.T) { + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": { + Format: model.RelationFormat_status, + Options: vocab("Backlog", "In progress", "Done"), + }, + }, nil) + // "Abandoned" sorts first alphabetically and would jump the queue + b.OptionId(domain.RelationKey("stage"), "Abandoned") + b.OptionId(domain.RelationKey("stage"), "Zebra") + + assert.Equal(t, + []string{"Backlog", "In progress", "Done", "Abandoned", "Zebra"}, + mintedOptions(t, b)) +} + +// with nothing declared, discovery order still produces real order ids +func TestBatch_UndeclaredOnlyStillGetsOrderIds(t *testing.T) { + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": {Format: model.RelationFormat_status}, + }, nil) + b.OptionId(domain.RelationKey("stage"), "Zebra") + b.OptionId(domain.RelationKey("stage"), "Abandoned") + assert.Equal(t, []string{"Zebra", "Abandoned"}, mintedOptions(t, b), + "first seen first, not alphabetical") +} + +// order ids are per property, so two selects do not interleave +func TestBatch_OrderIdsArePerProperty(t *testing.T) { + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": {Format: model.RelationFormat_status, Options: vocab("A", "B")}, + "priority": {Format: model.RelationFormat_status, Options: vocab("Low", "High")}, + }, nil) + firsts := map[string]string{} + for _, p := range b.pending { + if p.sbType != model.SmartBlockType_STRelationOption { + continue + } + d := p.snapshot.Details.Fields + key := d[detailRelationKey].GetStringValue() + if _, seen := firsts[key]; !seen { + firsts[key] = d[detailOrderId].GetStringValue() + } + } + require.Len(t, firsts, 2) + assert.Equal(t, firsts["stage"], firsts["priority"], + "each property starts its own sequence at the same midpoint") +} + +func TestBatch_DeclaredColorsReachTheOption(t *testing.T) { + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": {Format: model.RelationFormat_status, Options: []anyblockjson.OptionDefinition{ + {Name: "Backlog", Color: "grey"}, + {Name: "In progress", Color: "blue"}, + {Name: "Done", Color: "lime"}, + }}, + }, nil) + assert.Equal(t, + map[string]string{"Backlog": "grey", "In progress": "blue", "Done": "lime"}, + mintedColors(t, b, "stage")) +} + +// A vocabulary that declares no colors still gets distinct ones: the palette +// is cycled in declaration order, so a five-status select does not render as +// five identical chips. Deterministic, unlike constant.RandomOptionColor. +func TestBatch_ColorlessVocabularyCyclesThePalette(t *testing.T) { + newFixture := func() *batch { + return newBatch(map[string]anyblockbatch.FormatInfo{ + "tag": {Format: model.RelationFormat_tag, + Options: vocab("design", "research", "infra", "ops")}, + }, nil) + } + want := map[string]string{ + "design": "grey", "research": "yellow", "infra": "orange", "ops": "red", + } + + assert.Equal(t, want, mintedColors(t, newFixture(), "tag")) + assert.Equal(t, want, mintedColors(t, newFixture(), "tag"), + "same input, same colors on every run") +} + +// the cycle never hands out a color the vocabulary names explicitly +func TestBatch_ColorCycleSkipsDeclaredColors(t *testing.T) { + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": {Format: model.RelationFormat_status, Options: []anyblockjson.OptionDefinition{ + {Name: "Backlog"}, + {Name: "In progress", Color: "yellow"}, // second in the palette + {Name: "Done"}, + }}, + }, nil) + assert.Equal(t, + map[string]string{"Backlog": "grey", "In progress": "yellow", "Done": "orange"}, + mintedColors(t, b, "stage"), + "Done takes orange, not the claimed yellow") +} + +// a vocabulary claiming the whole palette leaves the cycle nothing to pick: +// it must still terminate and produce a real color +func TestBatch_ColorCycleTerminatesWhenPaletteFullyClaimed(t *testing.T) { + declared := make([]anyblockjson.OptionDefinition, 0, len(constant.OptionColors())+1) + for i, c := range constant.OptionColors() { + declared = append(declared, anyblockjson.OptionDefinition{ + Name: fmt.Sprintf("claim%d", i), Color: c.String()}) + } + declared = append(declared, anyblockjson.OptionDefinition{Name: "uncolored"}) + + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "tag": {Format: model.RelationFormat_tag, Options: declared}, + }, nil) + assert.Contains(t, constant.OptionColors(), + constant.OptionColor(mintedColors(t, b, "tag")["uncolored"])) +} + +// a value nobody declared continues its property's cycle rather than +// restarting it, so it does not collide with the vocabulary's first color +func TestBatch_DiscoveredValueContinuesTheColorCycle(t *testing.T) { + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": {Format: model.RelationFormat_status, Options: vocab("Backlog", "Done")}, + }, nil) + b.OptionId(domain.RelationKey("stage"), "Abandoned") + + assert.Equal(t, + map[string]string{"Backlog": "grey", "Done": "yellow", "Abandoned": "orange"}, + mintedColors(t, b, "stage")) +} + +// colors are per property: one select's palette position is not advanced by +// another's, the way order ids are already scoped (TestBatch_OrderIdsArePerProperty) +func TestBatch_ColorsArePerProperty(t *testing.T) { + b := newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": {Format: model.RelationFormat_status, Options: vocab("A", "B")}, + "priority": {Format: model.RelationFormat_status, Options: vocab("Low", "High")}, + }, nil) + assert.Equal(t, map[string]string{"A": "grey", "B": "yellow"}, mintedColors(t, b, "stage")) + assert.Equal(t, map[string]string{"Low": "grey", "High": "yellow"}, mintedColors(t, b, "priority")) +} + +// A property's target types are written as ids, split the same way +// PropertyId splits relations: a type this bundle defines is referenced by +// its own document id so the importer relinks it, a bundled type by its +// bundled url — the form recommendedRelations already uses for _br. +func TestBatch_ObjectTypeIdsSplitLocalAndBundled(t *testing.T) { + b := newBatch(nil, map[string]string{"wikiPerson": "type-person"}) + got := b.objectTypeIds(anyblockjson.PropertyDefinition{ + Key: "owner", + ObjectTypes: []string{"wikiPerson", "participant"}, + }) + assert.Equal(t, []string{"type-person", "_otparticipant"}, got) +} + +func TestBatch_NoObjectTypesLeavesPropertyUntargeted(t *testing.T) { + b := newBatch(nil, nil) + assert.Nil(t, b.objectTypeIds(anyblockjson.PropertyDefinition{Key: "owner"})) +} + +// The converter half of anyblockbatch's CheckTargetTypes ordering: a type this +// bundle DEFINES wins over the bundled type of the same name, even when its +// document carries no id — and then the target is the empty string, which +// names nothing anywhere. This is not a behaviour to keep, it is the reason +// CheckTargetTypes has to ask typeIds before it asks the bundle; the lint +// rejects such a bundle so this never runs in anger. If this assertion ever +// changes, TestCheckTargetTypes_BundledKeyDefinedLocallyWithoutIdIsReported +// has to change with it. +func TestBatch_IdlessLocalTypeYieldsAnEmptyTargetId(t *testing.T) { + require.True(t, bundle.HasObjectTypeByKey("page"), + "the fixture only bites while `page` really is bundled, i.e. while the two arms disagree") + b := newBatch(nil, map[string]string{"page": ""}) + assert.Equal(t, []string{""}, b.objectTypeIds(anyblockjson.PropertyDefinition{ + Key: "owner", + ObjectTypes: []string{"page"}, + }), "the local arm wins and has nothing to offer — not the bundled url") +} + +// End to end, over the real seam: a bundle whose property_internal_keys legend backs a +// slug (§3) must mint the property's Relation object and its declared select +// vocabulary under the STORED key, because that is the key the value's detail +// is written under. When the format table was keyed by the spelling, the +// format was never found: the value passed through raw, no Relation was minted +// at all, and the declared options sat on a relation nothing referenced. +func TestBatch_LegendBackedPropertyMintsItsRelationAndOptions(t *testing.T) { + const storedKey = "6a32d4856761631534b22f85" + const legend = `"property_internal_keys": {"priority": "` + storedKey + `"},` + + dir := t.TempDir() + typeDoc := filepath.Join(dir, "task.type.json") + require.NoError(t, os.WriteFile(typeDoc, []byte(`{"version": 2, "kind": "object_type", + "internal_key": "task", "id": "type-task", `+legend+` + "type_settings": {"property_definitions": [{"property": "priority", "name": "Priority", "format": "select", + "options": ["High", "Low"]}]}}`), 0o644)) + + formats, err := anyblockbatch.ScanFormats([]string{typeDoc}) + require.NoError(t, err) + b := newBatch(formats, map[string]string{"task": "type-task"}) + + // types first, exactly as OrderTypesFirst arranges the walk — the type + // document is what drives PropertyId, and so what mints the Relation + _, _, _, err = convertFile(dir, typeDoc, b, false, nil) + require.NoError(t, err) + + _, snap := convertDoc(t, b, "one.json", `{"version": 2, "id": "obj-1", "type": "task", `+legend+` + "properties": {"priority": "High"}}`) + + // the value reached the detail under the stored key, as an option id + optionID := snap.Details.GetFields()[storedKey] + require.NotNil(t, optionID, "the legend-backed value must land on the stored key") + got := optionID.GetListValue().GetValues() + require.Len(t, got, 1, "a resolved select value is an option id list, not the raw string %q", + optionID.GetStringValue()) + + // and that id is the DECLARED option, minted up front by newBatch under + // the same stored key — not one discovered from the value with no order id + var relations, options int + byID := map[string]*model.SmartBlockSnapshotBase{} + for _, p := range b.pending { + byID[p.id] = p.snapshot + switch p.sbType { + case model.SmartBlockType_STRelation: + relations++ + assert.Equal(t, storedKey, p.snapshot.Details.Fields[detailRelationKey].GetStringValue()) + assert.Equal(t, "Priority", p.snapshot.Details.Fields[detailName].GetStringValue()) + case model.SmartBlockType_STRelationOption: + options++ + assert.Equal(t, storedKey, p.snapshot.Details.Fields[detailRelationKey].GetStringValue()) + assert.NotEmpty(t, p.snapshot.Details.Fields[detailOrderId].GetStringValue(), + "a declared option is pre-minted with an order id; one discovered from a value is not") + } + } + assert.Equal(t, 1, relations, "exactly one Relation object, for the stored key") + assert.Equal(t, 2, options, "both declared options, and no third discovered under the spelling") + + require.Contains(t, byID, got[0].GetStringValue()) + assert.Equal(t, "High", + byID[got[0].GetStringValue()].Details.Fields[detailName].GetStringValue()) +} diff --git a/cmd/anyblockconvert/convert.go b/cmd/anyblockconvert/convert.go new file mode 100644 index 0000000000..c9b9462ca0 --- /dev/null +++ b/cmd/anyblockconvert/convert.go @@ -0,0 +1,175 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// convertFile parses one AnyBlock JSON document and returns its object id +// (read back off the resulting snapshot, since a document may omit "id" and +// get a generated one) alongside the reconstructed snapshot. warn receives +// every warning-grade issue — an authored thing that converts but silently +// does nothing (a dropped back-relation value, an ignored bundled name, a +// grouping with nothing to group on); leaving it nil discards them, which is +// what the per-document severity tier exists to prevent. +func convertFile(inDir, path string, b *batch, normalizeIndent bool, warn func(anyblockjson.Issue)) (string, model.SmartBlockType, *model.SmartBlockSnapshotBase, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", 0, nil, fmt.Errorf("read: %w", err) + } + + genId := genIdFactory(fallbackSeed(inDir, path)) + opts := anyblockjson.Options{ + ResolveFormat: b.resolveFormat, + ResolveOptions: b, + ResolveProperties: b, + // the batch's key vocabulary: a property minted for a spelling binds + // that spelling batch-wide, so every document's detail keys land on + // the one minted key (§2e) + Keys: b, + GenerateId: genId, + NormalizeIndent: normalizeIndent, + OnWarning: warn, + } + + sbType, snap, err := anyblockjson.Unmarshal(data, opts) + if err != nil { + return "", 0, nil, fmt.Errorf("unmarshal: %w", err) + } + patchObjectTypes(sbType, snap) + patchTemplateTarget(sbType, snap, b) + + id := snap.Details.GetFields()["id"].GetStringValue() + if id == "" { + return "", 0, nil, fmt.Errorf("converted snapshot has no id") + } + return id, sbType, snap, nil +} + +// patchObjectTypes fills in the one snapshot field pkg/lib/anyblockjson +// leaves for the wiring to set: kind: "object_type" documents carry their +// identity in the envelope's "internal_key" field, not "type" (SPEC.md §2a), so +// Unmarshal never populates ObjectTypes for them. Relation/RelationOption +// documents parsed straight out of the source folder (rather than minted by +// this tool) get the same treatment, as do chat/discussion documents, whose +// bundled type key is fixed by the kind. +func patchObjectTypes(sbType model.SmartBlockType, snap *model.SmartBlockSnapshotBase) { + if len(snap.ObjectTypes) > 0 { + return + } + switch sbType { + case model.SmartBlockType_STType: + snap.ObjectTypes = []string{bundle.TypeKeyObjectType.URL()} + case model.SmartBlockType_STRelation: + snap.ObjectTypes = []string{bundle.TypeKeyRelation.URL()} + case model.SmartBlockType_STRelationOption: + snap.ObjectTypes = []string{bundle.TypeKeyRelationOption.URL()} + case model.SmartBlockType_ChatDerivedObject: + snap.ObjectTypes = []string{bundle.TypeKeyChatDerived.URL()} + case model.SmartBlockType_DiscussionObject: + snap.ObjectTypes = []string{bundle.TypeKeyDiscussion.URL()} + } +} + +// patchTemplateTarget wires a template to the type it is a template *for*. +// §2's templateFor reaches the snapshot as objectTypes[1] and nothing else, but +// that entry is a derived cache: a type's templates are found by querying the +// targetObjectType detail (core/block/template/templateimpl. +// queryTemplatesByType), and the derivation only runs the other way +// (core/block/editor/template.go, util/builtintemplate). Without the detail the +// template imports fine and belongs to no type — invisible everywhere a type +// offers its templates. +// +// The value is the target type document's own id, so the pb importer relinks it +// with every other reference in the batch; anyblockbatch.CheckTemplateTargets +// has already rejected the bundle if it cannot resolve. An authored +// targetObjectType — what a round-tripped export carries — stays authoritative, +// and is rewritten as a plain string: object-format property values normalize to +// single-element lists (SPEC.md §11), while this relation is maxCount 1 and +// every reader takes it as a string. +func patchTemplateTarget(sbType model.SmartBlockType, snap *model.SmartBlockSnapshotBase, b *batch) { + if sbType != model.SmartBlockType_Template { + return + } + id := firstString(snap.Details.GetFields()[detailTargetObjectType]) + if id == "" { + if len(snap.ObjectTypes) < 2 { + return + } + key, err := bundle.TypeKeyFromUrl(snap.ObjectTypes[1]) + if err != nil { + return + } + if id, _ = b.targetTypeId(string(key)); id == "" { + return + } + } + if snap.Details == nil { + snap.Details = &types.Struct{Fields: map[string]*types.Value{}} + } + snap.Details.Fields[detailTargetObjectType] = strVal(id) +} + +// firstString reads a detail written either as a string or as the +// single-element list an object-format property normalizes to. +func firstString(v *types.Value) string { + if s := v.GetStringValue(); s != "" { + return s + } + for _, item := range v.GetListValue().GetValues() { + if s := item.GetStringValue(); s != "" { + return s + } + } + return "" +} + +// genIdFactory returns a deterministic id generator seeded from a document's +// file path: anyblockjson.Options.GenerateId is called once for a missing +// envelope id and again for every block missing its own id (SPEC.md §9), so +// it must produce a fresh value each call, not a fixed one. +func genIdFactory(seed string) func() string { + n := 0 + return func() string { + n++ + return fmt.Sprintf("%s-%d", seed, n) + } +} + +// fallbackSeed turns a file's path relative to the input root into a stable, +// filesystem-and-id-charset-safe slug, so a re-run of this tool over +// unchanged input produces the same generated ids. +func fallbackSeed(inDir, path string) string { + rel, err := filepath.Rel(inDir, path) + if err != nil { + rel = path + } + rel = strings.TrimSuffix(rel, filepath.Ext(rel)) + return sanitizeId(rel) +} + +func sanitizeId(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + // A leading `_` is the platform's address space (§1), which no + // bundle-local id may enter, so a file named `_drafts.json` cannot seed + // one. Escaping the prefix rather than rejecting the file keeps a legal + // filename legal; `_` stays admissible everywhere else in the id, which is + // where a real key like `completion_status` needs it. + return strings.TrimLeft(b.String(), "_") +} diff --git a/cmd/anyblockconvert/convert_test.go b/cmd/anyblockconvert/convert_test.go new file mode 100644 index 0000000000..f8a5d523dd --- /dev/null +++ b/cmd/anyblockconvert/convert_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// convertDoc writes a document to a temp dir and converts it, the way the +// directory walk does. +func convertDoc(t *testing.T, b *batch, name, body string) (model.SmartBlockType, *model.SmartBlockSnapshotBase) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) + _, sbType, snap, err := convertFile(dir, path, b, false, nil) + require.NoError(t, err) + return sbType, snap +} + +// A template's target type has to reach the snapshot as the targetObjectType +// detail: that is the relation a type's templates are queried by +// (core/block/template/templateimpl.queryTemplatesByType). objectTypes[1] +// alone is a derived cache nothing reconstructs it from, so a template +// converted without the detail imports as an object no type lists. +func TestConvert_TemplateTargetTypeBecomesDetail(t *testing.T) { + b := newBatch(nil, map[string]string{"wikiPage": "type-wiki-page"}) + sbType, snap := convertDoc(t, b, "wiki-article.template.json", `{ + "version": 2, + "kind": "template", + "id": "template-wiki-article", + "type": "template", + "template_for": "wikiPage", + "properties": {"name": "Wiki Article"} + }`) + + require.Equal(t, model.SmartBlockType_Template, sbType) + assert.Equal(t, []string{"ot-template", "ot-wikiPage"}, snap.ObjectTypes) + assert.Equal(t, "type-wiki-page", + snap.Details.GetFields()[detailTargetObjectType].GetStringValue(), + "the target type document's own id, so the pb importer relinks it") +} + +// An authored targetObjectType (what a round-tripped export carries) stays +// authoritative, and lands as a plain string: object-format property values +// normalize to single-element lists (§11), but this relation is maxCount 1 and +// every reader takes it as a string. +func TestConvert_AuthoredTargetObjectTypeWinsAndIsScalar(t *testing.T) { + b := newBatch(nil, map[string]string{"wikiPage": "type-wiki-page"}) + _, snap := convertDoc(t, b, "wiki-guide.template.json", `{ + "version": 2, + "kind": "template", + "id": "template-wiki-guide", + "type": "template", + "template_for": "wikiPage", + "properties": {"name": "Wiki Guide", "targetObjectType": "type-authored"} + }`) + + assert.Equal(t, "type-authored", + snap.Details.GetFields()[detailTargetObjectType].GetStringValue()) +} + +// Nothing to wire: a non-template keeps its details untouched even when its +// type key happens to be one the batch defines. +func TestConvert_NonTemplateGetsNoTargetObjectType(t *testing.T) { + b := newBatch(nil, map[string]string{"wikiPage": "type-wiki-page"}) + _, snap := convertDoc(t, b, "page.json", `{ + "version": 2, + "id": "page-1", + "type": "wikiPage", + "properties": {"name": "A page"} + }`) + + assert.NotContains(t, snap.Details.GetFields(), detailTargetObjectType) +} + +// convert surfaces the per-document warning tier: an authored thing that +// converts but silently does nothing must not be invisible in the tool that +// produces the archive. A groupBy on a table view is exactly that — it +// survives the round trip, so nothing downstream drops it, but no view +// type other than kanban/calendar ever groups by it (§6.2). +func TestConvert_SurfacesDocumentWarnings(t *testing.T) { + b := newBatch(nil, nil) + dir := t.TempDir() + path := filepath.Join(dir, "tasks.json") + require.NoError(t, os.WriteFile(path, []byte(`{ + "version": 2, + "id": "set-1", + "properties": {"name": "Tasks"}, + "blocks": [{"type": "dataview", "views": [{"name": "All", "group_by": "status"}]}] + }`), 0o644)) + + var warnings []string + _, _, snap, err := convertFile(dir, path, b, false, func(is anyblockjson.Issue) { + warnings = append(warnings, is.String()) + }) + + require.NoError(t, err) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0], "do not group") + + var view *model.BlockContentDataviewView + for _, blk := range snap.Blocks { + if dv := blk.GetDataview(); dv != nil { + require.Len(t, dv.Views, 1) + view = dv.Views[0] + } + } + require.NotNil(t, view, "dataview block survived") + assert.Equal(t, "status", view.GroupRelationKey, + "the warned group_by is kept, not dropped — a table view just never honours it") +} + +// A document with no id gets one derived from its file path, so a filename is +// an id-minting surface like any other. `_` opens the platform's address space +// (§1) and no bundle-local id may enter it, but refusing the file would make a +// perfectly legal filename illegal — so the prefix is escaped instead. Only the +// prefix: `completion_status` is a real key and `my_page` a fine id. +func TestFallbackSeed_DoesNotMintAPlatformId(t *testing.T) { + for _, tc := range []struct{ path, want string }{ + {"_drafts.json", "drafts"}, + {"__notes.json", "notes"}, + {"objects/_set.json", "objects-_set"}, + {"my_page.json", "my_page"}, + {"a b.json", "a-b"}, + } { + seed := fallbackSeed("/in", filepath.Join("/in", tc.path)) + assert.Equal(t, tc.want, seed, tc.path) + assert.False(t, anyblockjson.IsPlatformId(genIdFactory(seed)()), + "a generated id must not enter the platform namespace: %s", tc.path) + } +} diff --git a/cmd/anyblockconvert/dictionary_test.go b/cmd/anyblockconvert/dictionary_test.go new file mode 100644 index 0000000000..fd8e82f4ae --- /dev/null +++ b/cmd/anyblockconvert/dictionary_test.go @@ -0,0 +1,75 @@ +package main + +// dictionary_test.go pins the §2f import wiring end to end: a bundle whose +// only declaration of a property is the dictionary still converts — the +// entry feeds the format table (so the undeclared-format gate passes and the +// value decodes) and the property is minted up front, with the FULL declared +// shape, whether or not any type lists it. + +import ( + "os" + "path/filepath" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// How this can fail, piece by piece of the main.go wiring: skip reading +// properties.json (CheckPropertyFormats reports the key undeclared and the +// run errors); read it but do not merge into the format table (same); merge +// but drop the pre-mint loop (the value decodes but NO relation object +// exists in the archive — the outDir assertion goes red); or shed one of +// the five §2e members between the entry and mintRelation (the details +// assertions catch the seam). +func TestRun_DictionaryDeclaredPropertyConverts(t *testing.T) { + // given: a bundle with no type documents at all — the dictionary is the + // only declaration the property has + inDir := t.TempDir() + outDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(inDir, "properties.json"), []byte(`{"version":2, + "properties":[ + {"property":"6a32d4856761631534b22f85","name":"Budget","format":"number", + "description":"Planned spend","max_count":1,"readonly":true,"default_value":100}]}`), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(inDir, "objects"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(inDir, "objects", "a.json"), []byte(`{"version":2, + "id":"obj-a", + "property_internal_keys":{"budget":"6a32d4856761631534b22f85"}, + "properties":{"name":"A","budget":250}}`), 0o644)) + + // when + require.NoError(t, run(inDir, outDir, false, false, formatPb)) + + // then: the relation object exists in the archive, with the whole shape + relPath := "" + filepath.Walk(filepath.Join(outDir, "relations"), func(p string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() { + relPath = p + } + return nil + }) + require.NotEmpty(t, relPath, "the dictionary-declared property must be minted without any type listing it") + data, err := os.ReadFile(relPath) + require.NoError(t, err) + var sw pb.SnapshotWithType + require.NoError(t, proto.Unmarshal(data, &sw)) + det := sw.Snapshot.GetData().GetDetails().GetFields() + assert.Equal(t, "Budget", det["name"].GetStringValue()) + assert.Equal(t, float64(model.RelationFormat_number), det["relationFormat"].GetNumberValue()) + assert.Equal(t, "Planned spend", det["description"].GetStringValue()) + assert.Equal(t, float64(1), det["relationMaxCount"].GetNumberValue()) + assert.True(t, det["relationReadonlyValue"].GetBoolValue()) + assert.Equal(t, float64(100), det["relationDefaultValue"].GetNumberValue()) + + // and the value decoded against the declared format + objData, err := os.ReadFile(filepath.Join(outDir, "objects", "obj-a.pb")) + require.NoError(t, err) + var obj pb.SnapshotWithType + require.NoError(t, proto.Unmarshal(objData, &obj)) + assert.Equal(t, float64(250), + obj.Snapshot.GetData().GetDetails().GetFields()["6a32d4856761631534b22f85"].GetNumberValue()) +} diff --git a/cmd/anyblockconvert/files.go b/cmd/anyblockconvert/files.go new file mode 100644 index 0000000000..e389efa6d9 --- /dev/null +++ b/cmd/anyblockconvert/files.go @@ -0,0 +1,185 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// copyBundleFiles copies a bundle's files/ directory — the real binary +// assets a "file_object" document's "source" property points at (SPEC.md +// §2c, §3: iconImage and any object-level image/file reference resolve by +// looking up that property) — into the output archive at the same relative +// path. +// +// anyblockconvert only ever discovers *.json documents +// (anyblockbatch.DiscoverJSONFiles), and anyblockinstall.sh's zip step only +// packs whatever this tool wrote to outDir. Neither step ever reads raw +// bytes off disk on its own, so without this, a bundle authoring +// "source": "files/icon.png" against a real PNG converts clean and installs +// with no icon: core/block/import/pb's normalizeFilePath resolves that path +// against the *archive* being imported, not the source bundle, and finds +// nothing there. +func copyBundleFiles(inDir, outDir string) (int, error) { + src := filepath.Join(inDir, "files") + info, err := os.Stat(src) + if os.IsNotExist(err) { + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("stat %s: %w", src, err) + } + if !info.IsDir() { + return 0, fmt.Errorf("%s exists but is not a directory", src) + } + + dst := filepath.Join(outDir, "files") + var n int + walkErr := filepath.Walk(src, func(p string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if fi.IsDir() || fi.Name() == ".DS_Store" { + return nil + } + rel, relErr := filepath.Rel(src, p) + if relErr != nil { + return relErr + } + if copyErr := copyFile(p, filepath.Join(dst, rel)); copyErr != nil { + return fmt.Errorf("copy %s: %w", rel, copyErr) + } + n++ + return nil + }) + return n, walkErr +} + +func copyFile(src, dst string) error { + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Close() +} + +// danglingSource is a document whose "source" property (SPEC.md §3: the +// property a fileObject's real bytes are found by) names a files/ path that +// does not exist in the bundle. Exactly the class of bug CheckTargetTypes +// catches for object references: valid JSON, valid schema, silently wrong +// once installed — here it's a blank icon or a dead file block instead of a +// dangling id. +type danglingSource struct { + file, source string +} + +// checkFileSources scans every document's top-level "properties.source" for +// a files/-relative path and confirms it exists under inDir. It only looks +// at the object-level "source" property (the one iconImage/fileObject +// resolution reads via bundle.RelationKeySource) — not block-level file +// content, which carries its own objectId/hash instead. +func checkFileSources(inDir string, files []string) ([]danglingSource, error) { + var out []danglingSource + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var doc struct { + Properties map[string]any `json:"properties"` + } + if err := json.Unmarshal(data, &doc); err != nil { + // malformed JSON is reported by the real schema validation this + // tool already runs elsewhere; skip it here rather than double-report + continue + } + src, ok := doc.Properties["source"].(string) + if !ok || !strings.HasPrefix(src, "files/") { + continue + } + if _, statErr := os.Stat(filepath.Join(inDir, filepath.FromSlash(src))); os.IsNotExist(statErr) { + out = append(out, danglingSource{file: f, source: src}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].file < out[j].file }) + return out, nil +} + +func reportDanglingSources(ds []danglingSource) string { + var b strings.Builder + for _, d := range ds { + fmt.Fprintf(&b, " %s: source %q names no file in the bundle's files/ directory\n", d.file, d.source) + } + return b.String() +} + +// copyManifestBlobs copies each manifest-bound blob (§2c, v0.47) into the +// output archive at its own bundle-relative path. The map, not the files/ +// layout convention, is the binding — an authored bundle points at assets +// laid out however it likes — so this walks the MAP, where copyBundleFiles +// walks the conventional directory; a blob under files/ is copied by both, +// and the second copy is a byte-identical overwrite. CheckManifestFiles has +// already refused a path that escapes the bundle or names nothing, so a +// failure here is an I/O error, not a dangling binding. +func copyManifestBlobs(inDir, outDir string, blobs map[string]string) (int, error) { + paths := make([]string, 0, len(blobs)) + seen := map[string]bool{} + for _, rel := range blobs { + if !seen[rel] { + seen[rel] = true + paths = append(paths, rel) + } + } + sort.Strings(paths) + var n int + for _, rel := range paths { + src := filepath.Join(inDir, filepath.FromSlash(rel)) + dst := filepath.Join(outDir, filepath.FromSlash(rel)) + if err := copyFile(src, dst); err != nil { + return n, fmt.Errorf("copy %s: %w", rel, err) + } + n++ + } + return n, nil +} + +// bindBlobSource writes a converted file snapshot's `source` detail from +// the manifest binding — the pb importer's own contract for locating a +// file's bytes (normalizeFilePath resolves bundle.RelationKeySource against +// the archive). The clobber the format banished from its DOCUMENTS is +// legitimate here: the archive is a transport artifact, and this detail is +// how that transport has always carried the path. +func bindBlobSource(snap *model.SmartBlockSnapshotBase, blobPath string) { + if snap == nil { + return + } + if snap.Details == nil { + snap.Details = &types.Struct{} + } + if snap.Details.Fields == nil { + snap.Details.Fields = map[string]*types.Value{} + } + snap.Details.Fields["source"] = &types.Value{Kind: &types.Value_StringValue{StringValue: blobPath}} +} diff --git a/cmd/anyblockconvert/files_test.go b/cmd/anyblockconvert/files_test.go new file mode 100644 index 0000000000..0524ce47e4 --- /dev/null +++ b/cmd/anyblockconvert/files_test.go @@ -0,0 +1,184 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pb" +) + +func TestCopyBundleFiles_NoFilesDir(t *testing.T) { + in := t.TempDir() + out := t.TempDir() + + n, err := copyBundleFiles(in, out) + require.NoError(t, err) + assert.Equal(t, 0, n) + _, statErr := os.Stat(filepath.Join(out, "files")) + assert.True(t, os.IsNotExist(statErr), "no files/ dir should be created when the bundle has none") +} + +func TestCopyBundleFiles_CopiesAndSkipsDSStore(t *testing.T) { + in := t.TempDir() + out := t.TempDir() + + filesDir := filepath.Join(in, "files") + require.NoError(t, os.MkdirAll(filesDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(filesDir, "icon.png"), []byte("fake-png-bytes"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(filesDir, ".DS_Store"), []byte("junk"), 0o644)) + + n, err := copyBundleFiles(in, out) + require.NoError(t, err) + assert.Equal(t, 1, n) + + got, err := os.ReadFile(filepath.Join(out, "files", "icon.png")) + require.NoError(t, err) + assert.Equal(t, "fake-png-bytes", string(got)) + + _, statErr := os.Stat(filepath.Join(out, "files", ".DS_Store")) + assert.True(t, os.IsNotExist(statErr), ".DS_Store must not ride along as a bogus archive entry") +} + +func TestCopyBundleFiles_PreservesSubdirectories(t *testing.T) { + in := t.TempDir() + out := t.TempDir() + + nested := filepath.Join(in, "files", "sub") + require.NoError(t, os.MkdirAll(nested, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nested, "cover.jpg"), []byte("jpg"), 0o644)) + + n, err := copyBundleFiles(in, out) + require.NoError(t, err) + assert.Equal(t, 1, n) + + got, err := os.ReadFile(filepath.Join(out, "files", "sub", "cover.jpg")) + require.NoError(t, err) + assert.Equal(t, "jpg", string(got)) +} + +func writeJSON(t *testing.T, dir, name, body string) string { + t.Helper() + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) + return path +} + +func TestCheckFileSources_FlagsMissingFile(t *testing.T) { + in := t.TempDir() + f := writeJSON(t, in, "icon.json", `{ + "version": 2, + "kind": "file_object", + "id": "icon-1", + "properties": {"name": "icon-1", "source": "files/icon.png"} + }`) + + dangling, err := checkFileSources(in, []string{f}) + require.NoError(t, err) + require.Len(t, dangling, 1) + assert.Equal(t, "files/icon.png", dangling[0].source) +} + +func TestCheckFileSources_PassesWhenFileExists(t *testing.T) { + in := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(in, "files"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(in, "files", "icon.png"), []byte("x"), 0o644)) + f := writeJSON(t, in, "icon.json", `{ + "version": 2, + "kind": "file_object", + "id": "icon-1", + "properties": {"name": "icon-1", "source": "files/icon.png"} + }`) + + dangling, err := checkFileSources(in, []string{f}) + require.NoError(t, err) + assert.Empty(t, dangling) +} + +func TestCheckFileSources_IgnoresNonFilesSource(t *testing.T) { + in := t.TempDir() + // a "source" property that isn't a files/ path is someone else's use of + // the key (e.g. a bookmark's URL) and is none of this check's business + f := writeJSON(t, in, "bookmark.json", `{ + "version": 2, + "id": "bm-1", + "properties": {"name": "Anytype", "source": "https://anytype.io"} + }`) + + dangling, err := checkFileSources(in, []string{f}) + require.NoError(t, err) + assert.Empty(t, dangling) +} + +// The manifest `files` map has a reader, and this is it (§2c, v0.47): each +// binding is copied into the archive at its own relative path — the map, +// not the files/ convention, is the binding, so an authored bundle's +// assets/ layout travels — and the converted snapshot's `source` detail is +// written from the map, because that detail is the pb importer's own +// contract for locating a file's bytes (normalizeFilePath). Without this, +// a native bundle's blobs attach to nothing: the documents carry no path +// on purpose, and the archive would install with blank icons and dead +// file blocks. +// +// How this can fail: copy only files/ (the authored assets/ blob never +// reaches the archive); skip the source injection (the blob travels but no +// importer ever looks at it); or key the injection by the minted output id +// instead of the document's envelope id (the binding misses every +// re-minted document). +func TestRun_ManifestBindsBlobsIntoTheArchive(t *testing.T) { + inDir := t.TempDir() + outDir := t.TempDir() + write := func(rel, body string) { + path := filepath.Join(inDir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) + } + write("files/img.anyblock.json", `{"version":2,"kind":"file_object","id":"img-1","properties":{"name":"Logo"}}`) + write("assets/logo.png", "png bytes") + write("index.json", `{"version":2,"manifest":{"files":{"img-1":"assets/logo.png"}}}`) + + // when + require.NoError(t, run(inDir, outDir, false, false, formatPb)) + + // then: the blob travelled at its authored path… + copied, err := os.ReadFile(filepath.Join(outDir, "assets", "logo.png")) + require.NoError(t, err) + assert.Equal(t, "png bytes", string(copied)) + + // …and the snapshot carries the binding as its source detail + var found bool + filepath.Walk(outDir, func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() || filepath.Ext(p) != ".pb" { + return nil + } + data, readErr := os.ReadFile(p) + require.NoError(t, readErr) + var sw pb.SnapshotWithType + require.NoError(t, proto.Unmarshal(data, &sw)) + det := sw.Snapshot.GetData().GetDetails().GetFields() + if det["id"].GetStringValue() != "img-1" { + return nil + } + found = true + assert.Equal(t, "assets/logo.png", det["source"].GetStringValue(), + "the manifest binding reaches the archive as the source detail") + return nil + }) + require.True(t, found, "the file document must convert") + + t.Run("a binding the bundle cannot honour refuses the whole conversion", func(t *testing.T) { + badIn, badOut := t.TempDir(), t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(badIn, "files"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(badIn, "files", "img.anyblock.json"), + []byte(`{"version":2,"kind":"file_object","id":"img-1","properties":{"name":"Logo"}}`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(badIn, "index.json"), + []byte(`{"version":2,"manifest":{"files":{"img-1":"assets/missing.png"}}}`), 0o644)) + err := run(badIn, badOut, false, false, formatPb) + require.Error(t, err) + assert.Contains(t, err.Error(), "manifest file binding") + }) +} diff --git a/cmd/anyblockconvert/main.go b/cmd/anyblockconvert/main.go new file mode 100644 index 0000000000..6f5a098edf --- /dev/null +++ b/cmd/anyblockconvert/main.go @@ -0,0 +1,361 @@ +// anyblockconvert converts a directory tree of AnyBlock JSON documents +// (pkg/lib/anyblockjson) into a directory of old-format pb snapshots, laid +// out the way the bundled use-case archives are (util/builtinobjects/data/ +// *.zip): one .pb file per object, under objects/types/relations/ +// relationsOptions/templates subdirectories. The result can be fed straight +// to the existing pb importer (core/block/import/pb) or zipped into a +// builtinobjects archive. +// +// pkg/lib/anyblockjson.Unmarshal only ever sees one document at a time and +// deliberately leaves cross-document concerns to the caller (SPEC.md calls +// this "the import wiring's job"): resolving a custom property's format, +// and minting the Relation/RelationOption objects a custom property or +// select/multiSelect option needs to exist as (the pb importer relinks ids +// across a batch, but — unlike CSV/Notion import — does not synthesize +// missing relation or option objects on its own). This tool is that wiring, +// built the same way core/block/import/csv and core/block/import/notion +// build their Relation/RelationOption snapshots. +// +// Every document's own "id" (or a deterministic fallback derived from its +// file path, when omitted) passes through untouched: the pb importer already +// relinks references by matching literal id strings across the whole batch +// (core/block/import/common.UpdateLinksToObjects / +// UpdateObjectIDsInRelations), so there's nothing for this tool to rewrite +// there. +// +// Usage: +// +// go run ./cmd/anyblockconvert -in ~/usecase2/anyblock/01-company-wiki -out ./out +// +// Pass -format json to write jsonpb text (.json) instead of raw proto bytes +// (.pb) — human-readable, and still accepted by core/block/import/pb. +package main + +import ( + "flag" + "fmt" + "github.com/anyproto/anytype-heart/cmd/internal/anyblockbatch" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "os" + "path/filepath" +) + +func main() { + var ( + inDir = flag.String("in", "", "input directory of AnyBlock JSON documents (searched recursively)") + outDir = flag.String("out", "", "output directory for pb snapshots") + normalizeIndent = flag.Bool("normalize-indent", false, "clamp over-deep block indents instead of rejecting them (SPEC.md §4)") + lenient = flag.Bool("lenient", false, "downgrade undeclared-property errors to warnings (SPEC.md §3: such values pass through as raw JSON)") + format = flag.String("format", "pb", "output snapshot format: \"pb\" (raw proto, .pb) or \"json\" (jsonpb text, .json; human-readable, still importable via core/block/import/pb)") + ) + flag.Parse() + + if *inDir == "" || *outDir == "" { + flag.Usage() + fmt.Fprintln(os.Stderr, "\nboth -in and -out are required") + os.Exit(2) + } + var outFormat outputFormat + switch *format { + case "pb": + outFormat = formatPb + case "json": + outFormat = formatJSON + default: + fmt.Fprintf(os.Stderr, "invalid -format %q: must be \"pb\" or \"json\"\n", *format) + os.Exit(2) + } + if err := run(*inDir, *outDir, *normalizeIndent, *lenient, outFormat); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +func run(inDir, outDir string, normalizeIndent, lenient bool, format outputFormat) error { + files, err := anyblockbatch.DiscoverJSONFiles(inDir) + if err != nil { + return fmt.Errorf("discover input files: %w", err) + } + if len(files) == 0 { + return fmt.Errorf("no .json files found under %s", inDir) + } + + // types declare what objects reference, so convert them first + files, err = anyblockbatch.OrderTypesFirst(files) + if err != nil { + return fmt.Errorf("order input files: %w", err) + } + + // the `_` namespace belongs to the platform (§1), and the reserved + // index.json listings live in it. Checked before anything is written: an + // id this tool would refuse must not first appear as a converted snapshot + // on disk. + reservedIds, err := anyblockbatch.CheckBundleIds(files) + if err != nil { + return fmt.Errorf("check bundle ids: %w", err) + } + if len(reservedIds) > 0 { + return fmt.Errorf("%d object%s claiming a reserved id:\n%s", + len(reservedIds), plural2(len(reservedIds)), anyblockbatch.ReportTargets(reservedIds)) + } + + formats, err := anyblockbatch.ScanFormats(files) + if err != nil { + return fmt.Errorf("scan property formats: %w", err) + } + + // the property dictionary (§2f) is a declaration source beside the type + // documents: an author declares a property once, bundle-wide, without + // writing a relation document at all. Its entries join the format table + // (dictionary winning a stated conflict) and are pre-minted below, so a + // dictionary-declared property exists in the archive whether or not any + // type happens to list it. + var dictDefs []anyblockjson.PropertyDefinition + if dictPath, ok := anyblockbatch.PropertiesPath(inDir); ok { + dictFormats, defs, dictErr := anyblockbatch.DictionaryFormats(dictPath) + if dictErr != nil { + return fmt.Errorf("read property dictionary: %w", dictErr) + } + dictDefs = defs + formats = anyblockbatch.MergeDictionaryFormats(formats, dictFormats, func(format string, args ...any) { + fmt.Fprintf(os.Stderr, "warning: "+format+"\n", args...) + }) + } + + if shared, serr := anyblockbatch.CheckSharedSelects(files); serr == nil && len(shared) > 0 { + fmt.Fprintf(os.Stderr, "warning: %d select propert%s shared across types:\n%s", + len(shared), plural(len(shared)), anyblockbatch.ReportSharedSelects(shared)) + } + + // a property whose format nothing declares is decoded as raw JSON: dates + // stay strings, selects mint no options, and object references are never + // remapped, so cross-object links break silently after import + undeclared, err := anyblockbatch.CheckPropertyFormats(files, formats) + if err != nil { + return fmt.Errorf("check property formats: %w", err) + } + if len(undeclared) > 0 { + if !lenient { + return fmt.Errorf("%d propert%s with no declared format:\n%spass -lenient to convert anyway", + len(undeclared), plural(len(undeclared)), anyblockbatch.Report(undeclared)) + } + fmt.Fprintf(os.Stderr, "warning: %d propert%s with no declared format:\n%s", + len(undeclared), plural(len(undeclared)), anyblockbatch.Report(undeclared)) + } + typeIds, err := anyblockbatch.TypeIds(files) + if err != nil { + return fmt.Errorf("index type ids: %w", err) + } + badTargets, err := anyblockbatch.CheckTargetTypes(files, typeIds) + if err != nil { + return fmt.Errorf("check target types: %w", err) + } + if len(badTargets) > 0 { + return fmt.Errorf("%d unresolvable object_types target%s:\n%s", + len(badTargets), map[bool]string{true: "", false: "s"}[len(badTargets) == 1], + anyblockbatch.ReportTargets(badTargets)) + } + // a template whose target type cannot be wired imports as an object no type + // lists — valid, converted, and unreachable + badTemplates, err := anyblockbatch.CheckTemplateTargets(files, typeIds) + if err != nil { + return fmt.Errorf("check template targets: %w", err) + } + if len(badTemplates) > 0 { + return fmt.Errorf("%d template%s with no wirable target type:\n%s", + len(badTemplates), map[bool]string{true: "", false: "s"}[len(badTemplates) == 1], + anyblockbatch.ReportTemplateTargets(badTemplates)) + } + // the bundle index is loaded BEFORE conversion: its manifest `files` + // map is the authoritative binding between a file document and its + // bytes (§2c, v0.47), and this tool is that map's reader — each bound + // blob is copied into the archive and the converted snapshot's `source` + // detail is written from the map, because `source` is the pb importer's + // own contract (normalizeFilePath resolves it against the archive). The + // clobber that was banished from the FORMAT document is legitimate at + // the archive boundary: the archive is a transport artifact, not a + // document. + var idx *anyblockjson.Index + idxPath, hasIndex := anyblockbatch.IndexPath(inDir) + if hasIndex { + data, readErr := os.ReadFile(idxPath) + if readErr != nil { + return fmt.Errorf("read %s: %w", idxPath, readErr) + } + if idx, err = anyblockjson.UnmarshalIndex(data); err != nil { + return fmt.Errorf("%s: %w", anyblockjson.IndexFileName, err) + } + if bad := anyblockbatch.CheckManifestFiles(idx, inDir, files); len(bad) > 0 { + return fmt.Errorf("%d manifest file binding%s that cannot be honoured:\n%s", + len(bad), plural2(len(bad)), anyblockbatch.ReportTargets(bad)) + } + for _, unbound := range anyblockbatch.UnboundFileDocuments(idx, files) { + fmt.Fprintf(os.Stderr, "warning: file document %q has no manifest.files binding — it converts, but its bytes did not travel and it will install without content\n", unbound) + } + } + manifestBlobs := map[string]string{} + if idx != nil && idx.Manifest != nil { + manifestBlobs = idx.Manifest.Files + } + + // a fileObject's real bytes are found by its "source" property + // (SPEC.md §3) at import time — catch a bundle pointing at a file that + // was never placed under files/ now, not as a silently blank icon later + danglingSources, err := checkFileSources(inDir, files) + if err != nil { + return fmt.Errorf("check file sources: %w", err) + } + if len(danglingSources) > 0 { + return fmt.Errorf("%d dangling file source%s:\n%s", + len(danglingSources), plural2(len(danglingSources)), reportDanglingSources(danglingSources)) + } + + b := newBatch(formats, typeIds) + + // dictionary-declared properties exist up front, with the FULL declared + // shape — description, include_time, max_count, readonly, default_value + // all reach mintRelation (§2e: a member the file admits is never shed at + // the seam). Entry order is the dictionary's canonical sorted order, so + // minted ids are stable across runs like everything else in the batch. + for _, def := range dictDefs { + b.PropertyId(def) + } + + if err := os.MkdirAll(outDir, 0o755); err != nil { + return fmt.Errorf("mkdir %s: %w", outDir, err) + } + + // real binary assets (SPEC.md §2c, §3: iconImage, fileObject "source") + // live in files/ alongside the JSON documents; DiscoverJSONFiles only + // ever sees the *.json ones, so nothing else copies these into outDir + copiedFiles, err := copyBundleFiles(inDir, outDir) + if err != nil { + return fmt.Errorf("copy bundle files: %w", err) + } + if copiedFiles > 0 { + fmt.Printf("copied %d file(s) into %s\n", copiedFiles, filepath.Join(outDir, "files")) + } + // manifest-bound blobs may live ANYWHERE the author laid them out — + // the map, not the files/ convention, is the binding (§2c) — so each is + // copied at its own relative path; ones under files/ were copied above + // and this overwrite is a no-op for them + copiedBlobs, err := copyManifestBlobs(inDir, outDir, manifestBlobs) + if err != nil { + return fmt.Errorf("copy manifest blobs: %w", err) + } + if copiedBlobs > 0 { + fmt.Printf("bound %d manifest blob(s)\n", copiedBlobs) + } + + var failed int + var converted int + var warned int + for _, f := range files { + id, sbType, snap, err := convertFile(inDir, f, b, normalizeIndent, func(is anyblockjson.Issue) { + warned++ + fmt.Fprintf(os.Stderr, "warning: %s: %v\n", f, is) + }) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL %s: %v\n", f, err) + failed++ + continue + } + // the manifest binding reaches the archive as the `source` detail — + // the pb importer's own contract for locating a file's bytes + if blobPath, bound := manifestBlobs[id]; bound { + bindBlobSource(snap, blobPath) + } + if err := writeSnapshot(outDir, id, sbType, snap, format); err != nil { + fmt.Fprintf(os.Stderr, "FAIL %s: write: %v\n", f, err) + failed++ + continue + } + converted++ + } + + // the bundle index (§2c) becomes two outputs: the profile file, which the + // installer reads for the space's homepage, and a Widget snapshot, which is + // how the sidebar reaches a space installed as an experience (see + // widgets.go). Written after the snapshots so a failed conversion does not + // leave either pointing at nothing. + if hasIndex { + if dangling := anyblockbatch.CheckIndexTargets(idx, files); len(dangling) > 0 { + return fmt.Errorf("%d unresolvable reference%s in %s:\n%s", + len(dangling), plural2(len(dangling)), anyblockjson.IndexFileName, + anyblockbatch.ReportTargets(dangling)) + } + names, nameErr := anyblockbatch.ObjectNames(files) + if nameErr != nil { + return fmt.Errorf("index object names: %w", nameErr) + } + if err := writeProfile(outDir, idx, names); err != nil { + return fmt.Errorf("write profile: %w", err) + } + // the Widget snapshot carries the id "widgets"; an object claiming the + // same one would share both that id — and so the importer's relinking + // entry for it — and the output file with the sidebar. That id is also + // the wire spelling of the reserved `_widgets` homepage, so + // CheckBundleIds has already refused it above, for its own reason and + // before anything was written. This is the backstop for the case where + // the two stop being the same string. + if _, taken := names[anyblockjson.WidgetsObjectId]; taken { + return fmt.Errorf("an object in the bundle has id %q, which is reserved for the sidebar snapshot (SPEC.md §2c) — rename it", anyblockjson.WidgetsObjectId) + } + if err := writeWidgets(outDir, idx, format); err != nil { + return fmt.Errorf("write widgets: %w", err) + } + entry := idx.EffectiveEntryPoint() + if entry == "" { + entry = "(nothing — no widget names an object)" + } + home := idx.SpaceHomepage() + if home == "" { + home = "(the widgets screen)" + } + fmt.Printf("profile written: space %q, homepage %s\n", idx.Name, home) + fmt.Printf("widgets written: %d sidebar widget(s), in index.json order\n", len(idx.Widgets)) + // TEMPORARY: on the built-in-archive path inject() opens + // widgets[0].targetObjectId, because pb.Profile has no entry-point + // field. (On the experience path nothing opens once at all — see §2c — + // so there the entrypoint only matters through the homepage fallback.) + if declared := idx.EntryPoint(); declared != "" && declared != entry { + fmt.Fprintf(os.Stderr, "warning: entrypoint %q is not the first widget, so on the built-in-archive path it is not what opens — inject uses widgets[0] (%s)\n", + declared, entry) + } + } else { + fmt.Fprintf(os.Stderr, "warning: no %s — the space gets no name, no entry point and no sidebar (SPEC.md \u00a72c)\n", + anyblockjson.IndexFileName) + } + + for _, p := range b.pending { + if err := writeSnapshot(outDir, p.id, p.sbType, p.snapshot, format); err != nil { + return fmt.Errorf("write synthesized %s: %w", p.id, err) + } + } + + fmt.Printf("\n%d documents converted, %d failed", converted, failed) + if warned > 0 { + fmt.Printf(", %d warning(s)", warned) + } + fmt.Println() + fmt.Printf("synthesized %d relations, %d relation options\n", b.relationCount(), b.optionCount()) + fmt.Println("output:", outDir) + if failed > 0 { + return fmt.Errorf("%d documents failed to convert", failed) + } + return nil +} + +func plural(n int) string { + if n == 1 { + return "y" + } + return "ies" +} + +func plural2(n int) string { + if n == 1 { + return "" + } + return "s" +} diff --git a/cmd/anyblockconvert/mint_test.go b/cmd/anyblockconvert/mint_test.go new file mode 100644 index 0000000000..c4af2513dc --- /dev/null +++ b/cmd/anyblockconvert/mint_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" +) + +// A property an author declared by NAME or spelling gets a fresh internal key +// minted, the way the app mints one when a user creates a property +// (objectcreator/relation.go: bson.NewObjectId().Hex()). A property that +// STATED its internal_key keeps it exactly — that is the whole point of +// stating one: a bundle re-imported elsewhere reproduces the same stored key. +// +// This tool used to key the relation by the spelling in both cases, so a +// generated bundle imported with a stored key no real space would have. +// +// How this can fail: mint for a stated key and re-import stops reproducing +// the space; key by the spelling and a generated bundle diverges from what +// the app creates. +func TestMint_ASpellingGetsAFreshKeyAStatedKeyIsKept(t *testing.T) { + b := newBatch(nil, nil) + + authored := anyblockjson.PropertyDefinition{Key: "cooking_time", Name: "Cooking Time"} + id, ok := b.PropertyId(authored) + require.True(t, ok) + assert.NotContains(t, id, "cooking_time", "the spelling must not become the stored key") + + minted, ok := b.PropertyKey("cooking_time") + require.True(t, ok, "the spelling must be bound in the batch vocabulary") + assert.True(t, looksLikeMintedKey(minted), "a minted key is a bson id, got %q", minted) + + t.Run("the same spelling resolves to the same key across the batch", func(t *testing.T) { + // this is what makes minting safe: a recipe document carrying + // "cooking_time": 90 must land on the very key the dictionary minted, + // not write a detail no relation object describes + again, _ := b.PropertyId(authored) + assert.Equal(t, id, again) + + key2, _ := b.PropertyKey("cooking_time") + assert.Equal(t, minted, key2) + assert.Equal(t, "cooking_time", b.PropertySlug(minted), "and the binding inverts") + }) + + t.Run("a stated internal key is reproduced exactly", func(t *testing.T) { + stated := anyblockjson.PropertyDefinition{ + Key: "6a32d4856761631534b22f85", Name: "Project", KeyIsInternal: true} + id, ok := b.PropertyId(stated) + require.True(t, ok) + assert.Contains(t, id, "6a32d4856761631534b22f85") + }) + + t.Run("a bundled property is never minted", func(t *testing.T) { + id, ok := b.PropertyId(anyblockjson.PropertyDefinition{Key: "dueDate", Name: "Due date"}) + require.True(t, ok) + assert.Contains(t, id, "dueDate", "a bundled key resolves to its bundled url") + }) +} diff --git a/cmd/anyblockconvert/optionlegend_test.go b/cmd/anyblockconvert/optionlegend_test.go new file mode 100644 index 0000000000..19cca572f2 --- /dev/null +++ b/cmd/anyblockconvert/optionlegend_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/cmd/internal/anyblockbatch" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The `option_ids` legend (SPEC §3, §9a) is resolved inside +// pkg/lib/anyblockjson, but whether it does anything at all is decided HERE: +// the package honours a legend id only where the wired OptionResolver +// confirms it is a live option of that relation, and the confirming call is +// OptionName. This tool implements that resolver, so the legend is live for +// the flagship import path only as long as the two sides agree about what +// OptionName means — and they are declared in different packages, one as an +// interface method with two duties and one as a method on `batch`. +// +// They did not agree. OptionName was an export-only call when this tool was +// written ("what is this option id called?"), so `batch` stubbed it to false; +// the legend then made it the import-side liveness check, and every legend +// entry silently failed liveness — anyblockconvert ignored `option_ids` +// wholesale while nothing failed anywhere. Nothing structural stops that +// happening again, so the guard has to be a test that crosses the boundary. +// It asserts the two properties that matter across the seam and nothing about +// how either side is built: +// +// - an id this resolver SERVES is honoured, in preference to the name; +// - an id it does NOT serve is not, and never reaches the snapshot — the +// archive must not reference an option object it does not carry. +func TestBatch_OptionIdsLegendCrossesIntoThePackage(t *testing.T) { + newStageBatch := func() *batch { + return newBatch(map[string]anyblockbatch.FormatInfo{ + "stage": { + Format: model.RelationFormat_status, + Options: vocab("Blocked", "Done"), + }, + }, nil) + } + + t.Run("an id this batch serves wins over the name", func(t *testing.T) { + b := newStageBatch() + blocked, ok := b.OptionId(domain.RelationKey("stage"), "Blocked") + require.True(t, ok) + done, ok := b.OptionId(domain.RelationKey("stage"), "Done") + require.True(t, ok) + require.NotEqual(t, blocked, done, "the decoy only works if the two options are distinct objects") + + // the rename case the legend exists for: the value still spells the + // old name, the legend carries the id the name stood for, and the + // option now goes by something else. Name resolution would answer + // `done` — a DIFFERENT live option of the same relation, so the + // assertion below cannot pass by agreeing with the fallback. + _, snap := convertDoc(t, b, "renamed.json", stageDoc(t, "Done", map[string]string{"Done": blocked})) + + got := optionValues(t, snap) + assert.Equal(t, []string{blocked}, got, + "the legend id must win: %q is what resolving the name alone answers", done) + assertEveryOptionMinted(t, b, got) + }) + + t.Run("an id from another space is not honoured", func(t *testing.T) { + b := newStageBatch() + done, ok := b.OptionId(domain.RelationKey("stage"), "Done") + require.True(t, ok) + + // what every real bundle carries: ids minted by the space that + // exported it, naming nothing in the archive being built + _, snap := convertDoc(t, b, "foreign.json", stageDoc(t, "Done", map[string]string{"Done": "bafydecoy"})) + + got := optionValues(t, snap) + assert.Equal(t, []string{done}, got, + "a foreign id fails liveness, so the value resolves by name as if the legend were absent") + assertEveryOptionMinted(t, b, got) + }) +} + +// stageDoc is a one-property document: a `stage` value, and optionally the +// `option_ids` entry for it. +func stageDoc(t *testing.T, value string, legend map[string]string) string { + t.Helper() + doc := map[string]any{ + "version": 2, + "id": "obj-1", + "properties": map[string]any{"stage": value}, + } + if legend != nil { + doc["option_ids"] = map[string]any{"stage": legend} + } + raw, err := json.Marshal(doc) + require.NoError(t, err) + return string(raw) +} + +// optionValues reads the resolved `stage` value off the snapshot. +func optionValues(t *testing.T, snap *model.SmartBlockSnapshotBase) []string { + t.Helper() + v := snap.Details.GetFields()["stage"] + require.NotNil(t, v, "the value must reach the detail") + var out []string + for _, item := range v.GetListValue().GetValues() { + out = append(out, item.GetStringValue()) + } + require.NotEmpty(t, out, "a resolved select value is an option id list, not the raw string %q", v.GetStringValue()) + return out +} + +// assertEveryOptionMinted is the safety half: an option id the converter +// writes into a snapshot must be an object the same batch also carries, or +// the archive imports with a dangling reference. +func assertEveryOptionMinted(t *testing.T, b *batch, ids []string) { + t.Helper() + minted := map[string]bool{} + for _, p := range b.pending { + if p.sbType == model.SmartBlockType_STRelationOption { + minted[p.id] = true + } + } + for _, id := range ids { + assert.True(t, minted[id], "option id %q reaches the snapshot but no RelationOption object is minted for it", id) + } +} diff --git a/cmd/anyblockconvert/profile.go b/cmd/anyblockconvert/profile.go new file mode 100644 index 0000000000..ac54fd748f --- /dev/null +++ b/cmd/anyblockconvert/profile.go @@ -0,0 +1,104 @@ +package main + +// profile.go writes the archive's `profile` file from index.json (§2c). It is +// the one output that describes the bundle rather than an object: +// util/builtinobjects reads it with pb.Profile.Unmarshal, so it is raw +// protobuf regardless of the snapshot format. +// +// How much of it is honoured depends on which path installs the archive, and +// a bundle only ever takes one of them: +// +// - inject() — the built-in use-case archives — reads all of it: name, +// avatar, spaceDashboardId, and widgets (getWidgets + createWidgets). +// - CreateObjectsForExperience — what ObjectImportExperience calls, and so +// what every bundle this tool produces goes through — reads name, avatar +// and spaceDashboardId, on a NEW-space install (setWorkspaceSettings +// with isBundle=true, gated on isNewSpace, so a created space takes the +// bundle's identity and an existing space keeps its own). It never reads +// profile.widgets: getWidgets belongs to inject, and the one +// createWidgets call on this path is the Markdown/AI branch's, built +// from the manifest's dashboard page rather than from this file. +// +// The sidebar arrives as a Widget snapshot in the archive (widgets.go), +// which is how a real app export carries it. + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/constant" +) + +// widgetLayouts maps §2c layout names to the wire enum. Absent means link, +// the enum's own zero value. +var widgetLayouts = map[string]model.BlockContentWidgetLayout{ + "link": model.BlockContentWidget_Link, + "tree": model.BlockContentWidget_Tree, + "list": model.BlockContentWidget_List, + "compact_list": model.BlockContentWidget_CompactList, + "view": model.BlockContentWidget_View, +} + +// writeProfile renders index.json as the archive's profile file. +// +// Ids stay the bundle's own: the installer maps them through oldAnytypeID +// (builtinobjects.getNewObjectId), the same relinking every other reference +// gets, so nothing here needs the post-import ids. +// +// TEMPORARY: the entry point is carried as widgets[0], because pb.Profile has +// no field of its own for it — inject reads widgets[0].targetObjectId. A +// declared entrypoint that is not the first widget is therefore not honoured, +// and anyblockvalidate warns about it rather than this reordering the sidebar +// behind the author's back. +// +// Name, Avatar and Widgets are written for symmetry with the built-in +// archives, and are inert on the path a bundle takes — see the file comment. +// The sidebar the user gets comes from widgets.go. +func writeProfile(outDir string, idx *anyblockjson.Index, names map[string]string) error { + profile := &pb.Profile{ + Name: idx.Name, + } + + // spaceDashboardId is the space's homepage: an object id, or a reserved + // screen translated out of the format's `_` namespace into the bare name + // setWorkspaceSettings switches on (WireHomepage). An omitted homepage + // follows the entrypoint rather than defaulting to the widgets screen (§2c). + profile.SpaceDashboardId = anyblockjson.WireHomepage(idx.SpaceHomepage()) + + // the icon is referenced by id in the format and by name on the wire + if id := idx.IconImageId(); id != "" { + name, ok := names[id] + if !ok { + return fmt.Errorf("icon %q names no object in the bundle", id) + } + if name == "" { + return fmt.Errorf("icon %q has no name, and the installer resolves the space icon by name", id) + } + profile.Avatar = name + } + + // inert on the experience path: CreateObjectsForExperience never calls + // getWidgets, so nothing reads these. Kept so an archive this tool produces + // is also a valid built-in archive, where inject() does read them. + for i, w := range idx.Widgets { + layout, ok := widgetLayouts[w.Layout] + if w.Layout != "" && !ok { + return fmt.Errorf("widgets[%d]: unknown layout %q", i, w.Layout) + } + profile.Widgets = append(profile.Widgets, &pb.WidgetBlock{ + Layout: layout, + TargetObjectId: anyblockjson.WireWidgetTarget(w.Target), + ObjectLimit: w.Limit, + }) + } + + data, err := profile.Marshal() + if err != nil { + return fmt.Errorf("marshal profile: %w", err) + } + return os.WriteFile(filepath.Join(outDir, constant.ProfileFile), data, 0o644) +} diff --git a/cmd/anyblockconvert/profile_test.go b/cmd/anyblockconvert/profile_test.go new file mode 100644 index 0000000000..5c0009e860 --- /dev/null +++ b/cmd/anyblockconvert/profile_test.go @@ -0,0 +1,136 @@ +package main + +// The profile file is the only output describing the bundle rather than an +// object, and the only way an installed space gets a name, an entry point and +// a sidebar. builtinobjects reads it with pb.Profile.Unmarshal, so it must +// decode as one. + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/constant" +) + +func readBack(t *testing.T, dir string) *pb.Profile { + t.Helper() + data, err := os.ReadFile(filepath.Join(dir, constant.ProfileFile)) + require.NoError(t, err) + p := &pb.Profile{} + require.NoError(t, p.Unmarshal(data), "the installer reads this with pb.Profile.Unmarshal") + return p +} + +func TestWriteProfile(t *testing.T) { + dir := t.TempDir() + idx := &anyblockjson.Index{ + Name: "Company Wiki", + Entrypoint: "page-home", + Widgets: []anyblockjson.Widget{ + {Target: "page-home", Layout: "tree"}, + {Target: "type-page", Layout: "view", Limit: 6}, + {Target: "_favorite", Layout: "compact_list"}, + {Target: "chat-requests"}, // no layout: link, the zero value + }, + } + require.NoError(t, writeProfile(dir, idx, nil)) + + p := readBack(t, dir) + assert.Equal(t, "Company Wiki", p.Name) + // an omitted homepage follows the entrypoint, never the widgets screen + assert.Equal(t, "page-home", p.SpaceDashboardId) + + require.Len(t, p.Widgets, 4) + assert.Equal(t, model.BlockContentWidget_Tree, p.Widgets[0].Layout) + assert.Equal(t, model.BlockContentWidget_View, p.Widgets[1].Layout) + assert.Equal(t, int32(6), p.Widgets[1].ObjectLimit) + assert.Equal(t, model.BlockContentWidget_CompactList, p.Widgets[2].Layout) + assert.Equal(t, model.BlockContentWidget_Link, p.Widgets[3].Layout, "absent layout is link") + + // reserved targets pass through untouched; the installer knows them + assert.Equal(t, "favorite", p.Widgets[2].TargetObjectId, + "the platform prefix is the format's; the wire carries the bare listing name") + // TEMPORARY: inject opens widgets[0], which is why validate warns when + // the declared entrypoint is not first + assert.Equal(t, idx.EntryPoint(), p.Widgets[0].TargetObjectId) +} + +func TestWriteProfile_Homepage(t *testing.T) { + t.Run("explicit homepage wins over the entrypoint", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, writeProfile(dir, &anyblockjson.Index{ + Entrypoint: "page-welcome", Homepage: "page-dashboard", + }, nil)) + assert.Equal(t, "page-dashboard", readBack(t, dir).SpaceDashboardId) + }) + t.Run("a reserved homepage passes through", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, writeProfile(dir, &anyblockjson.Index{ + Entrypoint: "page-home", Homepage: "graph", + }, nil)) + assert.Equal(t, "graph", readBack(t, dir).SpaceDashboardId) + }) +} + +// an image icon is an object id in the format and the image's *name* on the wire +func TestWriteProfile_IconImage(t *testing.T) { + fileIcon := func(id string) *anyblockjson.Icon { + return &anyblockjson.Icon{Format: "file", File: id} + } + dir := t.TempDir() + names := map[string]string{"file-logo": "acme-logo"} + require.NoError(t, writeProfile(dir, &anyblockjson.Index{ + Name: "X", Icon: fileIcon("file-logo"), + }, names)) + assert.Equal(t, "acme-logo", readBack(t, dir).Avatar) + + t.Run("an unknown id fails rather than shipping a blank icon", func(t *testing.T) { + err := writeProfile(t.TempDir(), &anyblockjson.Index{Icon: fileIcon("file-missing")}, names) + require.Error(t, err) + assert.Contains(t, err.Error(), "names no object") + }) + t.Run("a nameless object fails: the installer resolves by name", func(t *testing.T) { + err := writeProfile(t.TempDir(), &anyblockjson.Index{Icon: fileIcon("file-x")}, + map[string]string{"file-x": ""}) + require.Error(t, err) + assert.Contains(t, err.Error(), "has no name") + }) + t.Run("an emoji icon leaves the wire avatar empty", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, writeProfile(dir, &anyblockjson.Index{ + Icon: &anyblockjson.Icon{Format: "emoji", Emoji: "📚"}, + }, names)) + assert.Empty(t, readBack(t, dir).Avatar) + }) +} + +func TestWriteProfile_UnknownLayout(t *testing.T) { + err := writeProfile(t.TempDir(), &anyblockjson.Index{ + Widgets: []anyblockjson.Widget{{Target: "a", Layout: "grid"}}, + }, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown layout") +} + +// builtinobjects.setWorkspaceSettings switches on the bare `widgets`/`graph` +// BEFORE it tries to resolve an id, so an untranslated `_graph` is looked up +// as an object, is not found, and silently falls back to the widgets screen. +func TestWriteProfile_ReservedHomepageIsTranslated(t *testing.T) { + for _, tc := range []struct{ homepage, want string }{ + {anyblockjson.HomepageGraph, domain.HomepageGraph}, + {anyblockjson.HomepageWidgets, domain.HomepageWidgets}, + {"page-home", "page-home"}, + } { + dir := t.TempDir() + require.NoError(t, writeProfile(dir, &anyblockjson.Index{Homepage: tc.homepage}, nil)) + assert.Equal(t, tc.want, readBack(t, dir).SpaceDashboardId, tc.homepage) + } +} diff --git a/cmd/anyblockconvert/widgets.go b/cmd/anyblockconvert/widgets.go new file mode 100644 index 0000000000..c98a00dc06 --- /dev/null +++ b/cmd/anyblockconvert/widgets.go @@ -0,0 +1,41 @@ +package main + +// widgets.go writes the archive's Widget snapshot from index.json (§2c) — the +// sidebar, as the path a bundle actually takes carries it. +// +// A bundle is installed with ObjectImportExperience, which reaches +// builtinobjects.CreateObjectsForExperience. That function reads the `profile` +// file for exactly one field, SpaceDashboardId (via setWorkspaceSettings); it +// never calls getWidgets or createWidgets, which belong to inject(), the +// built-in-archive path. So on this path profile.Widgets is inert and the +// sidebar has to arrive the way a real app export carries it: as a snapshot +// with sbType Widget, which core/block/import/pb.shouldImportSnapshot admits +// precisely when the import type is EXPERIENCE. +// +// The snapshot itself is anyblockjson.WidgetsSnapshot — one root block plus, +// per widget, a wrapper carrying the widget content and a link child carrying +// the target, the shape widget.createBlock builds in a live space and +// objectcreator.addWidgetBlock reads back on import. It lives in the package +// rather than here because the round-trip verifier holds the SAME function's +// output against the widget object it omits: one builder, so the tool that +// installs a sidebar and the check that promises nothing was lost cannot +// drift apart. + +import ( + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// writeWidgets renders index.json's sidebar state as the archive's Widget +// snapshot. A bundle declaring none gets no snapshot rather than an empty +// one. +func writeWidgets(outDir string, idx *anyblockjson.Index, format outputFormat) error { + snap, err := anyblockjson.WidgetsSnapshot(idx) + if err != nil { + return err + } + if snap == nil { + return nil + } + return writeSnapshot(outDir, anyblockjson.WidgetsObjectId, model.SmartBlockType_Widget, snap, format) +} diff --git a/cmd/anyblockconvert/widgets_test.go b/cmd/anyblockconvert/widgets_test.go new file mode 100644 index 0000000000..42961a1678 --- /dev/null +++ b/cmd/anyblockconvert/widgets_test.go @@ -0,0 +1,190 @@ +package main + +// The Widget snapshot is how a sidebar reaches a space installed as an +// experience: CreateObjectsForExperience reads only spaceDashboardId off the +// profile, so profile.widgets never becomes a widget. Everything asserted here +// is a shape the importer requires and fails silently without — a dropped +// widget produces no error anywhere. The builder itself is +// anyblockjson.WidgetsSnapshot (shared with the round-trip verifier's +// reconstruction check); what this file pins is the tool's half: the archive +// placement and the importer-facing contract. + +import ( + "os" + "path/filepath" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/block/editor/widget" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func sampleIndex() *anyblockjson.Index { + return &anyblockjson.Index{ + Name: "OKRs & Goals", + Entrypoint: "page-okr-hub", + Widgets: []anyblockjson.Widget{ + {Target: "page-okr-hub", Layout: "tree"}, + {Target: "type-objective", Layout: "view", Limit: 6, ViewId: "view-board"}, + {Target: "_favorite", Layout: "compact_list"}, + {Target: "chat-goal-proposals", CardStyle: "card", IconSize: "medium", + Description: "content", Properties: []string{"name"}}, + }, + } +} + +// blockIndex maps a snapshot's blocks by id, and returns the root — the block +// carrying smartblock content, which is the one anymark.AddRootBlock renames +// to the derived widgets id. +func blockIndex(t *testing.T, snap *model.SmartBlockSnapshotBase) (map[string]*model.Block, *model.Block) { + t.Helper() + byId := make(map[string]*model.Block, len(snap.Blocks)) + var root *model.Block + for _, b := range snap.Blocks { + require.NotContains(t, byId, b.Id, "block ids must be unique within the snapshot") + byId[b.Id] = b + if b.GetSmartblock() != nil { + require.Nil(t, root, "exactly one block may carry smartblock content") + root = b + } + } + require.NotNil(t, root, "without a smartblock block AddRootBlock appends a second root and orphans every wrapper") + return byId, root +} + +func TestWidgetsSnapshot_Shape(t *testing.T) { + idx := sampleIndex() + snap, err := anyblockjson.WidgetsSnapshot(idx) + require.NoError(t, err) + require.NotNil(t, snap) + + byId, root := blockIndex(t, snap) + assert.Equal(t, anyblockjson.WidgetsObjectId, root.Id, "the root is the snapshot's own id until the importer renames it") + require.Len(t, snap.Blocks, 1+2*len(idx.Widgets), "root, plus a wrapper and a link per widget") + + // root children order is sidebar order: updateWidgetObject walks + // state.Blocks(), a BFS from the root, and addWidgetBlock appends each + // widget to the live object in that order + require.Len(t, root.ChildrenIds, len(idx.Widgets)) + + wantLayouts := []model.BlockContentWidgetLayout{ + model.BlockContentWidget_Tree, + model.BlockContentWidget_View, + model.BlockContentWidget_CompactList, + model.BlockContentWidget_Link, + } + for i, w := range idx.Widgets { + wrapper := byId[root.ChildrenIds[i]] + require.NotNil(t, wrapper, "every root child must exist: an unreachable block is dropped by the BFS") + + wc := wrapper.GetWidget() + require.NotNil(t, wc, "a root child of the widget object is a wrapper") + assert.Equal(t, wantLayouts[i], wc.Layout, "widgets[%d] layout", i) + assert.Equal(t, w.Limit, wc.Limit, "widgets[%d] limit", i) + assert.Equal(t, w.ViewId, wc.ViewId, "widgets[%d] view_id", i) + assert.Equal(t, w.AutoAdded, wc.AutoAdded, "widgets[%d] auto_added", i) + + // addWidgetBlock reads ChildrenIds[0] and ignores the rest + require.Len(t, wrapper.ChildrenIds, 1, "a wrapper carries exactly one link") + link := byId[wrapper.ChildrenIds[0]] + require.NotNil(t, link) + require.NotNil(t, link.GetLink()) + assert.Equal(t, anyblockjson.WireWidgetTarget(w.Target), link.GetLink().TargetBlockId, + "widgets[%d] target", i) + assert.Empty(t, link.ChildrenIds) + + assert.Equal(t, link.Id+"-wrapper", wrapper.Id, + "the wrapper id convention core/block/editor/widget uses for stable wrappers") + } + + // the link child's display members ride on the last widget + last := byId[byId[root.ChildrenIds[3]].ChildrenIds[0]].GetLink() + assert.Equal(t, model.BlockContentLink_Card, last.CardStyle) + assert.Equal(t, model.BlockContentLink_SizeMedium, last.IconSize) + assert.Equal(t, model.BlockContentLink_Content, last.Description) + assert.Equal(t, []string{"name"}, last.Relations) + + // the object itself: a hidden dashboard, as an app export writes it + assert.Equal(t, anyblockjson.WidgetsObjectId, snap.Details.GetFields()[detailID].GetStringValue()) + assert.Equal(t, float64(model.ObjectType_dashboard), snap.Details.GetFields()[detailLayout].GetNumberValue()) + assert.True(t, snap.Details.GetFields()[detailIsHidden].GetBoolValue()) + assert.Equal(t, []string{"ot-dashboard"}, snap.ObjectTypes) +} + +// The snapshot has to land where core/block/import/pb finds it, as a +// pb.SnapshotWithType carrying sbType Widget — that type is what +// shouldImportSnapshot admits on an EXPERIENCE import. +func TestWriteWidgets_OnDisk(t *testing.T) { + dir := t.TempDir() + require.NoError(t, writeWidgets(dir, sampleIndex(), formatPb)) + + data, err := os.ReadFile(filepath.Join(dir, "objects", anyblockjson.WidgetsObjectId+".pb")) + require.NoError(t, err) + sw := &pb.SnapshotWithType{} + require.NoError(t, proto.Unmarshal(data, sw)) + assert.Equal(t, model.SmartBlockType_Widget, sw.SbType) + require.NotNil(t, sw.Snapshot.Data) + assert.Len(t, sw.Snapshot.Data.Blocks, 1+2*len(sampleIndex().Widgets)) + + t.Run("a bundle with no sidebar state writes nothing", func(t *testing.T) { + empty := t.TempDir() + require.NoError(t, writeWidgets(empty, &anyblockjson.Index{Name: "X"}, formatPb)) + _, err := os.Stat(filepath.Join(empty, "objects")) + assert.True(t, os.IsNotExist(err)) + }) + + // the auto-widget ledger is sidebar state: a bundle carrying only it + // still writes the snapshot, details and all, so the state is in the + // archive the day the importer starts reading it + t.Run("the auto-widget ledger alone still writes the snapshot", func(t *testing.T) { + ledger := t.TempDir() + require.NoError(t, writeWidgets(ledger, + &anyblockjson.Index{Name: "X", AutoWidgetTargets: []string{"_bin"}}, formatPb)) + data, err := os.ReadFile(filepath.Join(ledger, "objects", anyblockjson.WidgetsObjectId+".pb")) + require.NoError(t, err) + sw := &pb.SnapshotWithType{} + require.NoError(t, proto.Unmarshal(data, sw)) + auto := sw.Snapshot.Data.Details.GetFields()["autoWidgetTargets"] + require.NotNil(t, auto) + require.Len(t, auto.GetListValue().GetValues(), 1) + assert.Equal(t, "bin", auto.GetListValue().GetValues()[0].GetStringValue(), + "ledger entries are written in the importer's own spelling, like link targets") + }) +} + +// The listings the importer knows are bare words +// (widget.IsPredefinedWidgetTargetId); the format spells them in the platform +// namespace so a bundle object cannot shadow them. This is the boundary where +// the prefix has to come back off, and getting it wrong is worse than the bug +// it replaces: handleLinkBlock rewrites a target it does not recognise to +// addr.MissingObject, and WidgetObject.Init then strips the link AND its +// wrapper, so the widget disappears with nothing logged as an error. +func TestWidgetsSnapshot_ReservedTargetsAreWrittenInTheImporterSpelling(t *testing.T) { + idx := &anyblockjson.Index{} + for _, target := range anyblockjson.ReservedWidgetTargets() { + idx.Widgets = append(idx.Widgets, anyblockjson.Widget{Target: target}) + } + idx.Widgets = append(idx.Widgets, anyblockjson.Widget{Target: "page-home"}) + snap, err := anyblockjson.WidgetsSnapshot(idx) + require.NoError(t, err) + + var targets []string + for _, b := range snap.Blocks { + if l := b.GetLink(); l != nil { + targets = append(targets, l.TargetBlockId) + } + } + require.Len(t, targets, len(idx.Widgets)) + for i, target := range targets[:len(targets)-1] { + assert.True(t, widget.IsPredefinedWidgetTargetId(target), + "handleLinkBlock leaves a target alone only for these: %q (from %q)", target, idx.Widgets[i].Target) + } + assert.Equal(t, "page-home", targets[len(targets)-1], "an object id passes through") + assert.False(t, widget.IsPredefinedWidgetTargetId("_favorite"), + "the untranslated spelling is exactly what the importer does NOT know") +} diff --git a/cmd/anyblockconvert/write.go b/cmd/anyblockconvert/write.go new file mode 100644 index 0000000000..cd0ff688de --- /dev/null +++ b/cmd/anyblockconvert/write.go @@ -0,0 +1,81 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/gogo/protobuf/jsonpb" + "github.com/gogo/protobuf/proto" + + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// outputFormat selects how writeSnapshot serializes a snapshot to disk. +type outputFormat int + +const ( + formatPb outputFormat = iota + formatJSON +) + +var jsonMarshaler = jsonpb.Marshaler{Indent: " "} + +// subDirFor mirrors the folder layout of the bundled use-case archives +// (util/builtinobjects/data/*.zip: objects/types/relations/relationsOptions/ +// templates/profile/files). The pb importer (core/block/import/pb) walks the +// whole tree recursively and doesn't care about folder names — this is +// purely so a human skimming the output can find things. +func subDirFor(sbType model.SmartBlockType) string { + switch sbType { + case model.SmartBlockType_STType: + return "types" + case model.SmartBlockType_STRelation: + return "relations" + case model.SmartBlockType_STRelationOption: + return "relationsOptions" + case model.SmartBlockType_Template, model.SmartBlockType_BundledTemplate: + return "templates" + default: + return "objects" + } +} + +// writeSnapshot serializes a snapshot as pb.SnapshotWithType. With formatJSON +// it writes jsonpb text under a ".json" extension instead of raw proto bytes +// under ".pb" — core/block/import/pb accepts either extension on import, so +// this is both human-inspectable and directly importable. +func writeSnapshot(outDir, id string, sbType model.SmartBlockType, snap *model.SmartBlockSnapshotBase, format outputFormat) error { + sw := &pb.SnapshotWithType{ + SbType: sbType, + Snapshot: &pb.ChangeSnapshot{Data: snap}, + } + + ext := ".pb" + var data []byte + if format == formatJSON { + ext = ".json" + s, err := jsonMarshaler.MarshalToString(sw) + if err != nil { + return fmt.Errorf("marshal json: %w", err) + } + data = []byte(s) + } else { + var err error + data, err = proto.Marshal(sw) + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + } + + dir := filepath.Join(outDir, subDirFor(sbType)) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("mkdir %s: %w", dir, err) + } + path := filepath.Join(dir, sanitizeId(id)+ext) + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + return nil +} diff --git a/cmd/anyblockinstall/anyblockinstall.sh b/cmd/anyblockinstall/anyblockinstall.sh new file mode 100755 index 0000000000..8874d3c0d3 --- /dev/null +++ b/cmd/anyblockinstall/anyblockinstall.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# +# anyblockinstall — build an AnyBlock bundle into an importable archive and +# install it into a brand-new space, so a bundle can be looked at rather than +# only validated. +# +# anyblockinstall.sh --token --port

--bundle

[--name "Space"] +# +# Does three things: +# 1. anyblockconvert bundle -> pb snapshots, the `profile` file and the +# Widget snapshot that carries the sidebar (SPEC §2c) +# 2. zip archive laid out the way builtinobjects expects +# 3. grpcurl WorkspaceCreate, then ObjectImportExperience +# +# The middleware has no gRPC reflection, so the .proto files are passed +# explicitly; they are read from the repo this script lives in. + +set -euo pipefail + +die() { printf '\033[31merror:\033[0m %s\n' "$*" >&2; exit 1; } +step() { printf '\033[1m==>\033[0m %s\n' "$*"; } +note() { printf ' %s\n' "$*"; } + +TOKEN=""; PORT=""; BUNDLE=""; SPACE_NAME=""; KEEP=0; FORMAT="pb"; DRY=0 + +usage() { + sed -n '2,17p' "$0" | sed 's/^# \{0,1\}//' + cat <<'EOF' + +Options: + --token session token (gRPC `token` metadata header) + --port middleware gRPC port on 127.0.0.1 + --bundle AnyBlock bundle directory (the one holding index.json) + --name space name; defaults to index.json's `name` + --format pb|json snapshot format inside the archive (default: pb) + --keep keep the build directory and print its path + --dry-run build the archive and print the two calls, without making + them; implies --keep, and needs neither grpcurl nor a + running middleware +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --token) TOKEN="${2:-}"; shift 2 ;; + --port) PORT="${2:-}"; shift 2 ;; + --bundle) BUNDLE="${2:-}"; shift 2 ;; + --name) SPACE_NAME="${2:-}"; shift 2 ;; + --format) FORMAT="${2:-}"; shift 2 ;; + --keep) KEEP=1; shift ;; + --dry-run) DRY=1; KEEP=1; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown option: $1 (try --help)" ;; + esac +done + +[[ -n "$BUNDLE" ]] || die "--bundle is required" +[[ -d "$BUNDLE" ]] || die "no such bundle directory: $BUNDLE" +if [[ "$DRY" == 0 ]]; then + [[ -n "$TOKEN" ]] || die "--token is required" + [[ -n "$PORT" ]] || die "--port is required" + command -v grpcurl >/dev/null || die "grpcurl not found (brew install grpcurl)" +fi +command -v jq >/dev/null || die "jq not found (brew install jq)" +command -v zip >/dev/null || die "zip not found" + +# the repo root: this script lives in /cmd/anyblockinstall/ +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PROTO="pb/protos/service/service.proto" +[[ -f "$REPO/$PROTO" ]] || die "cannot find $PROTO under $REPO" + +BUNDLE="$(cd "$BUNDLE" && pwd)" # ObjectImportExperience needs an absolute path +BUILD="$(mktemp -d "${TMPDIR:-/tmp}/anyblockinstall.XXXXXX")" +cleanup() { [[ "$KEEP" == 1 ]] || rm -rf "$BUILD"; } +trap cleanup EXIT + +# ---------------------------------------------------------------- convert --- +step "Converting $(basename "$BUNDLE")" +( cd "$REPO" && go run ./cmd/anyblockconvert -in "$BUNDLE" -out "$BUILD/archive" -format "$FORMAT" ) \ + || die "conversion failed — fix the bundle before installing it" + +# both outputs come from index.json; without it the space gets no homepage and +# no sidebar (SPEC §2c). The widget snapshot is the sidebar on this path — +# ObjectImportExperience never reads profile.widgets. +[[ -f "$BUILD/archive/profile" ]] || die "no profile file was produced: the bundle needs an index.json (SPEC §2c), or the space gets no homepage and no sidebar" +[[ -f "$BUILD/archive/objects/widgets.$FORMAT" ]] || note "no widget snapshot: index.json declares no widgets, so the sidebar will be empty" + +# --------------------------------------------------------------------- zip --- +# Entries sit at the archive root (objects/, types/, ..., profile), matching +# util/builtinobjects/data/*.zip. -X drops the extra Finder attributes, and +# .DS_Store / __MACOSX would otherwise ride along as bogus objects. +step "Packing archive" +ARCHIVE="$BUILD/$(basename "$BUNDLE").zip" +find "$BUILD/archive" -name '.DS_Store' -delete +( cd "$BUILD/archive" && zip -r -X -q "$ARCHIVE" . -x '__MACOSX/*' ) +note "$ARCHIVE ($(du -h "$ARCHIVE" | cut -f1 | tr -d ' '), $(unzip -Z1 "$ARCHIVE" | wc -l | tr -d ' ') entries)" + +# space name: --name wins, else index.json's, else the directory name +if [[ -z "$SPACE_NAME" ]]; then + SPACE_NAME="$(jq -r '.name // empty' "$BUNDLE/index.json" 2>/dev/null || true)" + [[ -n "$SPACE_NAME" ]] || SPACE_NAME="$(basename "$BUNDLE")" +fi + +rpc() { # rpc + grpcurl -plaintext -import-path "$REPO" -proto "$PROTO" \ + -H "token: $TOKEN" -d "$2" \ + "127.0.0.1:$PORT" "anytype.ClientCommands/$1" +} + +# every response carries error.code; NULL is success +check() { # check + local code + code="$(jq -r '.error.code // "NULL"' <<<"$1")" + if [[ "$code" != "NULL" && "$code" != "null" && "$code" != "0" ]]; then + printf '%s\n' "$1" >&2 + die "$2 failed: $code — $(jq -r '.error.description // ""' <<<"$1")" + fi +} + +CREATE_REQ="$(jq -nc --arg n "$SPACE_NAME" '{details:{name:$n}, useCase:"NONE"}')" +IMPORT_PREVIEW="$(jq -nc --arg s '' --arg u "$ARCHIVE" --arg t "$SPACE_NAME" \ + '{spaceId:$s, url:$u, title:$t, isNewSpace:true, isAi:false}')" +if [[ "$DRY" == 1 ]]; then + step "Dry run — would call" + note "1. anytype.ClientCommands/WorkspaceCreate" + note " $CREATE_REQ" + note "2. anytype.ClientCommands/ObjectImportExperience" + note " $IMPORT_PREVIEW" + note "" + note "archive kept at $ARCHIVE" + exit 0 +fi + +step "Creating space \"$SPACE_NAME\"" +CREATE_RES="$(rpc WorkspaceCreate "$CREATE_REQ")" || die "WorkspaceCreate: is the middleware listening on 127.0.0.1:$PORT?" +check "$CREATE_RES" "WorkspaceCreate" + +SPACE_ID="$(jq -r '.spaceId // empty' <<<"$CREATE_RES")" +[[ -n "$SPACE_ID" ]] || { printf '%s\n' "$CREATE_RES" >&2; die "WorkspaceCreate returned no spaceId"; } +note "spaceId $SPACE_ID" + +# ------------------------------------------------------------------ import --- +step "Importing" +IMPORT_REQ="$(jq -nc --arg s "$SPACE_ID" --arg u "$ARCHIVE" --arg t "$SPACE_NAME" \ + '{spaceId:$s, url:$u, title:$t, isNewSpace:true, isAi:false}')" +IMPORT_RES="$(rpc ObjectImportExperience "$IMPORT_REQ")" || die "ObjectImportExperience call failed" +check "$IMPORT_RES" "ObjectImportExperience" + +step "Installed" +note "space $SPACE_NAME" +note "spaceId $SPACE_ID" +if ENTRY="$(jq -r '.entrypoint // (.widgets[0].target) // empty' "$BUNDLE/index.json" 2>/dev/null)" && [[ -n "$ENTRY" ]]; then + note "should open on: $ENTRY" +fi +if [[ "$KEEP" == 1 ]]; then note "build kept at $BUILD"; fi +exit 0 diff --git a/cmd/anyblockrecover/main.go b/cmd/anyblockrecover/main.go new file mode 100644 index 0000000000..38c8893f6e --- /dev/null +++ b/cmd/anyblockrecover/main.go @@ -0,0 +1,209 @@ +// anyblockrecover reconstructs AnyBlock JSON source documents from a +// directory of pb snapshots produced by cmd/anyblockconvert. It is the +// inverse of that tool: anyblockjson.Marshal turns each snapshot back into a +// document, using the batch's own relations/ and relationsOptions/ snapshots +// to resolve property formats, property names and option names — the three +// things the source expressed by name and the snapshots hold as ids. +// +// Recovered documents are not byte-identical to hand-authored ones: key order +// is canonical, and the synthesized relation/option snapshots are skipped +// (they are generated on every convert, not source). +// +// Usage: +// +// go run ./cmd/anyblockrecover -in ./out -out ~/usecase2/anyblock/01-company-wiki +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/gogo/protobuf/jsonpb" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +type propInfo struct { + name string + format model.RelationFormat +} + +type resolver struct { + props map[string]propInfo // relation key -> name/format + byId map[string]string // relation object id -> key + optName map[string]string // option object id -> name +} + +// format resolves a synthesized property; bundled keys are left to +// anyblockjson, which consults the bundle before any resolver. +func (r *resolver) format(key domain.RelationKey) (model.RelationFormat, bool) { + p, ok := r.props[string(key)] + return p.format, ok +} + +// lookup returns what is known about a property key, preferring the bundle — +// only synthesized relations appear in the batch's relations/ folder, so a +// bundled key like iconImage has no snapshot to read its format from. +func (r *resolver) lookup(key string) (propInfo, bool) { + if rel, err := bundle.GetRelation(domain.RelationKey(key)); err == nil && rel != nil { + return propInfo{name: rel.Name, format: rel.Format}, true + } + p, ok := r.props[key] + return p, ok +} + +func (r *resolver) OptionName(_ domain.RelationKey, id string) (string, bool) { + n, ok := r.optName[id] + return n, ok +} + +func (r *resolver) OptionId(_ domain.RelationKey, name string) (string, bool) { + return name, false +} + +func (r *resolver) PropertyById(id string) (anyblockjson.PropertyDefinition, bool) { + key, ok := r.byId[id] + if !ok { + // bundled properties arrive as _br + if strings.HasPrefix(id, "_br") { + key = strings.TrimPrefix(id, "_br") + } else { + return anyblockjson.PropertyDefinition{}, false + } + } + p, _ := r.lookup(key) + return anyblockjson.PropertyDefinition{ + Key: domain.RelationKey(key), + Name: p.name, + Format: p.format, + }, true +} + +func (r *resolver) PropertyId(def anyblockjson.PropertyDefinition) (string, bool) { + return string(def.Key), true +} + +func readSnapshot(path string) (*pb.SnapshotWithType, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + sw := &pb.SnapshotWithType{} + if err := jsonpb.Unmarshal(f, sw); err != nil { + return nil, fmt.Errorf("%s: %w", filepath.Base(path), err) + } + return sw, nil +} + +func detailString(d *pb.ChangeSnapshot, key string) string { + if d == nil || d.Data == nil || d.Data.Details == nil { + return "" + } + return d.Data.Details.Fields[key].GetStringValue() +} + +func main() { + inDir := flag.String("in", "", "directory of pb snapshots (cmd/anyblockconvert output)") + outDir := flag.String("out", "", "directory to write recovered AnyBlock JSON documents into") + flag.Parse() + if *inDir == "" || *outDir == "" { + flag.Usage() + os.Exit(2) + } + + res := &resolver{ + props: map[string]propInfo{}, + byId: map[string]string{}, + optName: map[string]string{}, + } + + // pass 1: build the resolvers from the synthesized relation/option snapshots + for _, sub := range []string{"relations", "relationsOptions"} { + paths, _ := filepath.Glob(filepath.Join(*inDir, sub, "*.json")) + for _, p := range paths { + sw, err := readSnapshot(p) + if err != nil { + fmt.Fprintln(os.Stderr, "skip:", err) + continue + } + id := detailString(sw.Snapshot, "id") + key := detailString(sw.Snapshot, "relationKey") + name := detailString(sw.Snapshot, "name") + if sub == "relations" { + f := sw.Snapshot.Data.Details.Fields["relationFormat"].GetNumberValue() + res.props[key] = propInfo{name: name, format: model.RelationFormat(int32(f))} + res.byId[id] = key + } else { + res.optName[id] = name + } + } + } + + opts := anyblockjson.Options{ + ResolveFormat: res.format, + ResolveOptions: res, + ResolveProperties: res, + } + + // pass 2: recover every type/object document + var recovered, failed int + for _, sub := range []string{"types", "objects", "templates"} { + paths, _ := filepath.Glob(filepath.Join(*inDir, sub, "*.json")) + for _, p := range paths { + sw, err := readSnapshot(p) + if err != nil { + fmt.Fprintln(os.Stderr, "FAIL:", err) + failed++ + continue + } + data, err := anyblockjson.Marshal(sw.SbType, sw.Snapshot.Data, opts) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s: %v\n", filepath.Base(p), err) + failed++ + continue + } + dir, name := routeFor(sw, filepath.Base(p)) + target := filepath.Join(*outDir, dir) + if err := os.MkdirAll(target, 0o755); err != nil { + fmt.Fprintln(os.Stderr, "FAIL:", err) + failed++ + continue + } + if err := os.WriteFile(filepath.Join(target, name), data, 0o644); err != nil { + fmt.Fprintln(os.Stderr, "FAIL:", err) + failed++ + continue + } + recovered++ + } + } + fmt.Printf("%d documents recovered, %d failed\noutput: %s\n", recovered, failed, *outDir) +} + +// routeFor mirrors the source layout the bundles use: types/, chats/, pages/, +// objects/. The snapshot only knows its smartblock type, so page-vs-object is +// taken from the id slug the author chose. +func routeFor(sw *pb.SnapshotWithType, base string) (dir, name string) { + id := detailString(sw.Snapshot, "id") + switch sw.SbType { + case model.SmartBlockType_STType: + return "types", strings.TrimPrefix(strings.TrimSuffix(base, ".json"), "type-") + ".type.json" + case model.SmartBlockType_ChatDerivedObject, model.SmartBlockType_DiscussionObject: + return "chats", strings.TrimPrefix(base, "chat-") + } + if strings.HasPrefix(id, "page-") { + return "pages", strings.TrimPrefix(base, "page-") + } + return "objects", base +} + +var _ = json.Marshal diff --git a/cmd/anyblockroundtrip/main.go b/cmd/anyblockroundtrip/main.go new file mode 100644 index 0000000000..31d3a15490 --- /dev/null +++ b/cmd/anyblockroundtrip/main.go @@ -0,0 +1,828 @@ +// anyblockroundtrip verifies the AnyBlock JSON round-trip (pkg/lib/anyblockjson) +// against a real account: it recovers the account from a mnemonic, exports every +// object of every space to pb snapshots, converts each snapshot pb → AnyBlock +// JSON → pb, and checks the §11 contract (Export ∘ Import byte-stable, no +// unexpected data loss). Every inconsistency leaves an artifact directory with +// the original snapshot, both JSON generations, and a report for triage. +// +// Usage: +// +// go run ./cmd/anyblockroundtrip -root-path ~/anyblockroundtrip-repo -out ./roundtrip-out +// +// The mnemonic is read from $ANYTYPE_MNEMONIC or the -mnemonic flag. The root +// path is the account repo directory: point it at a copy of an existing data +// dir to skip network sync, or at an empty dir to sync from the network (slow +// on large accounts; stop the desktop app first if you reuse its data dir — +// two processes cannot share one repo). +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/anyproto/any-sync/app" + "github.com/gogo/protobuf/jsonpb" + "github.com/gogo/protobuf/proto" + + "github.com/anyproto/anytype-heart/core" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/core/event" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/compose" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/snapshotdiff" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/storeresolver" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + libcore "github.com/anyproto/anytype-heart/pkg/lib/core" + "github.com/anyproto/anytype-heart/pkg/lib/database" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/space" +) + +func main() { + var ( + mnemonic = flag.String("mnemonic", os.Getenv("ANYTYPE_MNEMONIC"), "account mnemonic (default $ANYTYPE_MNEMONIC)") + accountId = flag.String("account-id", os.Getenv("ANYTYPE_ACCOUNT_ID"), "account id (default $ANYTYPE_ACCOUNT_ID; derived from the mnemonic when empty)") + rootPath = flag.String("root-path", "", "account repo directory (copy of a data dir, or empty dir to sync)") + outDir = flag.String("out", "roundtrip-out", "output directory for artifacts and summary") + spaceFilter = flag.String("space", "", "comma-separated space ids to check (default: all)") + limit = flag.Int("limit", 0, "max objects per space (0 = all)") + keepExports = flag.Bool("keep-exports", false, "keep the raw pb export directories for passing objects too") + dumpJSON = flag.Bool("dump-json", false, "write each object's AnyBlock JSON beside its .pb, rendered with the SPACE's resolvers (implies -keep-exports)") + refNames = flag.Bool("ref-names", false, "render the READ shape: every object reference carries its informative #name suffix (SPEC.md §9)") + native = flag.Bool("native", false, "drive the NATIVE exporter (core/block/export/anyblock) per space and verify layout, classification, blob binding, determinism and fidelity against a legacy pb export taken in the same process") + nativeFiles = flag.Bool("native-files", false, "native mode: stream file blobs too (may fetch from the file node; off by default)") + ) + flag.Parse() + + if *mnemonic == "" || *rootPath == "" { + flag.Usage() + fmt.Fprintln(os.Stderr, "\nboth -mnemonic (or $ANYTYPE_MNEMONIC) and -root-path are required") + os.Exit(2) + } + if *dumpJSON { + *keepExports = true // the JSON is written beside the .pb, so the .pb must stay + } + if err := run(*mnemonic, *accountId, *rootPath, *outDir, *spaceFilter, *limit, *keepExports, *dumpJSON, *refNames, *native, *nativeFiles); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +// +// ---- account bootstrap ---- +// + +func rpcErr(name string, code int32, description string) error { + if code == 0 { + return nil + } + return fmt.Errorf("%s: code %d: %s", name, code, description) +} + +func run(mnemonic, accountId, rootPath, outDir, spaceFilter string, limit int, keepExports, dumpJSON, refNames, native, nativeFiles bool) error { + ctx := context.Background() + mw := core.New() + mw.SetEventSender(event.NewCallbackSender(func(*pb.Event) {})) + + if resp := mw.InitialSetParameters(ctx, &pb.RpcInitialSetParametersRequest{ + Platform: "cli", + Version: "0.0.0-anyblockroundtrip", + Workdir: rootPath, + DoNotSendLogs: true, + DoNotSaveLogs: true, + DoNotSendTelemetry: true, + }); resp.Error != nil && resp.Error.Code != 0 { + return rpcErr("InitialSetParameters", int32(resp.Error.Code), resp.Error.Description) + } + + if resp := mw.WalletRecover(ctx, &pb.RpcWalletRecoverRequest{ + RootPath: rootPath, + Mnemonic: mnemonic, + }); resp.Error != nil && resp.Error.Code != 0 { + return rpcErr("WalletRecover", int32(resp.Error.Code), resp.Error.Description) + } + + if accountId == "" { + // the same derivation CreateSession uses for its identity check; the + // mnemonic auth path returns an empty account id (GO-1854) + derived, err := libcore.WalletAccountAt(mnemonic, 0) + if err != nil { + return fmt.Errorf("derive account from mnemonic: %w", err) + } + accountId = derived.Identity.GetPublic().Account() + } + fmt.Println("account:", accountId) + + selectResp := mw.AccountSelect(ctx, &pb.RpcAccountSelectRequest{ + Id: accountId, + RootPath: rootPath, + }) + if selectResp.Error != nil && selectResp.Error.Code != 0 { + return rpcErr("AccountSelect", int32(selectResp.Error.Code), selectResp.Error.Description) + } + defer mw.AccountStop(ctx, &pb.RpcAccountStopRequest{}) + + a := mw.GetApp() + store := app.MustComponent[objectstore.ObjectStore](a) + spaceService := app.MustComponent[space.Service](a) + + spaces, err := listSpaces(store, spaceService.TechSpaceId()) + if err != nil { + return fmt.Errorf("list spaces: %w", err) + } + if spaceFilter != "" { + wanted := map[string]bool{} + for _, id := range strings.Split(spaceFilter, ",") { + wanted[strings.TrimSpace(id)] = true + } + var filtered []spaceInfo + for _, s := range spaces { + if wanted[s.id] { + filtered = append(filtered, s) + } + } + spaces = filtered + } + fmt.Printf("spaces to check: %d\n", len(spaces)) + + if native { + return runNative(ctx, mw, store, spaces, outDir, nativeFiles) + } + + summary := &summary{Account: accountId, Categories: map[string]int{}, IndentHistogram: map[int]int{}} + for _, s := range spaces { + fmt.Printf("\n== space %s (%s)\n", s.id, s.name) + ss, err := processSpace(ctx, mw, store, s.id, s.name, outDir, limit, keepExports, dumpJSON, refNames) + if err != nil { + fmt.Printf(" space failed: %v\n", err) + summary.SpaceErrors = append(summary.SpaceErrors, spaceError{SpaceId: s.id, Error: err.Error()}) + continue + } + ss.SpaceName = s.name + summary.Spaces = append(summary.Spaces, *ss) + summary.Total += ss.Total + summary.Passed += ss.Passed + summary.Failed += ss.Failed + for c, n := range ss.Categories { + summary.Categories[c] += n + } + for d, n := range ss.IndentHistogram { + summary.IndentHistogram[d] += n + } + summary.CellsWithChildren += ss.CellsWithChildren + summary.OmittedRelationDocs += ss.OmittedRelationDocs + summary.OmittedBytes += ss.OmittedBytes + summary.DictionaryInstalled += ss.DictionaryInstalled + summary.DictionaryEntries += ss.DictionaryEntries + summary.DictionaryBytes += ss.DictionaryBytes + summary.IndexBytes += ss.IndexBytes + } + + summaryPath := filepath.Join(outDir, "summary.json") + data, err := json.MarshalIndent(summary, "", " ") + if err != nil { + return fmt.Errorf("marshal summary: %w", err) + } + if err := os.WriteFile(summaryPath, data, 0o644); err != nil { + return fmt.Errorf("write summary: %w", err) + } + + fmt.Printf("\n==== total: %d objects, %d passed, %d failed\n", summary.Total, summary.Passed, summary.Failed) + for _, c := range sortedKeys(summary.Categories) { + fmt.Printf(" %-16s %d\n", c, summary.Categories[c]) + } + p50, p95, maxD := indentPercentiles(summary.IndentHistogram) + fmt.Printf(" block depth (max indent per object): p50=%d p95=%d max=%d\n", p50, p95, maxD) + fmt.Printf(" cells with children: %d\n", summary.CellsWithChildren) + if dumpJSON { + fmt.Printf(" §2f composition: omitted %d relation docs (%d bytes); dictionaries %d bytes (%d installed, %d entries); indexes %d bytes\n", + summary.OmittedRelationDocs, summary.OmittedBytes, + summary.DictionaryBytes, summary.DictionaryInstalled, summary.DictionaryEntries, summary.IndexBytes) + } + fmt.Println("summary:", summaryPath) + if summary.Failed > 0 || len(summary.SpaceErrors) > 0 { + fmt.Println("artifacts:", filepath.Join(outDir, "artifacts")) + return fmt.Errorf("%d objects failed round-trip", summary.Failed) + } + return nil +} + +// +// ---- spaces ---- +// + +type spaceInfo struct { + id string + name string +} + +func listSpaces(store objectstore.ObjectStore, techSpaceId string) ([]spaceInfo, error) { + records, err := store.SpaceIndex(techSpaceId).Query(database.Query{ + Filters: []database.FilterRequest{{ + RelationKey: bundle.RelationKeyResolvedLayout, + Condition: model.BlockContentDataviewFilter_Equal, + Value: domain.Int64(int64(model.ObjectType_spaceView)), + }}, + }) + if err != nil { + return nil, fmt.Errorf("query space views: %w", err) + } + var out []spaceInfo + seen := map[string]bool{} + for _, r := range records { + id := r.Details.GetString(bundle.RelationKeyTargetSpaceId) + if id == "" || seen[id] { + continue + } + seen[id] = true + out = append(out, spaceInfo{id: id, name: r.Details.GetString(bundle.RelationKeyName)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].id < out[j].id }) + return out, nil +} + +type spaceSummary struct { + SpaceId string `json:"spaceId"` + SpaceName string `json:"spaceName,omitempty"` + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Categories map[string]int `json:"categories,omitempty"` + // IndentHistogram maps each object's max block indent to how many + // objects have that maximum — verifies the "~6 typical max" depth datum. + IndentHistogram map[int]int `json:"indentHistogram,omitempty"` + // CellsWithChildren counts table cell blocks with real descendants (the + // §6.1 array-form trigger). + CellsWithChildren int `json:"cellsWithChildren"` + // The §2f composition: how many bundled-identical relation documents the + // dump omitted (their bytes, as the removed cost), and what the space's + // dictionary and manifest carry instead. + OmittedRelationDocs int `json:"omittedRelationDocs,omitempty"` + OmittedBytes int `json:"omittedBytes,omitempty"` + DictionaryInstalled int `json:"dictionaryInstalled,omitempty"` + DictionaryEntries int `json:"dictionaryEntries,omitempty"` + ManifestTypes int `json:"manifestTypes,omitempty"` + OptionDocs int `json:"optionDocs,omitempty"` + DictionaryBytes int `json:"dictionaryBytes,omitempty"` + IndexBytes int `json:"indexBytes,omitempty"` + // OrphanUsedKeys are referenced property keys with no definition + // anywhere — no relation object, not bundled — so the dictionary cannot + // state a format for them (§2f names every property it CAN). + OrphanUsedKeys []string `json:"orphanUsedKeys,omitempty"` +} + +type spaceError struct { + SpaceId string `json:"spaceId"` + Error string `json:"error"` +} + +type summary struct { + Account string `json:"account"` + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Categories map[string]int `json:"categories"` + IndentHistogram map[int]int `json:"indentHistogram"` + CellsWithChildren int `json:"cellsWithChildren"` + // the §2f composition, account-wide: what the omission removed and what + // the dictionary and manifest cost instead — the byte-delta headline. + OmittedRelationDocs int `json:"omittedRelationDocs,omitempty"` + OmittedBytes int `json:"omittedBytes,omitempty"` + DictionaryInstalled int `json:"dictionaryInstalled,omitempty"` + DictionaryEntries int `json:"dictionaryEntries,omitempty"` + DictionaryBytes int `json:"dictionaryBytes,omitempty"` + IndexBytes int `json:"indexBytes,omitempty"` + Spaces []spaceSummary `json:"spaces"` + SpaceErrors []spaceError `json:"spaceErrors,omitempty"` +} + +func processSpace(ctx context.Context, mw *core.Middleware, store objectstore.ObjectStore, + spaceId, spaceName, outDir string, limit int, keepExports, dumpJSON, refNames bool) (*spaceSummary, error) { + + exportDir := filepath.Join(outDir, "export", spaceId) + resp := mw.ObjectListExport(ctx, &pb.RpcObjectListExportRequest{ + SpaceId: spaceId, + Path: exportDir, + Format: model.Export_Protobuf, + IncludeArchived: true, + NoProgress: true, + }) + if resp.Error != nil && resp.Error.Code != 0 { + return nil, rpcErr("ObjectListExport", int32(resp.Error.Code), resp.Error.Description) + } + exportPath := resp.Path + if exportPath == "" { + exportPath = exportDir + } + + var files []string + err := filepath.WalkDir(exportPath, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(path, ".pb") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk export dir: %w", err) + } + sort.Strings(files) + if limit > 0 && len(files) > limit { + files = files[:limit] + } + + opts := storeresolver.New(store.SpaceIndex(spaceId)).Options() + // the read shape (§9): both legs of the round trip carry it, so the + // suffix is exercised as a fixpoint rather than only written + opts.RefNames = refNames + + composer := newSpaceComposer(opts, spaceName, exportPath) + ss := &spaceSummary{SpaceId: spaceId, Total: len(files), Categories: map[string]int{}, IndentHistogram: map[int]int{}} + for _, f := range files { + objectId := strings.TrimSuffix(filepath.Base(f), ".pb") + issues, artifacts, err := roundtripFile(f, opts) + if err != nil { + return nil, fmt.Errorf("object %s: %w", objectId, err) + } + // the §2f composition: a bundled-identical relation document is not + // written — its key travels in the dictionary's `installed` list — + // and the trip it takes instead (installed key → the reader's + // bundled table) is verified here, per omitted document, through the + // same comparator as everything else. + omitted := false + if artifacts != nil && artifacts.original != nil { + isOmitted, recon := composer.observeSnapshot(artifacts.original) + omitted = isOmitted + issues = append(issues, recon...) + } + if artifacts != nil { + if dumpJSON && artifacts.json1 != nil { + if omitted { + composer.omittedDocs++ + composer.omittedBytes += len(artifacts.json1) + } else { + // the document as this SPACE renders it: property and type + // labels through the space vocabulary, option values by name, + // participant refs folded to the identity and attribution as + // # (§9). Rendering with default Options + // instead produces a technically valid document in which every + // space-minted key is a bson id and every option value a CID — + // readable structure, unreadable content. + out := strings.TrimSuffix(f, ".pb") + ".anyblock.json" + if err := os.WriteFile(out, artifacts.json1, 0o644); err != nil { + return nil, fmt.Errorf("dump json for %s: %w", objectId, err) + } + if err := composer.observeWritten(artifacts.original, artifacts.json1, out); err != nil { + return nil, fmt.Errorf("observe written %s: %w", objectId, err) + } + } + } + if artifacts.json1 != nil { + ss.IndentHistogram[maxIndentOf(artifacts.json1)]++ + } + if artifacts.original != nil { + ss.CellsWithChildren += countCellsWithChildren(artifacts.original.Snapshot.GetData()) + } + } + if len(issues) == 0 { + ss.Passed++ + continue + } + ss.Failed++ + for _, is := range issues { + ss.Categories[is.category]++ + } + if err := writeArtifacts(filepath.Join(outDir, "artifacts", spaceId, objectId), f, issues, artifacts); err != nil { + return nil, fmt.Errorf("write artifacts for %s: %w", objectId, err) + } + fmt.Printf(" FAIL %s: %s\n", objectId, issueLine(issues)) + } + if dumpJSON { + if err := composer.finish(ss); err != nil { + return nil, fmt.Errorf("finish composition: %w", err) + } + } + fmt.Printf(" %d objects, %d passed, %d failed\n", ss.Total, ss.Passed, ss.Failed) + if dumpJSON { + fmt.Printf(" dictionary: %d installed, %d entries; manifest: %d types, %d options; omitted %d relation docs (%d bytes)\n", + ss.DictionaryInstalled, ss.DictionaryEntries, ss.ManifestTypes, ss.OptionDocs, + ss.OmittedRelationDocs, ss.OmittedBytes) + } + + if !keepExports { + if err := os.RemoveAll(exportDir); err != nil { + return nil, fmt.Errorf("clean export dir: %w", err) + } + } + return ss, nil +} + +// +// ---- round-trip ---- +// + +type issue struct { + category string + detail string +} + +func issueLine(issues []issue) string { + parts := make([]string, 0, len(issues)) + for _, is := range issues { + parts = append(parts, is.category) + } + return strings.Join(parts, ", ") +} + +// artifacts collects everything worth persisting when an object fails. +type artifactSet struct { + original *pb.SnapshotWithType + json1 []byte + json2 []byte + reimported *pb.SnapshotWithType +} + +// stripAttribution removes the two derived attribution members from a rendered +// document so two generations can be compared on what the format is actually +// responsible for. It is deliberately textual and deliberately narrow: it +// drops only a top-level line spelling `creator` or `lastModifiedBy` the way +// the document does — their display names, asked of the bundled vocabulary +// rather than restated, because a restated string is how this strip silently +// stopped stripping when the raw-name re-spell moved the members from +// `"creator"` to `"Created by"` (every attribution-bearing object in the +// account reported not_byte_stable at once). An attribution value appearing +// anywhere else still counts as a difference. +// +// When the stripped member was the LAST in its object, the previous line keeps +// a trailing comma the other generation never had — a blindspot this strip +// carried silently until v0.32 exposed it: the §2a settings lift moved +// `plural_name`/`recommended_layout` out of `properties`, which made the +// attribution line the final property on most type documents, and every one +// of them reported not_byte_stable over a comma. The comma is trimmed only +// when the strip actually removed the member between it and the closing +// brace, so a real difference on the neighbouring lines still counts. +var attributionMemberPrefixes = []string{ + `"` + (anyblockjson.BundledKeyVocabulary{}).PropertySlug("creator") + `":`, + `"` + (anyblockjson.BundledKeyVocabulary{}).PropertySlug("lastModifiedBy") + `":`, + // and the stored keys verbatim: in a space holding a custom name-twin + // the attribution claimant yields its plain name and spells its own key + `"creator":`, + `"lastModifiedBy":`, +} + +func stripAttribution(doc []byte) string { + var out []string + stripped := false + for _, line := range strings.Split(string(doc), "\n") { + t := strings.TrimSpace(line) + attribution := false + for _, prefix := range attributionMemberPrefixes { + if strings.HasPrefix(t, prefix) { + attribution = true + break + } + } + if attribution { + stripped = true + continue + } + if stripped && len(out) > 0 && strings.HasPrefix(t, "}") { + out[len(out)-1] = strings.TrimSuffix(out[len(out)-1], ",") + } + stripped = false + out = append(out, line) + } + // In a space holding a custom name-twin of an attribution key, the + // attribution SPELLING owes a legend entry (the vocabulary cannot + // uniquely invert it there), and that entry starts with the same member + // prefix, so the loop above has already removed it — but gen2, whose + // import dropped the attribution detail, may then have no legend AT ALL, + // while gen1 keeps an emptied wrapper. Collapse a legend the strip + // emptied, so the two generations are compared on what the format is + // responsible for. + collapsed := out[:0] + for i := 0; i < len(out); i++ { + t := strings.TrimSpace(out[i]) + if t == `"property_internal_keys": {` && i+1 < len(out) { + next := strings.TrimSpace(out[i+1]) + if next == "}" || next == "}," { + if next == "}" && len(collapsed) > 0 { + collapsed[len(collapsed)-1] = strings.TrimSuffix(collapsed[len(collapsed)-1], ",") + } + i++ // skip the closer too + continue + } + } + collapsed = append(collapsed, out[i]) + } + return strings.Join(collapsed, "\n") +} + +func roundtripFile(path string, opts anyblockjson.Options) ([]issue, *artifactSet, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("read snapshot: %w", err) + } + var sw pb.SnapshotWithType + if err := proto.Unmarshal(data, &sw); err != nil { + return nil, nil, fmt.Errorf("unmarshal snapshot: %w", err) + } + base := sw.Snapshot.GetData() + if base == nil { + return nil, nil, fmt.Errorf("snapshot has no data") + } + + arts := &artifactSet{original: &sw} + var issues []issue + fail := func(category, format string, args ...any) { + issues = append(issues, issue{category: category, detail: fmt.Sprintf(format, args...)}) + } + + json1, err := anyblockjson.Marshal(sw.SbType, base, opts) + if err != nil { + fail("export_error", "%v", err) + return issues, arts, nil + } + arts.json1 = json1 + + sbType2, reimported, err := anyblockjson.Unmarshal(json1, opts) + if err != nil { + fail("import_error", "%v", err) + return issues, arts, nil + } + arts.reimported = &pb.SnapshotWithType{SbType: sbType2, Snapshot: &pb.ChangeSnapshot{Data: reimported}} + if sbType2 != sw.SbType { + fail("kind_mismatch", "exported %v, reimported %v", sw.SbType, sbType2) + } + + json2, err := anyblockjson.Marshal(sbType2, reimported, opts) + if err != nil { + fail("reexport_error", "%v", err) + return issues, arts, nil + } + // Attribution is written from a DERIVED detail that import deliberately + // drops (SPEC §3): `creator` and `last_modified_by` — spelled + // # since v0.27 — are recovered from the + // object tree's own signature, so a real import re-derives them — but this + // round trip has no tree, so gen2 has nothing to write and the two + // generations differ on exactly those lines. Comparing them raw reports + // every object in an account as unstable and buries whatever else moved: + // measured, 37,011 of 37,429, and 4,000 of a 4,001 sample differed by + // nothing else. So the check strips the lines it knows are owed to the + // tree, the way the detail comparator already skips the internal keys. + if stripAttribution(json1) != stripAttribution(json2) { + arts.json2 = json2 + fail("not_byte_stable", "first divergence: %s", + firstDiff([]byte(stripAttribution(json1)), []byte(stripAttribution(json2)))) + } + + // the ORIGINAL smartblock type: how many type slots the envelope had is a + // question about the snapshot that went in (§2), and sbType2 is the answer + // the round trip produced — using it would make the diff agree with a + // round trip that changed the kind + for _, d := range snapshotdiff.Compare(base, reimported, sw.SbType, opts) { + fail("data_loss", "%s", d) + } + return issues, arts, nil +} + +// maxIndentOf reads the deepest block indent in an exported document — the +// per-object depth datum for the sweep histogram. +func maxIndentOf(jsonDoc []byte) int { + var doc struct { + Blocks []struct { + Indent int `json:"indent"` + } `json:"blocks"` + } + if err := json.Unmarshal(jsonDoc, &doc); err != nil { + return 0 + } + maxIndent := 0 + for _, b := range doc.Blocks { + if b.Indent > maxIndent { + maxIndent = b.Indent + } + } + return maxIndent +} + +// countCellsWithChildren counts table cell blocks carrying real descendants — +// the trigger for the §6.1 array-form cell encoding. +func countCellsWithChildren(base *model.SmartBlockSnapshotBase) int { + if base == nil { + return 0 + } + byId := map[string]*model.Block{} + for _, b := range base.Blocks { + if b != nil { + byId[b.Id] = b + } + } + n := 0 + for _, b := range base.Blocks { + if b == nil { + continue + } + if _, ok := b.Content.(*model.BlockContentOfTableRow); !ok { + continue + } + for _, cid := range b.ChildrenIds { + if cell := byId[cid]; cell != nil && len(cell.ChildrenIds) > 0 { + n++ + } + } + } + return n +} + +// indentPercentiles reads p50/p95/max of the per-object max-indent histogram. +func indentPercentiles(hist map[int]int) (p50, p95, maxDepth int) { + total := 0 + depths := make([]int, 0, len(hist)) + for d, n := range hist { + depths = append(depths, d) + total += n + } + if total == 0 { + return 0, 0, 0 + } + sort.Ints(depths) + maxDepth = depths[len(depths)-1] + cum := 0 + got50, got95 := false, false + for _, d := range depths { + cum += hist[d] + if !got50 && cum*100 >= total*50 { + p50, got50 = d, true + } + if !got95 && cum*100 >= total*95 { + p95, got95 = d, true + break + } + } + return p50, p95, maxDepth +} + +// firstDiff reports the first differing line between two JSON generations. +func firstDiff(a, b []byte) string { + la, lb := strings.Split(string(a), "\n"), strings.Split(string(b), "\n") + for i := 0; i < len(la) && i < len(lb); i++ { + if la[i] != lb[i] { + return fmt.Sprintf("line %d: %q vs %q", i+1, strings.TrimSpace(la[i]), strings.TrimSpace(lb[i])) + } + } + return fmt.Sprintf("lengths differ: %d vs %d lines", len(la), len(lb)) +} + +// +// ---- artifacts ---- +// + +func writeArtifacts(dir, pbPath string, issues []issue, arts *artifactSet) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + raw, err := os.ReadFile(pbPath) + if err != nil { + return fmt.Errorf("read original: %w", err) + } + if err := os.WriteFile(filepath.Join(dir, "original.pb"), raw, 0o644); err != nil { + return fmt.Errorf("write original.pb: %w", err) + } + marshaler := jsonpb.Marshaler{Indent: " "} + // typed parameter, not proto.Message: a nil *pb.SnapshotWithType wrapped + // in the interface would defeat the nil guard and crash jsonpb + writeProtoJSON := func(name string, m *pb.SnapshotWithType) error { + if m == nil { + return nil + } + s, err := marshaler.MarshalToString(m) + if err != nil { + return fmt.Errorf("jsonpb %s: %w", name, err) + } + return os.WriteFile(filepath.Join(dir, name), []byte(s), 0o644) + } + if err := writeProtoJSON("original.pb.json", arts.original); err != nil { + return err + } + if err := writeProtoJSON("reimported.pb.json", arts.reimported); err != nil { + return err + } + if arts.json1 != nil { + if err := os.WriteFile(filepath.Join(dir, "roundtrip.json"), arts.json1, 0o644); err != nil { + return fmt.Errorf("write roundtrip.json: %w", err) + } + } + if arts.json2 != nil { + if err := os.WriteFile(filepath.Join(dir, "roundtrip2.json"), arts.json2, 0o644); err != nil { + return fmt.Errorf("write roundtrip2.json: %w", err) + } + } + var report strings.Builder + for _, is := range issues { + fmt.Fprintf(&report, "[%s] %s\n", is.category, is.detail) + } + if err := os.WriteFile(filepath.Join(dir, "report.txt"), []byte(report.String()), 0o644); err != nil { + return fmt.Errorf("write report: %w", err) + } + return nil +} + +func sortedKeys(m map[string]int) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// +// ---- the §2f composition: dictionary, manifest, omitted relation documents ---- +// + +// spaceComposer is a thin adapter over the shared production composer +// (pkg/lib/anyblockjson/compose) — the §2f/§2c composition used to live +// here as a private copy, and moving it made this sweep an end-to-end test +// of the code the native exporter ships. What stays local is the sweep's +// own bookkeeping: the omitted-bytes tally (the composer never sees the +// bytes an omitted document WOULD have been) and the dump root the manifest +// paths are relative to. +type spaceComposer struct { + c *compose.Composer + // root is the dumped tree's root — the directory the kind folders hang + // off — so the manifest's paths and the two bundle files land relative + // to it. + root string + + wrote bool + omittedDocs int + omittedBytes int +} + +func newSpaceComposer(opts anyblockjson.Options, spaceName, root string) *spaceComposer { + return &spaceComposer{c: compose.NewComposer(opts, spaceName), root: root} +} + +// observeSnapshot classifies one snapshot through the shared composer, +// mapping its findings onto the sweep's issue shape. +func (s *spaceComposer) observeSnapshot(sw *pb.SnapshotWithType) (omitted bool, issues []issue) { + omitted, found := s.c.Observe(sw.SbType, sw.Snapshot.GetData()) + for _, f := range found { + issues = append(issues, issue{category: f.Category, detail: f.Detail}) + } + return omitted, issues +} + +// observeWritten records a written document by its dump-relative path — the +// spelling the manifest carries — with the marshalled bytes for the +// used-key census the dictionary's used-only rule needs. +func (s *spaceComposer) observeWritten(sw *pb.SnapshotWithType, data []byte, path string) error { + rel, err := filepath.Rel(s.root, path) + if err != nil { + return fmt.Errorf("rebase %s onto the dump root: %w", path, err) + } + s.wrote = true + return s.c.ObserveWritten(sw.SbType, sw.Snapshot.GetData(), data, filepath.ToSlash(rel)) +} + +// finish writes the space's properties.json and index.json at the dump +// root. The shared composer re-reads both through the package's own +// Unmarshal before handing them back — the bundle-level I1 discipline: a +// file this tool writes that the package refuses is a bug found now rather +// than at restore time. +func (s *spaceComposer) finish(ss *spaceSummary) error { + if !s.wrote { + return nil + } + idxData, dictData, stats, err := s.c.Finish() + if err != nil { + return fmt.Errorf("compose bundle files: %w", err) + } + if err := os.WriteFile(filepath.Join(s.root, anyblockjson.PropertiesFileName), dictData, 0o644); err != nil { + return fmt.Errorf("write property dictionary: %w", err) + } + if err := os.WriteFile(filepath.Join(s.root, anyblockjson.IndexFileName), idxData, 0o644); err != nil { + return fmt.Errorf("write index: %w", err) + } + ss.OmittedRelationDocs = s.omittedDocs + ss.OmittedBytes = s.omittedBytes + ss.DictionaryInstalled = stats.DictionaryInstalled + ss.DictionaryEntries = stats.DictionaryEntries + ss.ManifestTypes = stats.ManifestTypes + ss.OptionDocs = stats.OptionDocs + ss.DictionaryBytes = stats.DictionaryBytes + ss.IndexBytes = stats.IndexBytes + ss.OrphanUsedKeys = stats.OrphanUsedKeys + return nil +} diff --git a/cmd/anyblockroundtrip/native.go b/cmd/anyblockroundtrip/native.go new file mode 100644 index 0000000000..c511ed9863 --- /dev/null +++ b/cmd/anyblockroundtrip/native.go @@ -0,0 +1,693 @@ +package main + +// native.go — the -native mode: drive the REAL exporter +// (core/block/export/anyblock) over every space of a live account and verify +// what its unit tests cannot. The default mode round-trips the LEGACY pb +// export through the codec, which exercises the codec and (since the +// composer moved to pkg/lib/anyblockjson/compose) the composition — but +// never the exporter itself: its layout, kind classification, blob binding +// and concurrency had no real-data coverage until this mode existed. +// +// Ground truth per space is the legacy pb export taken in the same process, +// moments earlier, with the same flags (no files, so the pb path's +// Source-clobber never fires and its snapshots are unmutilated). For every +// document the native exporter writes, three fidelity checks run against +// that truth: +// +// - byte equality with Marshal(pb snapshot): the exporter builds its +// snapshot from live state (BlocksToSave, CombinedDetails, …) while the +// pb file is the pbc converter's rendering of the same state — if the +// two constructions differ in ANY corner, the canonical bytes differ +// and this check names the first line; +// - snapshotdiff against the pb snapshot after a full Unmarshal: the +// baseline data-loss measure, on the native bytes — "no worse than the +// pb sweep's 34/38,105" is checked here, object for object; +// - Marshal ∘ Unmarshal byte stability of the native document itself, +// attribution-stripped like the default mode (import drops the derived +// creator/last_modified_by, so gen2 cannot restate them). +// +// A pb document that the native bundle does NOT carry must be claimed by an +// omission predicate (space settings, profile page, widget, installed +// bundled relation) — anything else is a document the exporter LOST, the +// disqualifying failure. The reverse (native-only) and byte-diff cases can +// also arise from real state drift between the two exports (sync is live +// during the sweep); the per-category counts make that judgement possible. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/anyproto/any-sync/app" + "github.com/gogo/protobuf/proto" + + "github.com/anyproto/anytype-heart/cmd/internal/anyblockbatch" + "github.com/anyproto/anytype-heart/core" + "github.com/anyproto/anytype-heart/core/block/cache" + "github.com/anyproto/anytype-heart/core/block/export" + "github.com/anyproto/anytype-heart/core/block/export/anyblock" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/snapshotdiff" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/storeresolver" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/space/spacecore/typeprovider" +) + +// nativeDocExt is spelled out here rather than imported from compose: +// the layout check is only a cross-check if its vocabulary does not come +// from the code under test. +const nativeDocExt = ".anyblock.json" + +// nativeDirs is the settled bundle layout (EXPORTER_DESIGN.md §1.2, Q1). +var nativeDirs = map[string]bool{ + "objects": true, "types": true, "templates": true, "properties": true, + "options": true, "participants": true, "files": true, +} + +// legacyDirNames is what must NOT appear anywhere in a native bundle — the +// store-vocabulary layout the design replaced. +var legacyDirNames = map[string]bool{ + "relations": true, "relationsOptions": true, "filesObjects": true, "profile": true, +} + +// kindDirs maps a document's own declared `kind` to the directory it must +// land in. Deliberately a SECOND table, independent of compose.KindDirectory +// (which maps smartblock types): the classification cross-check is only a +// check if the two answers come from different roads. A kind not listed — +// or an omitted kind, which the format defines as `page` — belongs in +// objects/, the home for everything without a dedicated one. +var kindDirs = map[string]string{ + "object_type": "types", "bundled_object_type": "types", + "template": "templates", "bundled_template": "templates", + "property": "properties", "bundled_property": "properties", + "property_option": "options", + "participant": "participants", + "file_object": "files", "file": "files", +} + +type nativeSpaceSummary struct { + SpaceId string `json:"spaceId"` + SpaceName string `json:"spaceName,omitempty"` + Succeed int `json:"succeed"` + Docs int `json:"docs"` + Blobs int `json:"blobs"` + Omitted int `json:"omitted"` + DirCounts map[string]int `json:"dirCounts"` + Issues map[string]int `json:"issues,omitempty"` + IndexBytes int `json:"indexBytes,omitempty"` + DictBytes int `json:"dictBytes,omitempty"` +} + +type nativeSummary struct { + Account string `json:"account"` + Spaces int `json:"spaces"` + Docs int `json:"docs"` + Blobs int `json:"blobs"` + Omitted int `json:"omitted"` + DirCounts map[string]int `json:"dirCounts"` + Issues map[string]int `json:"issues"` + LossObjects int `json:"lossObjects"` + PerSpace []nativeSpaceSummary `json:"perSpace"` + SpaceErrors []spaceError `json:"spaceErrors,omitempty"` +} + +// runNative is the -native mode's whole flow over one account. +func runNative(ctx context.Context, mw *core.Middleware, store objectstore.ObjectStore, + spaces []spaceInfo, outDir string, includeFiles bool) error { + + a := mw.GetApp() + exporter := &anyblock.Exporter{ + Collector: app.MustComponent[export.Export](a), + Picker: app.MustComponent[cache.CachedObjectGetter](a), + ObjectStore: store, + SbtProvider: app.MustComponent[typeprovider.SmartBlockTypeProvider](a), + } + + sum := &nativeSummary{Account: "", Spaces: len(spaces), DirCounts: map[string]int{}, Issues: map[string]int{}} + for _, s := range spaces { + fmt.Printf("\n== native %s (%s)\n", s.id, s.name) + ss, err := processSpaceNative(ctx, mw, store, exporter, s, outDir, includeFiles) + if err != nil { + fmt.Printf(" space failed: %v\n", err) + sum.SpaceErrors = append(sum.SpaceErrors, spaceError{SpaceId: s.id, Error: err.Error()}) + continue + } + sum.PerSpace = append(sum.PerSpace, *ss) + sum.Docs += ss.Docs + sum.Blobs += ss.Blobs + sum.Omitted += ss.Omitted + for d, n := range ss.DirCounts { + sum.DirCounts[d] += n + } + for c, n := range ss.Issues { + sum.Issues[c] += n + } + if n := ss.Issues["native_data_loss_object"]; n > 0 { + sum.LossObjects += n + } + } + + data, err := json.MarshalIndent(sum, "", " ") + if err != nil { + return fmt.Errorf("marshal native summary: %w", err) + } + sumPath := filepath.Join(outDir, "native-summary.json") + if err := os.WriteFile(sumPath, data, 0o644); err != nil { + return fmt.Errorf("write native summary: %w", err) + } + + fmt.Printf("\n==== native: %d spaces, %d docs, %d blobs, %d omitted\n", sum.Spaces, sum.Docs, sum.Blobs, sum.Omitted) + for _, d := range sortedKeys(sum.DirCounts) { + fmt.Printf(" %-14s %d\n", d, sum.DirCounts[d]) + } + if len(sum.Issues) == 0 { + fmt.Println(" issues: none") + } + for _, c := range sortedKeys(sum.Issues) { + fmt.Printf(" ISSUE %-28s %d\n", c, sum.Issues[c]) + } + fmt.Println("summary:", sumPath) + hard := 0 + for c, n := range sum.Issues { + // codec-level loss is measured against the pb sweep's own baseline in + // the analysis, and timestamp drift is upstream state instability + // (stripVolatileDates); everything else here is an exporter defect + if c != "native_data_loss" && c != "native_data_loss_object" && c != "state_drift_timestamps" { + hard += n + } + } + if hard > 0 || len(sum.SpaceErrors) > 0 { + return fmt.Errorf("native sweep found %d issues, %d space errors", hard, len(sum.SpaceErrors)) + } + return nil +} + +func processSpaceNative(ctx context.Context, mw *core.Middleware, store objectstore.ObjectStore, + exporter *anyblock.Exporter, s spaceInfo, outDir string, includeFiles bool) (*nativeSpaceSummary, error) { + + ss := &nativeSpaceSummary{SpaceId: s.id, SpaceName: s.name, + DirCounts: map[string]int{}, Issues: map[string]int{}} + report := func(category, format string, args ...any) { + if ss.Issues[category] < 10 { // keep the log usable; the counts carry the rest + fmt.Printf(" ISSUE %s: %s\n", category, fmt.Sprintf(format, args...)) + } + ss.Issues[category]++ + } + + // 1. ground truth: the legacy pb export, same process, same flags — + // no files, so the Source-clobber never fires + pbDir := filepath.Join(outDir, "native-pb", s.id) + resp := mw.ObjectListExport(ctx, &pb.RpcObjectListExportRequest{ + SpaceId: s.id, + Path: pbDir, + Format: model.Export_Protobuf, + IncludeArchived: true, + NoProgress: true, + }) + if resp.Error != nil && resp.Error.Code != 0 { + return nil, rpcErr("ObjectListExport", int32(resp.Error.Code), resp.Error.Description) + } + pbRoot := resp.Path + if pbRoot == "" { + pbRoot = pbDir + } + defer os.RemoveAll(pbDir) + pbPaths := map[string]string{} // id → .pb path + err := filepath.WalkDir(pbRoot, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(p, ".pb") { + pbPaths[strings.TrimSuffix(filepath.Base(p), ".pb")] = p + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk pb export: %w", err) + } + + // 2. the native bundle + bundleRoot := filepath.Join(outDir, "native", s.id) + if err := os.RemoveAll(bundleRoot); err != nil { + return nil, fmt.Errorf("clean bundle dir: %w", err) + } + wr, err := anyblock.NewDirWriter(bundleRoot) + if err != nil { + return nil, fmt.Errorf("create bundle writer: %w", err) + } + req := anyblock.Request{ + SpaceId: s.id, + SpaceName: s.name, + IncludeArchived: true, + IncludeFiles: includeFiles, + } + result, err := exporter.Export(ctx, req, wr) + if err != nil { + return nil, fmt.Errorf("native export: %w", err) + } + ss.Succeed = result.Succeed + if result.DocErrors > 0 { + report("doc_emit_failed", "%d document(s) failed to emit", result.DocErrors) + } + if result.BlobErrors > 0 { + report("blob_stream_failed", "%d file blob(s) could not be streamed; documents travel without bytes", result.BlobErrors) + } + + // 3. layout, filenames, kinds + docs, blobs, err := checkBundleLayout(bundleRoot, ss, report) + if err != nil { + return nil, fmt.Errorf("check bundle layout: %w", err) + } + ss.Docs = len(docs) + ss.Blobs = len(blobs) + + // 4. the bundle files and the manifest's blob bindings + idx := checkBundleFiles(bundleRoot, docs, blobs, ss, report) + + // 5. fidelity against the pb ground truth + opts := storeresolver.New(store.SpaceIndex(s.id)).Options() + nativeIds := map[string]bool{} + for _, docPath := range docs { + // the filename stem is the ENVELOPE id; the pb export names files by + // the STORE id, which for a participant is the unfolded composite — + // try both spellings, and mark both as covered + id := strings.TrimSuffix(filepath.Base(docPath), nativeDocExt) + nativeIds[id] = true + pbPath, ok := pbPaths[id] + if !ok { + composite := domain.NewParticipantId(s.id, id) + if pbPath, ok = pbPaths[composite]; ok { + nativeIds[composite] = true + } + } + if !ok { + report("only_in_native", "%s has no pb ground truth (state drift, or a doc the legacy path skips)", id) + continue + } + checkNativeDoc(id, filepath.Join(bundleRoot, filepath.FromSlash(docPath)), pbPath, opts, report) + } + expectedOmissions := 0 + for id, pbPath := range pbPaths { + if nativeIds[id] { + continue + } + sw, err := readPbSnapshot(pbPath) + if err != nil { + report("pb_unreadable", "%s: %v", id, err) + continue + } + base := sw.Snapshot.GetData() + switch { + case anyblockjson.OmittedSpaceSettings(sw.SbType, base), + anyblockjson.OmittedProfilePage(sw.SbType, base), + anyblockjson.OmittedWidgetObject(sw.SbType, base): + expectedOmissions++ + default: + if _, ok := anyblockjson.OmittedBundledRelation(sw.SbType, base, opts); ok { + expectedOmissions++ + } else { + report("missing_in_native", "%s (%v) is in the pb export but not the native bundle and no omission predicate claims it", id, sw.SbType) + } + } + } + ss.Omitted = expectedOmissions + + // 6. determinism: export again, byte-compare the whole tree, drop the copy + detRoot := filepath.Join(outDir, "native-det", s.id) + if err := os.RemoveAll(detRoot); err != nil { + return nil, fmt.Errorf("clean determinism dir: %w", err) + } + detWr, err := anyblock.NewDirWriter(detRoot) + if err != nil { + return nil, fmt.Errorf("create determinism writer: %w", err) + } + if _, err := exporter.Export(ctx, req, detWr); err != nil { + return nil, fmt.Errorf("determinism export: %w", err) + } + compareTrees(bundleRoot, detRoot, report) + if err := os.RemoveAll(detRoot); err != nil { + return nil, fmt.Errorf("drop determinism tree: %w", err) + } + + _ = idx + fmt.Printf(" %d docs (%d blobs, %d omitted), dirs: %v\n", ss.Docs, ss.Blobs, ss.Omitted, dirLine(ss.DirCounts)) + return ss, nil +} + +// checkBundleLayout walks the bundle and verifies the settled layout: only +// the seven kind directories and the two bundle files at the root, no +// legacy names anywhere, every document named .anyblock.json with the +// stem equal to its own envelope id, every document in the directory its +// declared kind demands, and non-document files only in files/. +func checkBundleLayout(root string, ss *nativeSpaceSummary, report func(string, string, ...any)) (docs, blobs []string, err error) { + entries, err := os.ReadDir(root) + if os.IsNotExist(err) { + return nil, nil, nil // an empty space writes nothing + } + if err != nil { + return nil, nil, fmt.Errorf("read bundle root: %w", err) + } + for _, e := range entries { + name := e.Name() + if legacyDirNames[name] { + report("layout_legacy_name", "legacy entry %q at the bundle root", name) + continue + } + if e.IsDir() { + if !nativeDirs[name] { + report("layout_alien_entry", "unexpected directory %q at the bundle root", name) + } + continue + } + if name != anyblockjson.IndexFileName && name != anyblockjson.PropertiesFileName { + report("layout_alien_entry", "unexpected file %q at the bundle root", name) + } + } + err = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + parts := strings.SplitN(rel, "/", 3) + if len(parts) == 1 { + return nil // the two root files, checked above + } + if len(parts) > 2 { + report("layout_alien_entry", "nested path %q — the layout is one kind directory deep", rel) + return nil + } + dir, base := parts[0], parts[1] + if legacyDirNames[dir] { + report("layout_legacy_name", "legacy directory in %q", rel) + return nil + } + if !strings.HasSuffix(base, nativeDocExt) { + if dir != "files" { + report("layout_alien_entry", "non-document file %q outside files/", rel) + return nil + } + blobs = append(blobs, rel) + return nil + } + docs = append(docs, rel) + ss.DirCounts[dir]++ + id := strings.TrimSuffix(base, nativeDocExt) + var probe struct { + Id string `json:"id"` + Kind string `json:"kind"` + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + if err := json.Unmarshal(data, &probe); err != nil { + report("doc_unparsable", "%s: %v", rel, err) + return nil + } + if probe.Id != id { + report("stem_id_mismatch", "%s: filename stem %q, envelope id %q — the path stopped being a pure function of the id", rel, id, probe.Id) + } + expected := "objects" + if d, ok := kindDirs[probe.Kind]; ok { + expected = d + } + if dir != expected { + report("misfiled_kind", "%s declares kind %q and belongs in %s/", rel, probe.Kind, expected) + } + return nil + }) + if err != nil { + return nil, nil, fmt.Errorf("walk bundle: %w", err) + } + sort.Strings(docs) + sort.Strings(blobs) + return docs, blobs, nil +} + +// checkBundleFiles reads index.json and properties.json back through the +// package, runs the manifest's blob bindings cross-check, and verifies +// every on-disk blob is bound and no blob path collides with a document. +func checkBundleFiles(root string, docs, blobs []string, ss *nativeSpaceSummary, report func(string, string, ...any)) *anyblockjson.Index { + if len(docs) == 0 { + return nil + } + idxData, err := os.ReadFile(filepath.Join(root, anyblockjson.IndexFileName)) + if err != nil { + report("bundle_file_missing", "index.json: %v", err) + return nil + } + ss.IndexBytes = len(idxData) + idx, err := anyblockjson.UnmarshalIndex(idxData) + if err != nil { + report("bundle_file_invalid", "index.json: %v", err) + return nil + } + dictData, err := os.ReadFile(filepath.Join(root, anyblockjson.PropertiesFileName)) + if err != nil { + report("bundle_file_missing", "properties.json: %v", err) + } else { + ss.DictBytes = len(dictData) + if _, err := anyblockjson.UnmarshalPropertyDictionary(dictData); err != nil { + report("bundle_file_invalid", "properties.json: %v", err) + } + } + + docPaths := make([]string, 0, len(docs)) + docSet := map[string]bool{} + for _, d := range docs { + docPaths = append(docPaths, filepath.Join(root, filepath.FromSlash(d))) + docSet[d] = true + } + for _, bad := range anyblockbatch.CheckManifestFiles(idx, root, docPaths) { + report("manifest_files", "%s %s: %s", bad.Property, bad.Target, bad.Reason) + } + bound := map[string]bool{} + if idx.Manifest != nil { + for _, p := range idx.Manifest.Files { + bound[p] = true + if docSet[p] { + report("blob_doc_collision", "manifest binds %q, which is a document path", p) + } + } + } + for _, b := range blobs { + if !bound[b] { + report("blob_orphan", "blob %q on disk with no manifest.files entry", b) + } + } + return idx +} + +// checkNativeDoc runs the three fidelity checks for one document against +// its pb ground truth (see the file comment). +func checkNativeDoc(id, docPath, pbPath string, opts anyblockjson.Options, report func(string, string, ...any)) { + nativeJson, err := os.ReadFile(docPath) + if err != nil { + report("doc_unreadable", "%s: %v", id, err) + return + } + sw, err := readPbSnapshot(pbPath) + if err != nil { + report("pb_unreadable", "%s: %v", id, err) + return + } + base := sw.Snapshot.GetData() + if base == nil { + return + } + + json1, err := anyblockjson.Marshal(sw.SbType, base, opts) + if err != nil { + report("pb_marshal_error", "%s: %v", id, err) + return + } + if string(json1) != string(nativeJson) { + if stripVolatileDates(json1) == stripVolatileDates(nativeJson) { + report("state_drift_timestamps", "%s: only created/last-modified date lines differ (load-time-stamped, see stripVolatileDates)", id) + } else { + report("native_byte_diff", "%s: native bytes differ from Marshal(pb snapshot); first divergence: %s", + id, firstDiff(json1, nativeJson)) + } + } + + sbType2, reimported, err := anyblockjson.Unmarshal(nativeJson, opts) + if err != nil { + report("native_import_error", "%s: %v", id, err) + return + } + if sbType2 != sw.SbType { + report("native_kind_mismatch", "%s: pb %v, native reads back as %v", id, sw.SbType, sbType2) + } + if normalizeVolatileDates(base, reimported) { + report("state_drift_timestamps", "%s: created/last-modified date drifted between the two exports", id) + } + diffs := snapshotdiff.Compare(base, reimported, sw.SbType, opts) + for _, d := range diffs { + report("native_data_loss", "%s: %s", id, d) + } + if len(diffs) > 0 { + report("native_data_loss_object", "%s: %d finding(s)", id, len(diffs)) + } + + json2, err := anyblockjson.Marshal(sbType2, reimported, opts) + if err != nil { + report("native_reexport_error", "%s: %v", id, err) + return + } + if stripAttribution(nativeJson) != stripAttribution(json2) { + report("native_not_byte_stable", "%s: first divergence: %s", id, + firstDiff([]byte(stripAttribution(nativeJson)), []byte(stripAttribution(json2)))) + } +} + +// compareTrees byte-compares two directory trees — the corpus-scale +// determinism check (same space, exported twice, same bytes). +func compareTrees(a, b string, report func(string, string, ...any)) { + list := func(root string) map[string]string { + out := map[string]string{} + _ = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + rel, _ := filepath.Rel(root, p) + out[filepath.ToSlash(rel)] = p + return nil + }) + return out + } + first, second := list(a), list(b) + for rel, pa := range first { + pb2, ok := second[rel] + if !ok { + report("nondeterministic", "%s exists in the first export only", rel) + continue + } + da, ea := os.ReadFile(pa) + db, eb := os.ReadFile(pb2) + if ea != nil || eb != nil { + report("nondeterministic", "%s: unreadable (%v, %v)", rel, ea, eb) + continue + } + if string(da) != string(db) { + if stripVolatileDates(da) == stripVolatileDates(db) { + report("state_drift_timestamps", "%s: only load-time-stamped date lines differ between the two exports", rel) + } else { + report("nondeterministic", "%s: content differs between two exports; first divergence: %s", rel, firstDiff(da, db)) + } + } + } + for rel := range second { + if _, ok := first[rel]; !ok { + report("nondeterministic", "%s exists in the second export only", rel) + } + } +} + +func readPbSnapshot(path string) (*pb.SnapshotWithType, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read snapshot: %w", err) + } + var sw pb.SnapshotWithType + if err := proto.Unmarshal(data, &sw); err != nil { + return nil, fmt.Errorf("unmarshal snapshot: %w", err) + } + return &sw, nil +} + +func dirLine(counts map[string]int) string { + parts := make([]string, 0, len(counts)) + for _, d := range sortedKeys(counts) { + parts = append(parts, fmt.Sprintf("%s=%d", d, counts[d])) + } + return strings.Join(parts, " ") +} + +// stripVolatileDates removes the two LOAD-TIME-STAMPED details from a +// rendered document, the way stripAttribution removes the tree-derived +// pair: an object whose root change carries no creation date gets +// `createdDate = time.Now()` stamped on Apply +// (core/block/editor/smartblock/smartblock.go:733), so a fresh load — and +// close-after-write means every export IS a fresh load — re-mints it. +// Participant documents are the population that trips this (derived +// objects, no creation info in the tree). Two exports seconds apart then +// differ on exactly these lines with identical content otherwise; that is +// upstream state instability, not exporter nondeterminism — the legacy pb +// exporter re-exported after an eviction shifts the same way — and the +// classification keeps the determinism check sharp instead of failing the +// sweep on an app behaviour the exporter cannot control. +// volatileDateMemberPrefixes are the wire spellings of the two load-stamped +// date members, derived from the bundled name table rather than written out: +// the format spells a key by its display name, so hard-coding the retired +// slug silently stopped matching and the drift it classifies came back as a +// real difference. The stored keys are listed beside them for the space that +// holds a custom name-twin and makes the claimant spell its own key (§3). +var volatileDateMemberPrefixes = []string{ + `"` + (anyblockjson.BundledKeyVocabulary{}).PropertySlug("createdDate") + `":`, + `"` + (anyblockjson.BundledKeyVocabulary{}).PropertySlug("lastModifiedDate") + `":`, + `"createdDate":`, + `"lastModifiedDate":`, +} + +func hasAnyPrefix(s string, prefixes []string) bool { + for _, p := range prefixes { + if strings.HasPrefix(s, p) { + return true + } + } + return false +} + +func stripVolatileDates(doc []byte) string { + var out []string + stripped := false + for _, line := range strings.Split(string(doc), "\n") { + t := strings.TrimSpace(line) + if hasAnyPrefix(t, volatileDateMemberPrefixes) { + stripped = true + continue + } + if stripped && len(out) > 0 && strings.HasPrefix(t, "}") { + out[len(out)-1] = strings.TrimSuffix(out[len(out)-1], ",") + } + stripped = false + out = append(out, line) + } + return strings.Join(out, "\n") +} + +// normalizeVolatileDates aligns the two load-time-stamped details between +// the pb ground truth and the reimported native snapshot before the loss +// comparison, and reports whether they actually differed. Only a VALUE +// difference with both sides present is aligned — a side missing the key +// entirely stays visible to the comparator, so a dropped timestamp still +// counts as loss. +func normalizeVolatileDates(pbBase, re *model.SmartBlockSnapshotBase) (drifted bool) { + pf := pbBase.GetDetails().GetFields() + rf := re.GetDetails().GetFields() + if pf == nil || rf == nil { + return false + } + for _, key := range []string{"createdDate", "lastModifiedDate"} { + pv, rv := pf[key], rf[key] + if pv == nil || rv == nil { + continue + } + if pv.GetNumberValue() != rv.GetNumberValue() { + drifted = true + rf[key] = pv + } + } + return drifted +} diff --git a/cmd/anyblockvalidate/agreement_test.go b/cmd/anyblockvalidate/agreement_test.go new file mode 100644 index 0000000000..80353a1b74 --- /dev/null +++ b/cmd/anyblockvalidate/agreement_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/cmd/internal/anyblockbatch" +) + +// The two tools must agree about one bundle. §2f gave a property format TWO +// homes — a type's `type_settings.property_definitions` and the dictionary — +// and anyblockconvert merges the dictionary into the format table before +// running CheckPropertyFormats. When this tool did not, it refused a bundle +// the converter converted cleanly, and the repair text sent the author to the +// type home, undoing the dictionary they had correctly written. +// +// How this can fail: drop the dictionary merge in main.go and the bundle +// below reports undeclared formats that anyblockconvert accepts. +func TestValidate_AgreesWithConvertOnADictionaryDeclaredBundle(t *testing.T) { + // given a bundle whose formats are declared ONLY in the dictionary + dir := t.TempDir() + write := func(name, body string) { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644)) + } + write("properties.json", `{"version":2,"properties":[ + {"property":"episode_number","name":"Episode Number","format":"number"}, + {"property":"release_date","name":"Release Date","format":"date"}]}`) + write("ep1.json", `{"version":2,"kind":"page","id":"bafyreiep1", + "properties":{"name":"Ep 1","episode_number":1,"release_date":"2026-01-01T00:00:00Z"}}`) + + files := []string{filepath.Join(dir, "ep1.json")} + + // when the format table is built the way this tool builds it + formats, err := anyblockbatch.ScanFormats(files) + require.NoError(t, err) + dictFormats, _, err := anyblockbatch.DictionaryFormats(filepath.Join(dir, "properties.json")) + require.NoError(t, err) + merged := anyblockbatch.MergeDictionaryFormats(formats, dictFormats, func(string, ...any) {}) + + // then nothing is undeclared — which is what anyblockconvert concludes + undeclared, err := anyblockbatch.CheckPropertyFormats(files, merged) + require.NoError(t, err) + assert.Empty(t, undeclared, "a dictionary-declared format is declared (§2f)") + + // and without the merge it would have been refused, which is the bug. + // Re-scan rather than reusing `formats`: MergeDictionaryFormats folds + // into the map it is given, so the pre-merge table is gone by now. + bare, err := anyblockbatch.ScanFormats(files) + require.NoError(t, err) + withoutDict, err := anyblockbatch.CheckPropertyFormats(files, bare) + require.NoError(t, err) + assert.Len(t, withoutDict, 2, "the merge is what makes the two tools agree") +} + +// The repair sentence must name BOTH homes. Naming only the type home sends +// an author who declared the property in the dictionary to declare it again +// somewhere else. +// +// How this can fail: drop either home from Report's message. +func TestReport_NamesBothDeclarationHomes(t *testing.T) { + msg := anyblockbatch.Report([]anyblockbatch.Undeclared{{File: "x.json", Key: "episode_number"}}) + assert.Contains(t, msg, "properties.json") + assert.Contains(t, msg, "type_settings.property_definitions") +} diff --git a/cmd/anyblockvalidate/main.go b/cmd/anyblockvalidate/main.go new file mode 100644 index 0000000000..c66b28f14e --- /dev/null +++ b/cmd/anyblockvalidate/main.go @@ -0,0 +1,255 @@ +package main + +import ( + "fmt" + "github.com/anyproto/anytype-heart/cmd/internal/anyblockbatch" + "os" + "path/filepath" + "strings" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" +) + +func main() { + if len(os.Args) < 2 { + fmt.Println("usage: anyblockvalidate ...") + os.Exit(2) + } + var files, indexes, dictionaries []string + for _, arg := range os.Args[1:] { + _ = filepath.Walk(arg, func(p string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() || !strings.HasSuffix(p, ".json") { + return nil + } + // the bundle index (§2c) and the property dictionary (§2f) + // describe the bundle, not an object: each has its own schema + // and would fail every object-level check + if filepath.Base(p) == anyblockjson.IndexFileName { + indexes = append(indexes, p) + return nil + } + if filepath.Base(p) == anyblockjson.PropertiesFileName { + dictionaries = append(dictionaries, p) + return nil + } + files = append(files, p) + return nil + }) + } + // what the manifests bind as BLOBS is content, not documents: a FAT + // bundle legitimately carries .json blobs (12 in the corpus), and only + // the `files` map can tell them from an authored bare-.json document + // (§2c, v0.47). Excluded before any object-level check runs, or every + // bound .json blob is reported as an invalid document. + if blobs := anyblockbatch.ManifestBlobPaths(indexes); len(blobs) > 0 { + kept := files[:0] + for _, f := range files { + if !blobs[f] { + kept = append(kept, f) + } + } + files = kept + } + + fail, warned := 0, 0 + + // the `_` namespace is the platform's (§1). anyblockconvert refuses a + // bundle that mints an id in it, so this tool has to see it too — a bundle + // this one blesses and that one rejects is the worst of both. + if reserved, err := anyblockbatch.CheckBundleIds(files); err != nil { + fmt.Printf("READERR %v\n", err) + fail++ + } else if len(reserved) > 0 { + fmt.Printf("INVALID %d object(s) claiming a reserved id\n%s", len(reserved), anyblockbatch.ReportTargets(reserved)) + fail += len(reserved) + } + + if len(indexes) == 0 { + warned++ + fmt.Printf("warn no index.json found\n without one the space has no name, no entry point and no sidebar (§2c)\n") + } + for _, idxPath := range indexes { + data, err := os.ReadFile(idxPath) + if err != nil { + fmt.Printf("READERR %s: %v\n", idxPath, err) + fail++ + } else if idx, err := anyblockjson.UnmarshalIndex(data); err != nil { + fmt.Printf("INVALID %s\n %v\n", idxPath, err) + fail++ + } else { + dangling := anyblockbatch.CheckIndexTargets(idx, files) + // the manifest's blob bindings are index references too (§2c, + // v0.47), and theirs is the other silent failure: a file + // document whose bytes the entry promises and the archive does + // not carry + dangling = append(dangling, anyblockbatch.CheckManifestFiles(idx, filepath.Dir(idxPath), files)...) + // the inverse is warning-grade: a file document a PRESENT map + // does not bind is the signature of a partially failed export — + // its bytes did not travel, and the exporter said so by + // omitting the binding (§2c) + for _, id := range anyblockbatch.UnboundFileDocuments(idx, files) { + warned++ + fmt.Printf("warn %s: file document %q has no manifest.files binding — its bytes did not travel with this bundle\n", idxPath, id) + } + if len(dangling) > 0 { + fmt.Printf("INVALID %s\n%s", idxPath, anyblockbatch.ReportTargets(dangling)) + fail += len(dangling) + } else { + declared, effective := idx.EntryPoint(), idx.EffectiveEntryPoint() + home := idx.SpaceHomepage() + if home == "" { + home = "(the widgets screen)" + } + shown := effective + if shown == "" { + shown = "(nothing — no widget names an object)" + } + fmt.Printf("ok %s\n space homepage %s · %d sidebar widget(s)\n", + idxPath, home, len(idx.Widgets)) + // TEMPORARY: pb.Profile has no entry-point field, so the + // built-in-archive path (inject) opens widgets[0]. A declared + // entrypoint that is not the first widget is silently not + // honoured there. On the experience path — what a bundle + // actually takes — nothing opens once at all, so the entrypoint + // only reaches the space as the homepage fallback (§2c). + if declared != "" && declared != effective { + warned++ + fmt.Printf(" warn: entrypoint %q is not the first widget, so on the built-in-archive path\n"+ + " it is NOT what opens — inject uses widgets[0] (%s).\n"+ + " List the entrypoint first in widgets.\n", declared, effective) + } + } + } + } + for _, dictPath := range dictionaries { + data, err := os.ReadFile(dictPath) + if err != nil { + fmt.Printf("READERR %s: %v\n", dictPath, err) + fail++ + continue + } + // the codec TOLERATES an installed key its bundled table cannot name + // (a newer app's bundled property), but it now SAYS so through the + // same warn channel object documents have — this tool used to carry + // its own copy of that check, which meant the authoring surface knew + // something the format itself did not report. + var dictWarnings []anyblockjson.Issue + dict, err := anyblockjson.UnmarshalPropertyDictionaryWarn(data, func(i anyblockjson.Issue) { + dictWarnings = append(dictWarnings, i) + }) + if err != nil { + fmt.Printf("INVALID %s\n %v\n", dictPath, err) + fail++ + continue + } + fmt.Printf("ok %s\n %d installed key(s), %d defined propert%s\n", + dictPath, len(dict.Installed), len(dict.Properties), + map[bool]string{true: "y", false: "ies"}[len(dict.Properties) == 1]) + if len(dictWarnings) > 0 { + warned++ + for _, w := range dictWarnings { + fmt.Printf(" warn: %s\n", w.String()) + } + } + } + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + fmt.Printf("READERR %s: %v\n", f, err) + fail++ + continue + } + var warnings []anyblockjson.Issue + err = anyblockjson.ValidateWarn(data, func(i anyblockjson.Issue) { + warnings = append(warnings, i) + }) + if err != nil { + fmt.Printf("INVALID %s\n %v\n", f, err) + fail++ + continue + } + if len(warnings) > 0 { + warned++ + fmt.Printf("warn %s\n", f) + for _, w := range warnings { + fmt.Printf(" %v\n", w) + } + continue + } + fmt.Printf("ok %s\n", f) + } + if shared, serr := anyblockbatch.CheckSharedSelects(files); serr == nil && len(shared) > 0 { + warned += len(shared) + fmt.Printf("\nSHARED select properties:\n%s", anyblockbatch.ReportSharedSelects(shared)) + } + + // batch-wide: a property whose format no type declares converts to raw + // JSON — dates stay strings, selects mint no options, object references + // are never remapped. Per-file validation cannot see it. + if formats, ferr := anyblockbatch.ScanFormats(files); ferr == nil { + // the dictionary is the OTHER home a format can be declared in + // (§2f), and anyblockconvert merges it before running this exact + // check. Without the same merge this tool refuses a bundle the + // converter accepts — and its repair text then sends the author to + // the type home, undoing the dictionary they had written. + for _, dictPath := range dictionaries { + dictFormats, _, derr := anyblockbatch.DictionaryFormats(dictPath) + if derr != nil { + continue // the per-file pass above already reported it + } + formats = anyblockbatch.MergeDictionaryFormats(formats, dictFormats, + func(format string, args ...any) { + warned++ + fmt.Printf("warn "+format+"\n", args...) + }) + } + if undeclared, uerr := anyblockbatch.CheckPropertyFormats(files, formats); uerr == nil && len(undeclared) > 0 { + fail += len(undeclared) + fmt.Printf("\nUNDECLARED property formats (anyblockconvert will refuse these):\n%s", + anyblockbatch.Report(undeclared)) + } + + // batch-wide for the same reason: a view naming a property nothing + // declares is a filter that matches nothing, a sort that orders + // nothing, a column that stays empty — and it imports in silence. The + // CODEC cannot raise it, because a custom property whose stored key is + // already a legal spelling binds no legend entry, so inside one + // document a typo and a verbatim custom key look the same. Only here, + // holding every declaration in the bundle, are they distinguishable. + declared := map[string]bool{} + for key := range formats { + declared[key] = true + } + if bad, verr := anyblockbatch.CheckViewProperties(files, declared); verr == nil && len(bad) > 0 { + fail += len(bad) + fmt.Printf("\nVIEW slots naming a property nothing declares:\n%s", + anyblockbatch.ReportViewProperties(bad)) + } + } + + // every document this run judged, not just the object ones: `files` + // excludes index.json and properties.json while `fail` counts their + // failures alongside the batch-wide findings, so subtracting one from + // the other printed counts that never happened — "-1/0 valid" for a + // directory holding a single bad dictionary. + judged := len(files) + len(indexes) + len(dictionaries) + valid := judged - fail + if valid < 0 { + // more findings than documents: a batch-wide check can report + // several against one file. Say what is true — nothing passed. + valid = 0 + } + fmt.Printf("\n=== %d/%d valid, %d invalid", valid, judged, fail) + if warned > 0 { + // warnings do not fail the run: the document imports, part of it is + // just inert + fmt.Printf(", %d with warnings", warned) + } + fmt.Println(" ===") + if fail > 0 { + os.Exit(1) + } +} diff --git a/cmd/internal/anyblockbatch/dictionary_test.go b/cmd/internal/anyblockbatch/dictionary_test.go new file mode 100644 index 0000000000..336cc34043 --- /dev/null +++ b/cmd/internal/anyblockbatch/dictionary_test.go @@ -0,0 +1,112 @@ +package anyblockbatch + +// dictionary_test.go pins the batch's §2f wiring: the property dictionary is +// a declaration source beside the type documents, and the used-key scan is +// what decides which properties the dictionary must name. + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// DictionaryFormats reads properties.json into the same table ScanFormats +// builds — keyed by STORED key, since that is what the dictionary spells +// (§2f), with the declared vocabulary carried along so newBatch pre-mints +// it in declaration order. +// +// How this can fail: shed Options on the way into FormatInfo (the +// vocabulary assertion goes red and a dictionary-declared select mints no +// options), or decode `format` literally so a bundled short-text entry +// mints longtext. +func TestDictionaryFormats_ReadsEntries(t *testing.T) { + // given + dir := t.TempDir() + path := filepath.Join(dir, "properties.json") + require.NoError(t, os.WriteFile(path, []byte(`{"version":2, + "installed":["tag"], + "properties":[ + {"property":"6a32d4856761631534b22f85","name":"Stage","format":"select", + "options":["Now",{"name":"Later","color":"blue"}]}]}`), 0o644)) + + // when + formats, defs, err := DictionaryFormats(path) + + // then + require.NoError(t, err) + require.Len(t, defs, 1) + fi, ok := formats["6a32d4856761631534b22f85"] + require.True(t, ok, "the table is keyed by the stored key the file spells") + assert.Equal(t, model.RelationFormat_status, fi.Format) + assert.Equal(t, "Stage", fi.Name) + require.Len(t, fi.Options, 2) + assert.Equal(t, "Now", fi.Options[0].Name) + assert.Equal(t, "blue", fi.Options[1].Color) +} + +// On a conflict the dictionary wins, and the conflict is SAID: a type entry +// disagreeing with the dictionary means the bundle contradicts itself, and +// silence would let whichever file loaded last decide. A type-declared +// vocabulary survives when the dictionary entry declares none. +// +// How this can fail: keep first-seen on conflict (the format assertion goes +// red), stop warning (the warned assertion), or overwrite the options +// unconditionally (the vocabulary assertion). +func TestMergeDictionaryFormats_DictionaryWins(t *testing.T) { + // given + scanned := map[string]FormatInfo{ + "budget": {Format: model.RelationFormat_longtext, FormatName: "text", Name: "Budget"}, + "stage": {Format: model.RelationFormat_status, FormatName: "select", Name: "Stage", + Options: []anyblockjson.OptionDefinition{{Name: "Now"}, {Name: "Later"}}}, + } + dict := map[string]FormatInfo{ + "budget": {Format: model.RelationFormat_number, FormatName: "number", Name: "Budget"}, + "stage": {Format: model.RelationFormat_status, FormatName: "select", Name: "Stage"}, + } + var warned []string + warn := func(format string, args ...any) { warned = append(warned, format) } + + // when + got := MergeDictionaryFormats(scanned, dict, warn) + + // then + assert.Equal(t, model.RelationFormat_number, got["budget"].Format, "the dictionary wins the conflict") + require.Len(t, warned, 1, "and the conflict is said") + assert.Equal(t, []anyblockjson.OptionDefinition{{Name: "Now"}, {Name: "Later"}}, got["stage"].Options, + "a type-declared vocabulary survives a dictionary entry that declares none") +} + +// UsedPropertyKeys resolves through the same chain every scan runs — the +// document's own property_internal_keys legend, the bundled table, verbatim — and +// counts the two slots that reference a property: a `properties` member and +// a property-definition entry. `id`/`type` are envelope facts and never +// count. +// +// How this can fail: key the set by the raw spelling (the legend-backed +// case resolves to `severity` instead of the stored bson and the dictionary +// misses the real key), or start counting envelope members. +func TestUsedPropertyKeys_ResolvesTheChain(t *testing.T) { + files := writeDocs(t, map[string]string{ + "objects/a.json": `{"version":2, + "property_internal_keys": {"severity": "6a32d4856761631534b22f85"}, + "properties": {"severity": "high", "due_date": "2026-01-01", "id": "a1", "type": "task"}}`, + "types/t.json": `{"version":2,"kind":"object_type","internal_key":"task", + "type_settings":{"property_definitions":[{"property":"assignee","format":"objects"}]}}`, + }) + + used, err := UsedPropertyKeys(files) + + require.NoError(t, err) + assert.True(t, used["6a32d4856761631534b22f85"], "the legend binds the spelling to the stored key") + assert.True(t, used["dueDate"], "the bundled table resolves the legacy slug spelling") + assert.True(t, used["assignee"], "a property-definition entry is a reference") + assert.False(t, used["severity"], "the spelling itself is not a key") + assert.False(t, used["id"], "envelope facts are not property references") + assert.False(t, used["type"], "envelope facts are not property references") +} diff --git a/cmd/internal/anyblockbatch/propertyterm.go b/cmd/internal/anyblockbatch/propertyterm.go new file mode 100644 index 0000000000..3c3e6f9c6e --- /dev/null +++ b/cmd/internal/anyblockbatch/propertyterm.go @@ -0,0 +1,93 @@ +package anyblockbatch + +// propertyterm.go — the one place the batch binds a PROPERTY term to a stored +// key. typeterm.go's twin, on the other namespace (SPEC.md §3: "one rule, +// stated once, covering both namespaces"). +// +// The envelope `key` a property document carries is the raw STORED key and is +// never translated (SPEC.md §2). Every property SLOT is translated: the +// `properties` map's own keys, and `type_settings.property_definitions[].property`, carry a term that +// resolves through the §3 chain — the document's own `property_internal_keys` legend, +// then the bundled name table (with its forgiving fold), then verbatim. +// +// The scans below build and compare tables the CONVERTER then reads by the +// resolved stored key: anyblockjson hands Options.ResolveFormat the output of +// importer.propertyKey, and Options.ResolveProperties a PropertyDefinition +// whose Key is likewise resolved. Keying an untranslated table and reading it +// translated fails both ways, and every failure here is silent: +// +// - fail-open, the reason this matters: a bundle whose `property_internal_keys` +// legend backs a slug misses the format table entirely. The value passes +// through as raw JSON — a date stays a string, a select mints no option, +// an objects reference is never relinked — and NO Relation object is +// minted for the property at all, so the space has a detail keyed to a +// relation that does not exist. CheckPropertyFormats compared raw against +// raw, so it agreed with itself and reported clean; +// - fail-open again, one level down: newBatch pre-mints the declared select +// vocabulary under the raw spelling, so the options land on a relation key +// nothing ever asks for, and the values that DO arrive mint a second, +// order-less set under the resolved key; +// - fail-closed: `properties` spells bundled keys by display name (§3), +// and `bundle.GetRelationFormat("Due date")` does not know that spelling — +// only `dueDate`. So a document written the canonical way was reported as +// having no declared format, which anyblockconvert turns into a hard error +// unless -lenient. Resolving first folds the name arm and the stored-key +// arm into one lookup; +// - and CheckSharedSelects grouped by spelling, so two documents naming one +// stored key two ways did not merge — exactly the collision the check +// exists to warn about. +// +// So every scan has to run the codec's own chain before it keys or compares. + +import "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + +// propertyLegend is the `property_internal_keys` envelope legend (§2), decoded +// alongside whatever slots a scan reads. Per-document, like typeLegend: the +// legend is the statement THIS document makes about ITS spellings, so it must +// be decoded from the same file as the slot it resolves. +type propertyLegend map[string]string + +// resolvePropertyTerm binds one property term to the stored key it names, +// running the §3 chain in the property namespace exactly as the codec runs it +// here. +// +// "Here" is a package-only reader: anyblockconvert and anyblockvalidate pass +// no anyblockjson.Options.Keys, so anyblockjson resolves every property slot +// through the document's legend and then BundledKeyVocabulary — which is the +// same two steps below (importer.propertyKey). The legend lookup is this +// file's only original line; the rest of the chain is one call into the +// package's own exported vocabulary, which answers the bundled name table +// when it knows the term, the forgiving fold when exactly one candidate +// remains, and hands the term back untouched otherwise (that pass-through IS +// chain step 5, verbatim). +// +// Unlike resolveTypeTerm this carries no reservation: `template` is a TYPE +// spelling, and the property namespace has no term whose meaning the envelope +// fixes. +// +// TestLintResolvesPropertyTermsLikeTheCodec pins the composition against what +// anyblockjson.Unmarshal actually stores, so the two cannot drift apart +// silently. +func resolvePropertyTerm(legend propertyLegend, term string) string { + if term == "" { + return "" + } + if key, ok := legend[term]; ok && key != "" { + return key + } + key, _ := anyblockjson.BundledKeyVocabulary{}.PropertyKey(term) + return key +} + +// resolvedPropertyNote annotates a finding whose slot spelling differs from +// the stored key it resolves to, so the reported term stays the one the author +// can find in the file while the reason names what the converter will actually +// look for. resolvedNote's property-namespace twin. +// An empty `resolved` is not a resolution, it is an unset field on a value +// some other caller built, so it annotates nothing. +func resolvedPropertyNote(term, resolved string) string { + if resolved == "" || resolved == term { + return "" + } + return " (the converter looks for the stored key " + quote(resolved) + ")" +} diff --git a/cmd/internal/anyblockbatch/propertyterm_test.go b/cmd/internal/anyblockbatch/propertyterm_test.go new file mode 100644 index 0000000000..3431fb8fb4 --- /dev/null +++ b/cmd/internal/anyblockbatch/propertyterm_test.go @@ -0,0 +1,386 @@ +package anyblockbatch + +// The scans read TRANSLATED property slots — every key of `properties`, and +// every `type_properties[].key` — and hand the result to the converter, which +// asks for it by the STORED key anyblockjson resolved that term to (§3). Every +// test below is a bundle where those two spellings differ, which is the only +// way the defect can show. +// +// Each test states which way it fails without the fix — fail-open (a silent +// miss the scan waves through, the case the scan exists to catch) or +// fail-closed (a valid bundle rejected, which anyblockconvert turns into a +// hard error unless -lenient). + +import ( + "encoding/json" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// customPropertyKey is a space-minted (bson) property key, the shape a real +// space gives a user-created relation — the stored key a legend entry binds a +// spelling to. `priority` is deliberately chosen as its spelling: it is ALSO +// a bundled key, so a scan that reads the spelling raw does not merely miss, +// it confidently resolves to the wrong relation. +const customPropertyKey = "6a32d4856761631534b22f85" + +// --- ScanFormats ----------------------------------------------------------- + +// Fail-OPEN, the one that costs real data: the table is keyed by the spelling +// while the converter reads it by the stored key, so the format is never +// found. The value passes through as raw JSON and no Relation object is minted +// for the property at all — and nothing anywhere says so. +func TestScanFormats_LegendBackedKeyIsStoredResolved(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/task.type.json": `{"version": 2, "kind": "object_type", "internal_key": "task", "id": "type-task", + "property_internal_keys": {"priority": "` + customPropertyKey + `"}, + "type_settings": {"property_definitions": [{"property": "priority", "name": "Priority", "format": "select", + "options": ["High", "Low"]}]}}`, + }) + formats, err := ScanFormats(files) + require.NoError(t, err) + + require.Contains(t, formats, customPropertyKey, + "the converter looks the format up by the stored key the legend names") + assert.NotContains(t, formats, "priority", + "and never by the spelling, which here is a DIFFERENT (bundled) relation") + assert.Equal(t, model.RelationFormat_status, formats[customPropertyKey].Format) + assert.Len(t, formats[customPropertyKey].Options, 2, + "the declared vocabulary travels with the stored key, or newBatch pre-mints it on a relation nothing uses") +} + +// The fallback display name stays the SPELLING: a legend exists because the +// stored key is a bson nobody wants to read, and this name is what +// mintRelation writes when the entry declares none. +func TestScanFormats_FallbackNameIsTheSpelling(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/task.type.json": `{"version": 2, "kind": "object_type", "internal_key": "task", "id": "type-task", + "property_internal_keys": {"priority": "` + customPropertyKey + `"}, + "type_settings": {"property_definitions": [{"property": "priority", "format": "text"}]}}`, + }) + formats, err := ScanFormats(files) + require.NoError(t, err) + assert.Equal(t, "priority", formats[customPropertyKey].Name) +} + +// A key nobody translates is its own address (chain step 5, verbatim), so +// resolution must leave the ordinary case exactly as it was. +func TestScanFormats_UntranslatedKeyIsUnchanged(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/task.type.json": `{"version": 2, "kind": "object_type", "internal_key": "task", "id": "type-task", + "type_settings": {"property_definitions": [{"property": "wikiStage", "format": "select", "options": ["A"]}]}}`, + }) + formats, err := ScanFormats(files) + require.NoError(t, err) + assert.Contains(t, formats, "wikiStage") +} + +// A bundled property declared by its legacy derived slug — the pre-v0.48 +// canonical spelling, which the forgiving fold still resolves (§3 chain step +// 4) — has to land on the bundled STORED key, or the batch mints a second +// relation beside the bundled one for the same property. +func TestScanFormats_LegacySlugResolvesToTheBundledKey(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/task.type.json": `{"version": 2, "kind": "object_type", "internal_key": "task", "id": "type-task", + "type_settings": {"property_definitions": [{"property": "due_date", "format": "date"}]}}`, + }) + formats, err := ScanFormats(files) + require.NoError(t, err) + assert.Contains(t, formats, "dueDate") + assert.NotContains(t, formats, "due_date") +} + +// The strongest statement of the contract, and the one that cannot go vacuous: +// it asks the CODEC which keys it wants formats for, through the real +// Options.ResolveFormat seam, and requires the table to answer every one. +// Without resolution the codec asks for the bson and the table holds +// "priority", so the resolver is called and misses. +func TestScanFormats_AnswersEveryKeyTheCodecAsksFor(t *testing.T) { + const object = `{"version": 2, "type": "task", "id": "obj-1", + "property_internal_keys": {"priority": "` + customPropertyKey + `"}, + "properties": {"priority": "High", "wikiStage": "Draft"}}` + files := writeDocs(t, map[string]string{ + "types/task.type.json": `{"version": 2, "kind": "object_type", "internal_key": "task", "id": "type-task", + "property_internal_keys": {"priority": "` + customPropertyKey + `"}, + "type_settings": {"property_definitions": [{"property": "priority", "format": "select", "options": ["High"]}, + {"property": "wikiStage", "format": "select", "options": ["Draft"]}]}}`, + "objects/one.json": object, + }) + formats, err := ScanFormats(files) + require.NoError(t, err) + + var asked []string + var missed []string + _, _, err = anyblockjson.Unmarshal([]byte(object), anyblockjson.Options{ + ResolveFormat: func(key domain.RelationKey) (model.RelationFormat, bool) { + asked = append(asked, string(key)) + fi, ok := formats[string(key)] + if !ok { + missed = append(missed, string(key)) + return 0, false + } + return fi.Format, true + }, + }) + require.NoError(t, err) + + sort.Strings(asked) + require.Equal(t, []string{customPropertyKey, "wikiStage"}, asked, + "the codec asks by the resolved stored key — if this list changes, the table's key must follow") + assert.Empty(t, missed, "every key the codec asks for must be in the table ScanFormats built") +} + +// --- CheckPropertyFormats -------------------------------------------------- + +// Fail-OPEN: the document's own legend says `priority` here is a space-minted +// relation, not the bundled one. Reading the spelling raw hit +// bundle.GetRelationFormat("priority"), which answers `number` for a relation +// this value has nothing to do with, and the check reported clean — while the +// converter resolved to the bson, found no format, and passed the value +// through raw with no Relation minted. +func TestCheckPropertyFormats_LegendBackedMissIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "objects/one.json": `{"version": 2, "type": "page", "id": "obj-1", + "property_internal_keys": {"priority": "` + customPropertyKey + `"}, + "properties": {"priority": "High"}}`, + }) + _, bundled := bundle.GetRelationFormat(domain.RelationKey("priority")) + require.NoError(t, bundled, + "the fixture only bites while `priority` really is a bundled key the raw check would accept") + + formats, err := ScanFormats(files) + require.NoError(t, err) + undeclared, err := CheckPropertyFormats(files, formats) + require.NoError(t, err) + + require.Len(t, undeclared, 1) + assert.Equal(t, "priority", undeclared[0].Key, "the spelling, so the author can find it") + assert.Equal(t, customPropertyKey, undeclared[0].Resolved) + assert.Contains(t, Report(undeclared), customPropertyKey, + "the report must name the key a type_properties entry has to end up on") +} + +// Fail-CLOSED: a pre-v0.48 document spells bundled keys as their derived api +// slugs, which the forgiving fold still resolves (§3 chain step 4) — and +// `bundle` is keyed by stored keys: it has never heard of `due_date`. So a +// document spelled that way was reported as having no declared format, which +// anyblockconvert turns into a hard error unless -lenient. +func TestCheckPropertyFormats_LegacySlugIsDeclared(t *testing.T) { + files := writeDocs(t, map[string]string{ + "objects/one.json": `{"version": 2, "type": "page", "id": "obj-1", + "properties": {"due_date": "2026-01-01T00:00:00Z", "plural_name": "x", "description": "hi"}}`, + }) + formats, err := ScanFormats(files) + require.NoError(t, err) + undeclared, err := CheckPropertyFormats(files, formats) + require.NoError(t, err) + assert.Empty(t, undeclared, "%s", Report(undeclared)) +} + +// Resolution must not turn every miss into a pass: a key that is neither +// bundled, nor legend-backed, nor declared is still the thing this check is +// for. +func TestCheckPropertyFormats_UnknownKeyIsStillReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "objects/one.json": `{"version": 2, "type": "page", "id": "obj-1", + "properties": {"wikiStage": "Draft"}}`, + }) + undeclared, err := CheckPropertyFormats(files, map[string]FormatInfo{}) + require.NoError(t, err) + require.Len(t, undeclared, 1) + assert.Equal(t, "wikiStage", undeclared[0].Key) + assert.Equal(t, "wikiStage", undeclared[0].Resolved) + assert.NotContains(t, Report(undeclared), "stored key", + "a term that is its own address needs no annotation") +} + +// The whole point of resolving on both sides: a legend-backed value IS +// declared once some type declares the same stored key, however either spells +// it. Here the type spells the bson outright and the object spells the +// legend's spelling. +func TestCheckPropertyFormats_DeclarationAndUseMayDisagreeOnSpelling(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/task.type.json": `{"version": 2, "kind": "object_type", "internal_key": "task", "id": "type-task", + "type_settings": {"property_definitions": [{"property": "` + customPropertyKey + `", "format": "select", "options": ["High"]}]}}`, + "objects/one.json": `{"version": 2, "type": "task", "id": "obj-1", + "property_internal_keys": {"priority": "` + customPropertyKey + `"}, + "properties": {"priority": "High"}}`, + }) + formats, err := ScanFormats(files) + require.NoError(t, err) + undeclared, err := CheckPropertyFormats(files, formats) + require.NoError(t, err) + assert.Empty(t, undeclared, "%s", Report(undeclared)) +} + +// `id` and `type` are lifted into the envelope, and the codec skips them on +// the SPELLING before any resolution — so this must too, whatever a legend +// says about those terms. +func TestCheckPropertyFormats_EnvelopeLiftedKeysAreSkipped(t *testing.T) { + files := writeDocs(t, map[string]string{ + "objects/one.json": `{"version": 2, "id": "obj-1", "properties": {"id": "x", "type": "y"}}`, + }) + undeclared, err := CheckPropertyFormats(files, map[string]FormatInfo{}) + require.NoError(t, err) + assert.Empty(t, undeclared) +} + +// --- CheckSharedSelects ---------------------------------------------------- + +// Fail-OPEN: two types declare ONE stored key, one through its legend and one +// verbatim. Their option pools merge in the space — each type's board grows +// the other's empty columns — and grouping by spelling reported nothing. This +// is the collision an author cannot see by reading the two files side by side, +// which is exactly why the check exists. +func TestCheckSharedSelects_MergesAcrossSpellings(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/a.type.json": `{"version": 2, "kind": "object_type", "internal_key": "typeA", "id": "type-a", + "property_internal_keys": {"stage": "` + customPropertyKey + `"}, + "type_settings": {"property_definitions": [{"property": "stage", "format": "select", "options": ["One"]}]}}`, + "types/b.type.json": `{"version": 2, "kind": "object_type", "internal_key": "typeB", "id": "type-b", + "type_settings": {"property_definitions": [{"property": "` + customPropertyKey + `", "format": "select", "options": ["Two"]}]}}`, + }) + shared, err := CheckSharedSelects(files) + require.NoError(t, err) + + require.Len(t, shared, 1) + assert.Equal(t, customPropertyKey, shared[0].Key, "the stored key is what the space shares") + assert.ElementsMatch(t, []string{"typeA", "typeB"}, shared[0].Types) +} + +// The mirror, and the guard against over-merging: one SPELLING, two stored +// keys, because each document's legend binds it elsewhere. Those are two +// properties with two option pools, and reporting them as shared would be a +// false alarm about a merge that does not happen. +func TestCheckSharedSelects_OneSpellingTwoKeysIsNotShared(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/a.type.json": `{"version": 2, "kind": "object_type", "internal_key": "typeA", "id": "type-a", + "property_internal_keys": {"stage": "` + customPropertyKey + `"}, + "type_settings": {"property_definitions": [{"property": "stage", "format": "select", "options": ["One"]}]}}`, + "types/b.type.json": `{"version": 2, "kind": "object_type", "internal_key": "typeB", "id": "type-b", + "property_internal_keys": {"stage": "69bbfc78877a91b1d12d1a7c"}, + "type_settings": {"property_definitions": [{"property": "stage", "format": "select", "options": ["Two"]}]}}`, + }) + shared, err := CheckSharedSelects(files) + require.NoError(t, err) + assert.Empty(t, shared, "%s", ReportSharedSelects(shared)) +} + +// The ordinary case still works: one spelling, no legend anywhere. +func TestCheckSharedSelects_PlainSharedKeyIsStillReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/a.type.json": `{"version": 2, "kind": "object_type", "internal_key": "typeA", "id": "type-a", + "type_settings": {"property_definitions": [{"property": "wikiStage", "format": "select", "options": ["One"]}]}}`, + "types/b.type.json": `{"version": 2, "kind": "object_type", "internal_key": "typeB", "id": "type-b", + "type_settings": {"property_definitions": [{"property": "wikiStage", "format": "select", "options": ["Two"]}]}}`, + }) + shared, err := CheckSharedSelects(files) + require.NoError(t, err) + require.Len(t, shared, 1) + assert.Equal(t, "wikiStage", shared[0].Key) +} + +// --- the authored targetObjectType probe ----------------------------------- + +// Fail-CLOSED: `target_object_type` is the legacy derived-slug spelling of +// the detail, still resolved by the fold (§3 chain step 4), so it reaches the +// snapshot and patchTemplateTarget keeps it — but the check probed the map +// for the STORED key alone, missed it, and rejected a bundle the converter +// wires perfectly. +func TestCheckTemplateTargets_AuthoredTargetInLegacySlugSpellingPasses(t *testing.T) { + const doc = `{"version": 2, "kind": "template", "type": "template", "template_for": "page", + "properties": {"target_object_type": "type-page"}}` + requireCodecStoresTargetObjectType(t, doc, true) + + files := writeDocs(t, map[string]string{"templates/article.json": doc}) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + assert.Empty(t, bad, "%s", ReportTemplateTargets(bad)) +} + +// Fail-OPEN, the mirror: the legend rebinds the `targetObjectType` spelling +// onto another relation, so the detail is NOT written and the template needs +// template_for wired after all. Probing the raw spelling took this as authored +// and skipped the document whole — the template then imports belonging to no +// type, unreported. +func TestCheckTemplateTargets_LegendMovesTheAuthoredTargetAway(t *testing.T) { + const doc = `{"version": 2, "kind": "template", "type": "template", "template_for": "page", + "property_internal_keys": {"targetObjectType": "` + customPropertyKey + `"}, + "properties": {"targetObjectType": "type-page"}}` + requireCodecStoresTargetObjectType(t, doc, false) + + files := writeDocs(t, map[string]string{"templates/article.json": doc}) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Equal(t, "page", bad[0].Target) + assert.Contains(t, bad[0].Reason, "bundled") +} + +// requireCodecStoresTargetObjectType pins the fixture to the codec's own +// verdict, so a test claiming "the converter has an authored target here" +// cannot quietly stop being true. +func requireCodecStoresTargetObjectType(t *testing.T, doc string, want bool) { + t.Helper() + require.NoError(t, anyblockjson.Validate([]byte(doc)), + "the fixture must be a document the codec accepts, or the check would never see it") + _, snap, err := anyblockjson.Unmarshal([]byte(doc), anyblockjson.Options{}) + require.NoError(t, err) + _, stored := snap.Details.GetFields()[string(bundle.RelationKeyTargetObjectType)] + require.Equal(t, want, stored) +} + +// --- anti-drift ------------------------------------------------------------ + +// The scan's chain is a composition — the document's legend, then the +// package's exported bundled vocabulary — and a composition can drift from the +// codec it models. This pins it against the key anyblockjson.Unmarshal +// actually STORES the value under, so a future change to the chain that the +// scans do not follow fails here rather than in a production bundle. +func TestLintResolvesPropertyTermsLikeTheCodec(t *testing.T) { + cases := []struct { + name string + legend string + term string + value string + }{ + {"verbatim custom key", ``, "wikiStage", `"Draft"`}, + {"legacy bundled slug, resolved by the fold", ``, "due_date", `"2026-01-01T00:00:00Z"`}, + {"bundled stored key that is nobody's spelling", ``, "dueDate", `"2026-01-01T00:00:00Z"`}, + {"legend-backed spelling", `"property_internal_keys": {"priority": "` + customPropertyKey + `"},`, "priority", `"High"`}, + {"legend outranks the bundled table", `"property_internal_keys": {"due_date": "` + customPropertyKey + `"},`, "due_date", `"High"`}, + {"identity entry for a shadow stored key", `"property_internal_keys": {"due_date": "due_date"},`, "due_date", `"High"`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + doc := `{"version": 2, "id": "obj-1", ` + c.legend + + `"properties": {"` + c.term + `": ` + c.value + `}}` + require.NoError(t, anyblockjson.Validate([]byte(doc))) + + _, snap, err := anyblockjson.Unmarshal([]byte(doc), anyblockjson.Options{}) + require.NoError(t, err) + + var probe struct { + PropertyKeys propertyLegend `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal([]byte(doc), &probe)) + + stored := make([]string, 0, 1) + for k := range snap.Details.GetFields() { + if k != "id" { + stored = append(stored, k) + } + } + require.Len(t, stored, 1, "the value must reach the snapshot, or the case proves nothing") + assert.Equal(t, stored[0], resolvePropertyTerm(probe.PropertyKeys, c.term)) + }) + } +} diff --git a/cmd/internal/anyblockbatch/scan.go b/cmd/internal/anyblockbatch/scan.go new file mode 100644 index 0000000000..292813cd37 --- /dev/null +++ b/cmd/internal/anyblockbatch/scan.go @@ -0,0 +1,1234 @@ +// Package anyblockbatch holds the cross-document concerns of an AnyBlock JSON +// bundle. pkg/lib/anyblockjson is deliberately one-document-at-a-time and +// leaves these to "the import wiring" (SPEC.md §3); both anyblockconvert and +// anyblockvalidate are that wiring, so the batch-wide property-format +// registry and the checks over it live here rather than in either command. +package anyblockbatch + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/compose" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// FormatByName is the SPEC.md §3 format vocabulary anyblockjson accepts in +// typeProperties entries and property values. It's small and spec-fixed but +// unexported inside pkg/lib/anyblockjson, so it's replicated here. +// "text" is the only text format (§3): shorttext has no name of its own, so +// a property declared here as text is minted as longtext. Properties whose +// stored format is shorttext are the bundled ones, which this tool never +// mints — anyblockjson resolves those by key. +var FormatByName = map[string]model.RelationFormat{ + "text": model.RelationFormat_longtext, + "number": model.RelationFormat_number, + "select": model.RelationFormat_status, + "multi_select": model.RelationFormat_tag, + "date": model.RelationFormat_date, + "files": model.RelationFormat_file, + "checkbox": model.RelationFormat_checkbox, + "url": model.RelationFormat_url, + "email": model.RelationFormat_email, + "phone": model.RelationFormat_phone, + "objects": model.RelationFormat_object, +} + +// FormatInfo is what the batch knows about a custom property key, gathered +// from every objectType document's typeProperties (§2a) before any document +// is actually converted. The map that holds it is keyed by the STORED +// property key each entry's identity term resolves to (propertyterm.go), which +// is what the converter reads it by. +type FormatInfo struct { + Format model.RelationFormat + FormatName string + Name string + // Options is the declared select vocabulary, in display order (§2a), + // each entry carrying the color it declares (empty = the batch picks). + Options []anyblockjson.OptionDefinition +} + +type typePropRaw struct { + // Property is the entry's document-facing spelling, InternalKey the + // stored id export writes beside it (SPEC.md §2e). The prescan names an + // entry by whichever it states, spelling first — the same order the + // codec's authoredKey runs. + Property string `json:"property"` + InternalKey string `json:"internal_key"` + Name string `json:"name"` + Format string `json:"format"` + // OptionDefinition decodes both §2a forms (a bare name, or an object with + // a color), so the prescan shares one decoder with anyblockjson rather + // than restating the union. + Options []anyblockjson.OptionDefinition `json:"options"` + ObjectTypes []string `json:"object_types"` +} + +// term is the identity this entry states: its `property` spelling, else its +// `internal_key` (SPEC.md §2e). Empty for a name-only entry, which the +// prescans skip — the codec derives the spelling from the name at import. +func (tp typePropRaw) term() string { + if tp.Property != "" { + return tp.Property + } + return tp.InternalKey +} + +// resolvedKey is the stored key the entry's identity names, by the codec's +// own rule (SPEC.md §2e): an `internal_key` verbatim — a stored id is its +// own address and never re-enters the resolution chain — and a `property` +// spelling through the legend-then-bundled-table chain like every slot. +func (tp typePropRaw) resolvedKey(legend propertyLegend) string { + if tp.Property == "" && tp.InternalKey != "" { + return tp.InternalKey + } + return resolvePropertyTerm(legend, tp.term()) +} + +// typeSettingsRaw is the slice of the §2a group the batch scans read: the +// property definitions moved off the document root into +// `type_settings.property_definitions` in v0.32, and a scanner still reading +// the root would silently see no declarations at all. +type typeSettingsRaw struct { + PropertyDefinitions *[]typePropRaw `json:"property_definitions"` +} + +func (ts *typeSettingsRaw) definitions() *[]typePropRaw { + if ts == nil { + return nil + } + return ts.PropertyDefinitions +} + +// typeSettingsDefs is the nil-safe value form for scanners that only range. +func typeSettingsDefs(ts *typeSettingsRaw) []typePropRaw { + if defs := ts.definitions(); defs != nil { + return *defs + } + return nil +} + +type prescanDoc struct { + PropertyKeys propertyLegend `json:"property_internal_keys"` + TypeSettings *typeSettingsRaw `json:"type_settings"` +} + +// ScanFormats reads every document's typeProperties (§2a) once, up front, to +// build a single batch-wide property-key -> format table. typeProperties is +// the only place a custom property's format is declared in the AnyBlock JSON +// format: plain §3 property values don't self-describe their format, so a +// "person" object referencing "team" only resolves correctly if some type +// document's typeProperties already declared "team"'s format — regardless of +// which file the directory walk visits first. +// +// The table is keyed by the STORED key each entry's identity term resolves to, +// because that is what the converter reads it by: anyblockjson hands +// Options.ResolveFormat the output of importer.propertyKey, never the +// spelling. `type_settings.property_definitions[].property` is a translated slot (§3), so it runs the +// chain — this document's own property_internal_keys legend, the bundled table, +// verbatim — first; see propertyterm.go for what keying it raw costs. +func ScanFormats(files []string) (map[string]FormatInfo, error) { + out := map[string]FormatInfo{} + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var doc prescanDoc + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + if doc.TypeSettings.definitions() == nil { + continue + } + for _, tp := range *doc.TypeSettings.definitions() { + if tp.term() == "" { + continue + } + key := tp.resolvedKey(doc.PropertyKeys) + format, ok := FormatByName[tp.Format] + if !ok { + // unrecognized or absent format: leave unresolved so the + // property value passes through raw (degrades the same way + // an unresolved format does everywhere else in this format, + // SPEC.md §3). + continue + } + // the fallback display name is the SPELLING, not the resolved + // key: a legend exists precisely because the stored key is a + // minted bson nobody wants to read, and this name is what + // mintRelation writes when the entry declares none. + name := tp.Name + if name == "" { + name = tp.term() + } + if existing, seen := out[key]; seen && len(tp.Options) == 0 && len(existing.Options) > 0 { + // a second type referencing the same property need not + // repeat its vocabulary + continue + } + if existing, seen := out[key]; seen && existing.Format != format { + fmt.Fprintf(os.Stderr, "warning: %s: property %q declared with conflicting formats (%s vs %s) — keeping the first seen\n", + f, tp.term()+resolvedPropertyNote(tp.term(), key), existing.FormatName, tp.Format) + continue + } + out[key] = FormatInfo{Format: format, FormatName: tp.Format, Name: name, Options: tp.Options} + } + } + return out, nil +} + +// Undeclared is a property value whose format nothing in the batch +// declares. +type Undeclared struct { + File string + // Key is the term as spelled in the document, so the author can find it. + Key string + // Resolved is the stored key that term binds to (§3) — what the + // converter will actually look up, and what a property-definition entry has + // to end up naming for the finding to go away. Equal to Key whenever the + // document spells the stored key itself. + Resolved string +} + +// CheckPropertyFormats finds property values whose format cannot be resolved. +// Formats do not travel with values in this format (§3): a value is decoded +// against the format declared for its key, and the only declaration site is +// some type document's typeProperties (§2a) — a dataview's properties[] is a +// per-view cache the converter never reads. When nothing declares a key, the +// value passes through as raw JSON and every format-driven conversion is +// silently skipped: a date stays an RFC-3339 string instead of unix seconds, +// a select mints no RelationOption, an objects value keeps an unresolved id, +// and no Relation object is created for the property at all. +// +// Nothing downstream reports this — the document validates and converts — so +// the batch has to catch it. +// +// A `properties` key is a translated slot (§3): it spells the property's +// display name, and binds to a stored key through this document's own +// property_internal_keys legend, then the bundled name table, then verbatim +// (propertyterm.go). Both lookups below take the RESOLVED key — the bundled +// one because `bundle` is keyed by stored keys and knows nothing of +// `"Due date"` (or of the legacy `due_date` slug the fold still accepts), the +// batch one because the converter reads that table by the resolved key too. +// Comparing spellings instead was wrong both ways: it reported every +// canonically-spelled bundled property as undeclared (a hard error in +// anyblockconvert), and waved through a legend-backed spelling whose stored +// key nothing declares. +func CheckPropertyFormats(files []string, formats map[string]FormatInfo) ([]Undeclared, error) { + var out []Undeclared + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var doc struct { + PropertyKeys propertyLegend `json:"property_internal_keys"` + Properties map[string]json.RawMessage `json:"properties"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + keys := make([]string, 0, len(doc.Properties)) + for k := range doc.Properties { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + // id and type are lifted into the envelope, not property values — + // and the codec skips them on the SPELLING, before any + // resolution (importer.build), so this must too + if k == "id" || k == "type" { + continue + } + key := resolvePropertyTerm(doc.PropertyKeys, k) + if _, err := bundle.GetRelationFormat(domain.RelationKey(key)); err == nil { + continue + } + if _, declared := formats[key]; declared { + continue + } + out = append(out, Undeclared{File: f, Key: k, Resolved: key}) + } + } + return out, nil +} + +// DiscoverJSONFiles walks root and returns every .json object document, +// sorted so a batch is deterministic regardless of directory order. Three +// populations are excluded: the two bundle-level documents — the index +// (§2c) and the property dictionary (§2f) describe the bundle rather than +// an object, have their own schemas, and would fail every object-level +// check — and every file the index's manifest binds as a BLOB +// (ManifestBlobPaths). A FAT bundle legitimately carries blobs that are +// themselves .json files (12 in the corpus, `file_ext == "json"`), and an +// extension test cannot tell them from an authored bare-.json document — +// the manifest `files` map is the authority on which bytes are content +// rather than documents (§2c, v0.47; the exporter's own documents +// additionally carry the .anyblock.json double extension, SPEC §15 #1, so +// for OUR bundles the collision cannot even arise by name). +func DiscoverJSONFiles(root string) ([]string, error) { + var files []string + var indexes []string + err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if filepath.Base(p) == anyblockjson.IndexFileName { + indexes = append(indexes, p) + return nil + } + if strings.HasSuffix(p, ".json") && filepath.Base(p) != anyblockjson.PropertiesFileName { + files = append(files, p) + } + return nil + }) + if err != nil { + return nil, err + } + blobs := ManifestBlobPaths(indexes) + kept := files[:0] + for _, f := range files { + if !blobs[f] { + kept = append(kept, f) + } + } + files = kept + sort.Strings(files) + return files, nil +} + +// ManifestBlobPaths reads each index's manifest `files` map and returns the +// absolute paths of every bound blob — the set a document discovery must +// skip. An index that does not parse contributes nothing: discovery must +// not fail on a broken index (the index's own validation reports that), it +// just cannot exclude blobs the broken manifest would have named. +func ManifestBlobPaths(indexPaths []string) map[string]bool { + out := map[string]bool{} + for _, idxPath := range indexPaths { + data, err := os.ReadFile(idxPath) + if err != nil { + continue + } + idx, err := anyblockjson.UnmarshalIndex(data) + if err != nil || idx.Manifest == nil { + continue + } + dir := filepath.Dir(idxPath) + for _, blobPath := range idx.Manifest.Files { + out[filepath.Join(dir, filepath.FromSlash(blobPath))] = true + } + } + return out +} + +// Report renders undeclared properties as one line each, most useful first. +// A term whose legend moves it names the stored key too: the entry that fixes +// the finding has to resolve to THAT key, which the spelling alone does not +// say. +func Report(us []Undeclared) string { + var b strings.Builder + for _, u := range us { + // BOTH homes, deliberately: §2f gave a format two places it can be + // declared, and naming only one sends an author who wrote the other + // to undo it. + fmt.Fprintf(&b, " %s: property %q has no declared format%s — declare it in properties.json, "+ + "or in some type's type_settings.property_definitions\n", + u.File, u.Key, resolvedPropertyNote(u.Key, u.Resolved)) + } + return b.String() +} + +// TypeIds maps each type key the bundle defines to the id its document +// carries. A property targeting one of those types must reference it by that +// id, so the importer relinks it with every other reference in the batch. +func TypeIds(files []string) (map[string]string, error) { + out := map[string]string{} + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var probe struct { + Kind string `json:"kind"` + Key string `json:"internal_key"` + Id string `json:"id"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + if probe.Kind == "object_type" && probe.Key != "" { + // an id-less type still has to be registered: without it, + // objectTypeIds cannot tell "defined here, but unaddressable" + // from "bundled", and silently emits a bundled url for a type + // that is not bundled + out[probe.Key] = probe.Id + } + } + return out, nil +} + +// OrderTypesFirst puts type documents ahead of everything else, preserving +// relative order within each group. A bundle's types declare the schema its +// objects reference — property formats, select vocabularies, the relations +// that must exist — so converting them first means every declaration is in +// place before the first usage of it. Alphabetically the walk yields +// chats/, objects/, pages/, types/, i.e. exactly backwards. +func OrderTypesFirst(files []string) ([]string, error) { + var types, rest []string + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var probe struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + if probe.Kind == "object_type" { + types = append(types, f) + } else { + rest = append(rest, f) + } + } + return append(types, rest...), nil +} + +// SharedSelect is a select property declared by more than one type. +type SharedSelect struct { + // Key is the STORED property key the declarations share — what the space + // keys the one option pool by, and what makes them the same property even + // when two documents spell it differently (§3). + Key string + Types []string // type keys, in declaration order +} + +// CheckSharedSelects finds select/multiSelect properties declared by more +// than one type. Properties are space-wide, not per-type: two types sharing +// one select share one option pool, so their vocabularies merge into a +// single dropdown and each type's board grows the other's empty columns. +// +// That is right for a property whose value set is genuinely common — `tag` +// exists to be shared — and wrong for the lifecycle selects every schema +// reaches for, where "Status" on a Task and "Status" on a Project name +// different things. Splitting them (taskStatus / projectStatus, labelled +// "Task status") keeps each vocabulary clean. +// +// Reported rather than rejected: only the author knows whether the union is +// the point. +// +// Grouped by the STORED key each `key` term resolves to (§3, propertyterm.go), +// not by its spelling: what merges two vocabularies is naming one property, +// and two documents naming it two ways — one through its property_internal_keys +// legend, one verbatim — merge exactly as hard as two spelling it alike. +// Grouping by spelling missed precisely the collision an author cannot see by +// reading the files side by side. +func CheckSharedSelects(files []string) ([]SharedSelect, error) { + type decl struct { + types []string + seen map[string]bool + } + byKey := map[string]*decl{} + var order []string + + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var doc struct { + Kind string `json:"kind"` + Key string `json:"internal_key"` + PropertyKeys propertyLegend `json:"property_internal_keys"` + TypeSettings *typeSettingsRaw `json:"type_settings"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + if doc.Kind != "object_type" { + continue + } + // the envelope `internal_key` is the raw stored key and is never translated + // (§2), so it is the right label for the type as it stands + typeName := doc.Key + if typeName == "" { + typeName = filepath.Base(f) + } + for _, tp := range typeSettingsDefs(doc.TypeSettings) { + if tp.Format != "select" && tp.Format != "multi_select" { + continue + } + key := tp.resolvedKey(doc.PropertyKeys) + d, ok := byKey[key] + if !ok { + d = &decl{seen: map[string]bool{}} + byKey[key] = d + order = append(order, key) + } + if !d.seen[typeName] { + d.seen[typeName] = true + d.types = append(d.types, typeName) + } + } + } + + var out []SharedSelect + for _, key := range order { + if d := byKey[key]; len(d.types) > 1 { + out = append(out, SharedSelect{Key: key, Types: d.types}) + } + } + return out, nil +} + +// ReportSharedSelects renders shared selects as one line each. +func ReportSharedSelects(ss []SharedSelect) string { + var b strings.Builder + for _, s := range ss { + fmt.Fprintf(&b, " property %q is a select shared by %d types (%s) — one option pool space-wide, so their vocabularies merge; split per type unless the union is the point\n", + s.Key, len(s.Types), strings.Join(s.Types, ", ")) + } + return b.String() +} + +// CheckTargetTypes finds objectTypes entries that cannot resolve to anything. +// A target key must name either a bundled type or a type this bundle defines +// *and* gives an id — a document without an id has nothing for a reference to +// point at, and the reference would otherwise be emitted as a bundled url for +// a type that is not bundled: valid, converted, and dangling. +// +// object_types is a translated type-key slot (§2a), so each entry runs the §3 +// chain — this document's own type_internal_keys legend, the bundled table, verbatim — +// before anything is looked up. typeIds is keyed by the untranslated envelope +// key (§2), and matching a term against it raw is both a fail-closed and a +// fail-open bug; see typeterm.go. +// +// The arms are ordered the way batch.objectTypeIds orders them — LOCAL first, +// bundled only as the fallthrough — because a lint that asks the questions in +// a different order than the code it lints answers a different question. +// Checking bundled first short-circuited a bundle that defines an +// `object_type` document with a bundled key and no `id`: the converter takes +// the local arm, finds the empty id, and appends an EMPTY STRING to +// relationFormatObjectTypes, which names nothing and is invisible in every +// UI — while the lint saw `page` in the bundle table and reported clean. +func CheckTargetTypes(files []string, typeIds map[string]string) ([]BadTarget, error) { + var out []BadTarget + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var doc struct { + TypeKeys typeLegend `json:"type_internal_keys"` + TypeSettings *typeSettingsRaw `json:"type_settings"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + for _, tp := range typeSettingsDefs(doc.TypeSettings) { + for _, target := range tp.ObjectTypes { + key := resolveTypeTerm(doc.TypeKeys, target) + id, defined := typeIds[key] + note := resolvedNote(target, key) + shadows := "" + if bundle.HasObjectTypeByKey(domain.TypeKey(key)) { + shadows = " (this bundle defines a document with that key, and the converter prefers it over the bundled type of the same name)" + } + switch { + case defined && id == "": + out = append(out, BadTarget{File: f, Property: tp.term(), Target: target, Reason: "that type is defined here but its document carries no \"id\", so nothing can reference it" + shadows + note}) + case defined: + // a document in this bundle, with an id to point at + case bundle.HasObjectTypeByKey(domain.TypeKey(key)): + // bundled, and this bundle does not shadow it + default: + out = append(out, BadTarget{File: f, Property: tp.term(), Target: target, Reason: "no such type: not bundled, and not defined by this bundle" + note}) + } + } + } + } + return out, nil +} + +// BadTarget is an objectTypes entry that cannot resolve to a type. +type BadTarget struct { + File string + Property string + Target string + Reason string +} + +// BadTemplateTarget is a template whose target type cannot be wired. +type BadTemplateTarget struct { + File string + Target string // the templateFor key, empty when the document has none + Reason string +} + +// CheckTemplateTargets finds templates that would import belonging to no type. +// A type's templates are found by querying the targetObjectType detail +// (core/block/template/templateimpl.queryTemplatesByType), and that detail +// holds the target type's *object id* — so templateFor has to name a type this +// bundle defines and gives an id, exactly like an objectTypes target. +// +// Unlike an objectTypes target, a bundled type key is not good enough: a +// bundled url in an object-format detail is passed through untouched on import +// (common.UpdateObjectIDsInRelations -> isBundledObjects), so "_otpage" would +// survive as a literal and match no type in the space. Real exports have a +// type document for every type their templates target, bundled ones included +// (util/builtinobjects/data/*.zip), and so must a bundle here. +// +// Nothing downstream reports any of this — the document validates, converts +// and imports; the template simply never appears under a type — so the batch +// has to catch it. +// +// The gate is `kind`, and only `kind` (§2, v0.22). Whether a document IS a +// template is not a fact about its type term: it used to be read off the +// stored key that term resolved to, through this document's own legend and the +// bundled table, and that made the same field answer two unrelated questions. +// `template_for` is still translated (§3, typeterm.go), because it names a +// type and has to be matched against the bundle's type ids. +func CheckTemplateTargets(files []string, typeIds map[string]string) ([]BadTemplateTarget, error) { + var out []BadTemplateTarget + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var doc struct { + Kind string `json:"kind"` + Type string `json:"type"` + TemplateFor string `json:"template_for"` + TypeKeys typeLegend `json:"type_internal_keys"` + PropertyKeys propertyLegend `json:"property_internal_keys"` + Properties map[string]json.RawMessage `json:"properties"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + if doc.Kind != templateKind { + continue + } + if authoredTargetObjectType(doc.PropertyKeys, doc.Properties) { + // an explicit id (what a round-tripped export carries) is what the + // converter keeps, whatever templateFor says + continue + } + if doc.TemplateFor == "" { + out = append(out, BadTemplateTarget{File: f, + Reason: `no "template_for": the template would belong to no type, and no type would list it`}) + continue + } + key := resolveTypeTerm(doc.TypeKeys, doc.TemplateFor) + id, defined := typeIds[key] + note := resolvedNote(doc.TemplateFor, key) + switch { + case !defined && bundle.HasObjectTypeByKey(domain.TypeKey(key)): + out = append(out, BadTemplateTarget{File: f, Target: doc.TemplateFor, + Reason: `that type is bundled, but a template's target must be a document in this bundle: a bundled url is never relinked on import, so it would match no type — add an object_type document with this key and an "id"` + note}) + case !defined: + out = append(out, BadTemplateTarget{File: f, Target: doc.TemplateFor, + Reason: "no such type: not bundled, and not defined by this bundle" + note}) + case id == "": + out = append(out, BadTemplateTarget{File: f, Target: doc.TemplateFor, + Reason: `that type is defined here but its document carries no "id", so nothing can reference it` + note}) + } + } + return out, nil +} + +// authoredTargetObjectType reports whether the document writes the +// targetObjectType detail itself — the value patchTemplateTarget keeps +// whatever template_for says. +// +// The converter reads that detail off the CONVERTED snapshot, i.e. under the +// stored key, so the question here is which SPELLING lands on it — a +// translated property slot, resolved through this document's own +// property_internal_keys legend, then the bundled name table and its fold, +// then verbatim (§3). Probing the map for the stored key alone was wrong both +// ways: a document spelling the detail by its display name — or by the legacy +// `target_object_type` slug, which the fold still resolves — reached the +// detail while this check missed it, reporting a template the converter wires +// perfectly (a hard error in anyblockconvert); and a legend rebinding the `targetObjectType` +// spelling onto some other key means the detail is NOT written, which this +// check took as authored and skipped — the template then imports belonging to +// no type, unreported, which is the whole point of the check. +func authoredTargetObjectType(legend propertyLegend, props map[string]json.RawMessage) bool { + for slug := range props { + if resolvePropertyTerm(legend, slug) == string(bundle.RelationKeyTargetObjectType) { + return true + } + } + return false +} + +// ReportTemplateTargets renders unwirable template targets, one per line. +func ReportTemplateTargets(bs []BadTemplateTarget) string { + var b strings.Builder + for _, t := range bs { + if t.Target == "" { + fmt.Fprintf(&b, " %s: %s\n", t.File, t.Reason) + continue + } + fmt.Fprintf(&b, " %s: template_for %q — %s\n", t.File, t.Target, t.Reason) + } + return b.String() +} + +// ReportTargets renders unresolvable target types, one per line. +func ReportTargets(bs []BadTarget) string { + var b strings.Builder + for _, t := range bs { + fmt.Fprintf(&b, " %s: property %q targets %q — %s\n", t.File, t.Property, t.Target, t.Reason) + } + return b.String() +} + +// IndexPath returns the bundle index's path, and whether it exists. +func IndexPath(root string) (string, bool) { + p := filepath.Join(root, anyblockjson.IndexFileName) + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p, true + } + return "", false +} + +// PropertiesPath returns the bundle's property dictionary path (§2f), and +// whether it exists — IndexPath's rule, at IndexPath's location: both +// bundle-level documents live at the bundle root. +func PropertiesPath(root string) (string, bool) { + p := filepath.Join(root, anyblockjson.PropertiesFileName) + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p, true + } + return "", false +} + +// CheckBundleIds finds documents that claim an id in the platform's reserved +// `_` namespace (§1). Nothing a bundle ships may live there. +// +// It is not a tidiness rule. The pb importer resolves a link target through +// the bundle's own id map FIRST (common.UpdateLinksToObjects) and only then +// asks widget.IsPredefinedWidgetTargetId, so an object whose id equals a +// reserved listing captures every widget that meant the listing — silently, +// with no finding from any check and no error at import. Keeping the two +// namespaces disjoint by a prefix is what makes that unrepresentable, and it +// stays true as listings are added, which a per-word reservation does not. +// +// The same prefix also covers the bundled objects (`_otpage`, `_brdue_date`) +// and the platform's other addresses (`_missing_object`, `_participant_…`): a +// bundle minting one of those ids would collide with the object the space +// already has. +// +// It also covers the four listings' and two screens' BARE spellings, which is +// the half a prefix rule alone would miss — see IsReservedBundleId. +func CheckBundleIds(files []string) ([]BadTarget, error) { + var out []BadTarget + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var probe struct { + Id string `json:"id"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + if !anyblockjson.IsReservedBundleId(probe.Id) { + continue + } + var reason string + switch { + case anyblockjson.IsReservedWidgetTarget(probe.Id): + reason = fmt.Sprintf("this is a reserved index.json widget listing (the inventory is %s), "+ + "not an id a bundle may mint — the importer resolves a widget target through the bundle's "+ + "own ids first, so this object would silently capture every widget naming the listing", + strings.Join(anyblockjson.ReservedWidgetTargets(), ", ")) + case anyblockjson.IsReservedHomepage(probe.Id): + reason = "this is a reserved index.json homepage screen, not an id a bundle may mint" + case anyblockjson.IsPlatformId(probe.Id): + reason = "an id may not begin with \"_\": that prefix is the platform's own address space " + + "(bundled types and relations, participants, _missing_object) and the reserved index.json " + + "listings and screens — an object minting one of those ids shadows it, and a widget or " + + "homepage naming it silently gets this object instead of the built-in" + default: + reason = "this is what a reserved index.json listing or screen is called on the wire " + + "(widget.IsPredefinedWidgetTargetId, setWorkspaceSettings), which is where a widget " + + "target lands after the format's leading \"_\" is translated off — so an object with " + + "this id silently captures the built-in that the whole bundle, not just this document, " + + "may want to name" + } + out = append(out, BadTarget{File: f, Property: "id", Target: probe.Id, Reason: reason}) + } + return out, nil +} + +// CheckIndexTargets finds index.json references that name nothing the bundle +// defines. Reserved homepages and reserved widget targets name built-in +// screens and listings, so they are not expected to resolve. +// +// A widget target is checked harder than the others, because it is the only +// reference in the format whose failure is silent: an unresolvable link target +// becomes addr.MissingObject (common.handleLinkBlock), and WidgetObject.Init +// then removes the link and its wrapper. No error reaches the import result — +// the widget simply is not there. +func CheckIndexTargets(idx *anyblockjson.Index, files []string) []BadTarget { + ids := map[string]bool{} + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + continue + } + var probe struct { + Id string `json:"id"` + } + if json.Unmarshal(data, &probe) == nil && probe.Id != "" { + ids[probe.Id] = true + } + } + + var out []BadTarget + if e := idx.Entrypoint; e != "" && !ids[e] { + out = append(out, BadTarget{ + File: anyblockjson.IndexFileName, Property: "entrypoint", Target: e, + Reason: "no object with that id in the bundle — the install would open nothing", + }) + } + if h := idx.Homepage; h != "" && !anyblockjson.IsReservedHomepage(h) && !ids[h] { + out = append(out, BadTarget{ + File: anyblockjson.IndexFileName, Property: "homepage", Target: h, + Reason: "no object with that id in the bundle (and it is not a reserved homepage)", + }) + } + for i, w := range idx.Widgets { + switch { + case anyblockjson.IsReservedWidgetTarget(w.Target): + if anyblockjson.IsImportableWidgetTarget(w.Target) { + continue + } + // unreachable while the two inventories agree — every reserved + // listing is importable today — but the day one is added to the + // format before the importer learns it, this is the check that + // keeps the failure loud instead of a widget silently gone + out = append(out, BadTarget{ + File: anyblockjson.IndexFileName, Property: fmt.Sprintf("widgets[%d]", i), Target: w.Target, + Reason: "a reserved listing the importer does not recognise " + + "(widget.IsPredefinedWidgetTargetId), so this link is rewritten to " + + "_missing_object and the widget is dropped without an error", + }) + case !ids[w.Target]: + out = append(out, BadTarget{ + File: anyblockjson.IndexFileName, Property: fmt.Sprintf("widgets[%d]", i), Target: w.Target, + Reason: "no object with that id in the bundle (and it is not a reserved widget target)", + }) + } + } + // the auto-widget ledger's entries are target-shaped references too; an + // entry naming nothing is not silent-lossy like a widget (nothing renders + // it), but it points a restored client at an object that is not there + for i, target := range idx.AutoWidgetTargets { + if anyblockjson.IsReservedWidgetTarget(target) || ids[target] { + continue + } + out = append(out, BadTarget{ + File: anyblockjson.IndexFileName, Property: fmt.Sprintf("auto_widget_targets[%d]", i), Target: target, + Reason: "no object with that id in the bundle (and it is not a reserved widget target)", + }) + } + return out +} + +// ObjectNames maps every id the bundle defines to that object's name. The +// installer resolves a space icon by image name rather than by id +// (builtinobjects.getNewAvatarId), so the wiring needs this to turn an +// index.json iconImage reference into the name the profile carries. +func ObjectNames(files []string) (map[string]string, error) { + out := map[string]string{} + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var probe struct { + Id string `json:"id"` + Properties struct { + Name string `json:"name"` + } `json:"properties"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + if probe.Id != "" { + out[probe.Id] = probe.Properties.Name + } + } + return out, nil +} + +// DictionaryFormats reads the bundle's property dictionary (§2f) into the +// same batch-wide table ScanFormats builds, plus the full definitions for +// pre-minting. The dictionary is where an author declares a property WITHOUT +// writing a relation document — the same vocabulary as a type's +// property-definition entry, one file for the whole bundle — so its entries +// join the format registry exactly as type-declared ones do, and the caller +// merges with the dictionary as the authority: the dictionary is the +// property's one home (§2e), a type entry its per-type use. +// +// Keys arrive as STORED keys (§2f), so unlike ScanFormats there is no legend +// chain to run: the table is keyed by what the file spells. +func DictionaryFormats(path string) (map[string]FormatInfo, []anyblockjson.PropertyDefinition, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("read %s: %w", path, err) + } + dict, err := anyblockjson.UnmarshalPropertyDictionary(data) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", anyblockjson.PropertiesFileName, err) + } + out := map[string]FormatInfo{} + for _, def := range dict.Properties { + out[string(def.Key)] = FormatInfo{ + Format: def.Format, + FormatName: anyblockjson.FormatName(def.Format), + Name: def.Name, + Options: def.Options, + } + } + return out, dict.Properties, nil +} + +// MergeDictionaryFormats folds the dictionary's table into the type-scanned +// one, dictionary winning on a conflict — with the conflict SAID, because a +// type entry disagreeing with the dictionary means the bundle contradicts +// itself and silence would let whichever file loaded last decide. A type +// entry that declares a vocabulary the dictionary entry omits keeps that +// vocabulary: the dictionary defines the property, the type may still be the +// place its options were spelled out. +func MergeDictionaryFormats(scanned, dict map[string]FormatInfo, warn func(format string, args ...any)) map[string]FormatInfo { + for key, d := range dict { + // a BUNDLED key's definition is the code table's, in every space and + // offline — the dictionary cannot override it, and the + // tools do not pretend to: the format table below is consulted only + // for keys the bundled table does not answer for. Said out loud, + // because the entry is otherwise accepted in silence and the author + // is left believing a redefinition took effect. The same run then + // warns from the bundled table about a value the entry declared + // legal, which reads as the tool contradicting itself. + if rel, err := bundle.GetRelation(domain.RelationKey(key)); err == nil && rel != nil { + if d.Format != rel.Format { + warn("property %q is BUNDLED: the dictionary says %s, the bundled table says %s, "+ + "and the bundled table wins here and in every reader — "+ + "the entry documents the property, it cannot redefine it", + key, d.FormatName, anyblockjson.FormatName(rel.Format)) + } + continue + } + existing, seen := scanned[key] + if seen && existing.Format != d.Format { + warn("property %q: a type declares %s but the dictionary says %s — the dictionary wins, because "+ + "the dictionary is the file that defines the property", + key, existing.FormatName, d.FormatName) + } + if seen && len(d.Options) == 0 && len(existing.Options) > 0 { + d.Options = existing.Options + } + scanned[key] = d + } + return scanned +} + +// UsedPropertyKeys reports every STORED property key the bundle's documents +// reference — the population the dictionary's `properties` list names (§2f, +// used-only). Two slots count as a reference, resolved through the same +// chain every scan here runs (a document's own property_internal_keys legend, the +// bundled table, verbatim): a `properties` member on any document, and a +// `type_settings.property_definitions[].property`. A dataview's column list is +// deliberately NOT one — it is a per-view cache carrying its own inline +// format (§6.2), so a key that appears there and nowhere else gives a +// reader nothing to look up. +// The scan itself is compose.UsedPropertyKeysFromBytes — promoted there so +// production composition (which cannot re-read a zip entry and must scan the +// marshalled bytes before the write) and this file-level convenience run one +// implementation rather than two that drift. +func UsedPropertyKeys(files []string) (map[string]bool, error) { + out := map[string]bool{} + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + used, err := compose.UsedPropertyKeysFromBytes(data) + if err != nil { + return nil, fmt.Errorf("scan %s: %w", f, err) + } + for key := range used { + out[key] = true + } + } + return out, nil +} + +// CheckManifestFiles finds manifest `files` entries that do not bind +// (§2c, v0.47). The map is the bundle's only authoritative binding between +// a file document and its bytes — the document itself carries no path — so +// an entry that fails here fails silently at import: the file object +// arrives and its content does not. +// +// Three refusals, each on what the manifest STATES (a bundle whose map +// omits a file document is like a bundle with no manifest — walked, not +// refused — because whether an absent blob is intended is the thin-bundle +// marker's future question, SPEC §15 #20): +// +// - the key names no document in the bundle — the binding is for nothing; +// - the path escapes the bundle root — every manifest path is relative to +// index.json, and one that climbs out points at bytes the archive does +// not carry; +// - no file exists at the path — the blob the entry promises is missing. +func CheckManifestFiles(idx *anyblockjson.Index, indexDir string, files []string) []BadTarget { + if idx == nil || idx.Manifest == nil || len(idx.Manifest.Files) == 0 { + return nil + } + ids := map[string]bool{} + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + continue + } + var probe struct { + Id string `json:"id"` + } + if json.Unmarshal(data, &probe) == nil && probe.Id != "" { + ids[probe.Id] = true + } + } + keys := make([]string, 0, len(idx.Manifest.Files)) + for key := range idx.Manifest.Files { + keys = append(keys, key) + } + sort.Strings(keys) + var out []BadTarget + for _, key := range keys { + blobPath := idx.Manifest.Files[key] + prop := fmt.Sprintf("manifest.files[%q]", key) + if !ids[key] { + out = append(out, BadTarget{ + File: anyblockjson.IndexFileName, Property: prop, Target: key, + Reason: "no document with that id in the bundle — this binds bytes to nothing", + }) + continue + } + if filepath.IsAbs(blobPath) || escapesDir(blobPath) { + out = append(out, BadTarget{ + File: anyblockjson.IndexFileName, Property: prop, Target: blobPath, + Reason: "the path points outside the bundle — manifest paths are relative to index.json and stay inside it", + }) + continue + } + if st, err := os.Stat(filepath.Join(indexDir, filepath.FromSlash(blobPath))); err != nil || st.IsDir() { + out = append(out, BadTarget{ + File: anyblockjson.IndexFileName, Property: prop, Target: blobPath, + Reason: "no file at that path — the blob this entry promises is missing from the bundle", + }) + } + } + return out +} + +// UnboundFileDocuments lists the file documents a PRESENT manifest `files` +// map does not bind — bytes that did not travel (§2c). Warning-grade, not a +// refusal, and the gate on the map's presence is the point: a bundle with +// no map at all is a metadata-only export (files off, or pre-v0.47), which +// is a legitimate mode — but a bundle that binds SOME file documents and +// not others is the signature of a partially failed export (the exporter +// writes the document and omits the binding when a blob cannot be +// streamed), and silence there leaves the reader believing the bytes are +// somewhere. +func UnboundFileDocuments(idx *anyblockjson.Index, files []string) []string { + if idx == nil || idx.Manifest == nil || len(idx.Manifest.Files) == 0 { + return nil + } + var out []string + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + continue + } + var probe struct { + Id string `json:"id"` + Kind string `json:"kind"` + } + if json.Unmarshal(data, &probe) != nil { + continue + } + if probe.Kind != "file_object" && probe.Kind != "file" { + continue + } + if probe.Id == "" || idx.Manifest.Files[probe.Id] != "" { + continue + } + out = append(out, probe.Id) + } + sort.Strings(out) + return out +} + +// escapesDir reports a relative path that climbs above its base. +func escapesDir(p string) bool { + clean := filepath.Clean(filepath.FromSlash(p)) + return clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) +} + +// CheckViewProperties reports a view slot — a filter leaf, a sort or a +// column — naming a property nothing in the bundle can resolve. +// +// This is the silent class, and the one a per-document reader cannot judge. +// A filter on a property that exists nowhere does not fail: it matches +// nothing, so the view shows everything it was meant to narrow. Asked to +// "also exclude archived items", six of nine agents produced exactly that — +// a document that validates, imports and round-trips byte-stably, with a new +// filter that is a no-op. +// +// The codec cannot raise it. A custom property whose stored key is already a +// legal spelling — `aroma_notes`, and 112 more in a 77-space corpus — binds +// no legend entry, because the spelling IS the key. Inside one document that +// is indistinguishable from a typo. Only a reader holding the whole bundle +// can tell them apart, and that is this function: `declared` carries every +// key the bundle declares anywhere — the property dictionary, each type's +// property definitions, and every document's legend. +// +// No real export trips it: 1,517 filter leaves across the corpus, none +// unresolved. +func CheckViewProperties(files []string, declared map[string]bool) ([]BadViewProperty, error) { + var out []BadViewProperty + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read %s: %w", f, err) + } + var doc struct { + PropertyKeys propertyLegend `json:"property_internal_keys"` + Blocks []struct { + Views []struct { + Id string `json:"id"` + Filters []json.RawMessage `json:"filters"` + Sorts []viewPropSlot `json:"sorts"` + Columns []viewPropSlot `json:"columns"` + } `json:"views"` + Properties []struct { + Property string `json:"property"` + } `json:"properties"` + } `json:"blocks"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("parse %s: %w", f, err) + } + for _, b := range doc.Blocks { + // a dataview's own properties list declares what its views may + // name, whether or not anything else in the bundle does + local := map[string]bool{} + for _, p := range b.Properties { + local[p.Property] = true + } + for _, v := range b.Views { + at := v.Id + if at == "" { + at = "(unnamed view)" + } + judge := func(prop, what string) { + if prop == "" || local[prop] { + return + } + key := resolvePropertyTerm(doc.PropertyKeys, prop) + if declared[key] || declared[prop] || bundle.HasRelation(domain.RelationKey(key)) { + return + } + out = append(out, BadViewProperty{ + File: f, View: at, Slot: what, Property: prop, + }) + } + for _, raw := range v.Filters { + for _, prop := range filterLeafProperties(raw) { + judge(prop, "filter") + } + } + for _, s := range v.Sorts { + judge(s.Property, "sort") + } + for _, c := range v.Columns { + judge(c.Property, "column") + } + } + } + } + return out, nil +} + +type viewPropSlot struct { + Property string `json:"property"` +} + +// BadViewProperty is a view slot naming a property nothing in the bundle +// declares. +type BadViewProperty struct { + File string + View string // the view's id + Slot string // "filter", "sort" or "column" + Property string +} + +// ReportViewProperties renders the findings, one line each, saying what the +// slot does instead of what it was meant to do. +func ReportViewProperties(bs []BadViewProperty) string { + effect := map[string]string{ + "filter": "narrows nothing, so the view shows everything", + "sort": "leaves the order untouched", + "column": "stays empty", + } + var b strings.Builder + for _, x := range bs { + what := effect[x.Slot] + if what == "" { + what = "does nothing" + } + fmt.Fprintf(&b, " %s\n view %q: the %s on %q %s — no document, no type and no property "+ + "dictionary declares that property, and it is not a bundled one. Nothing reports it at import.\n", + x.File, x.View, x.Slot, x.Property, what) + } + return b.String() +} + +// filterLeafProperties returns the properties a filter node names, descending +// through group nodes, which name none by design. +func filterLeafProperties(raw json.RawMessage) []string { + var node struct { + Property string `json:"property"` + Filters []json.RawMessage `json:"filters"` + } + if json.Unmarshal(raw, &node) != nil { + return nil + } + if len(node.Filters) > 0 { + var out []string + for _, sub := range node.Filters { + out = append(out, filterLeafProperties(sub)...) + } + return out + } + if node.Property == "" { + return nil + } + return []string{node.Property} +} diff --git a/cmd/internal/anyblockbatch/scan_test.go b/cmd/internal/anyblockbatch/scan_test.go new file mode 100644 index 0000000000..0ea6838600 --- /dev/null +++ b/cmd/internal/anyblockbatch/scan_test.go @@ -0,0 +1,433 @@ +package anyblockbatch + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" +) + +// writeDocs lays out a bundle in a temp dir and returns its files, sorted the +// way DiscoverJSONFiles returns them. +func writeDocs(t *testing.T, docs map[string]string) []string { + t.Helper() + dir := t.TempDir() + for name, body := range docs { + path := filepath.Join(dir, name) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) + } + files, err := DiscoverJSONFiles(dir) + require.NoError(t, err) + return files +} + +const wikiPageType = `{"version": 2, "kind": "object_type", "internal_key": "wikiPage", "id": "type-wiki-page"}` + +func TestCheckTemplateTargets_ResolvableTargetPasses(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, + "templates/article.json": `{"version": 2, "kind": "template", "type": "template", "template_for": "wikiPage"}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + + bad, err := CheckTemplateTargets(files, typeIds) + require.NoError(t, err) + assert.Empty(t, bad) +} + +// a bundled type key is not enough: UpdateObjectIDsInRelations passes bundled +// ids through untouched, so "_otpage" would survive import literally and match +// no type in the space +func TestCheckTemplateTargets_BundledTargetIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "templates/note.json": `{"version": 2, "kind": "template", "type": "template", "template_for": "page"}`, + }) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Equal(t, "page", bad[0].Target) + assert.Contains(t, ReportTemplateTargets(bad), "bundled") +} + +func TestCheckTemplateTargets_UndefinedTargetIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "templates/article.json": `{"version": 2, "kind": "template", "type": "template", "template_for": "wikiPage"}`, + }) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Equal(t, "wikiPage", bad[0].Target) +} + +// a type document with no id has nothing for the detail to point at +func TestCheckTemplateTargets_IdlessTargetIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": `{"version": 2, "kind": "object_type", "internal_key": "wikiPage"}`, + "templates/article.json": `{"version": 2, "kind": "template", "type": "template", "template_for": "wikiPage"}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + + bad, err := CheckTemplateTargets(files, typeIds) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Contains(t, bad[0].Reason, "no \"id\"") +} + +// a template naming no target type at all belongs to nothing — the app refuses +// to create one (objectcreator.createTemplate), and import does not +func TestCheckTemplateTargets_MissingTemplateForIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "templates/orphan.json": `{"version": 2, "kind": "template", "type": "template"}`, + }) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Empty(t, bad[0].Target) +} + +// an authored targetObjectType is the value the converter keeps, so the +// document is wired whatever templateFor resolves to +func TestCheckTemplateTargets_AuthoredTargetObjectTypePasses(t *testing.T) { + files := writeDocs(t, map[string]string{ + "templates/article.json": `{"version": 2, "kind": "template", "type": "template", "template_for": "page", + "properties": {"targetObjectType": "type-page"}}`, + }) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + assert.Empty(t, bad) +} + +func TestCheckTemplateTargets_NonTemplatesAreIgnored(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, + "objects/page.json": `{"version": 2, "type": "wikiPage"}`, + }) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + assert.Empty(t, bad) +} + +// A widget target is the one reference in the format whose failure is silent: +// handleLinkBlock rewrites an unresolvable target to _missing_object and +// WidgetObject.Init then removes the link and its wrapper, so the widget is +// gone without an error anywhere. +func TestCheckIndexTargets_Widgets(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, + "objects/home.json": `{"version": 2, "type": "wikiPage", "id": "page-home"}`, + }) + + index := func(targets ...string) *anyblockjson.Index { + idx := &anyblockjson.Index{} + for _, target := range targets { + idx.Widgets = append(idx.Widgets, anyblockjson.Widget{Target: target}) + } + return idx + } + + t.Run("an object in the bundle passes", func(t *testing.T) { + assert.Empty(t, CheckIndexTargets(index("page-home"), files)) + }) + + // the whole inventory widget.IsPredefinedWidgetTargetId knows, which + // handleLinkBlock leaves alone + t.Run("the importable reserved listings pass", func(t *testing.T) { + assert.Empty(t, CheckIndexTargets(index(anyblockjson.ReservedWidgetTargets()...), files)) + }) + + // The listings used to be bare words, and the pb importer consults the + // bundle's own id map BEFORE widget.IsPredefinedWidgetTargetId + // (common.UpdateLinksToObjects), so an object with id `set` captured every + // widget that meant the Sets listing — with no finding here and no error + // at import. The bare word is now an ordinary id and the listing is + // `_set`, which CheckBundleIds forbids any object from claiming. + t.Run("a bundle object can no longer shadow a listing", func(t *testing.T) { + shadow := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, + "objects/sets.json": `{"version": 2, "type": "wikiPage", "id": "set"}`, + }) + assert.Empty(t, CheckIndexTargets(index("_set"), shadow), + "the listing resolves as a listing, whatever ids the bundle ships") + for _, target := range anyblockjson.ReservedWidgetTargets() { + assert.True(t, anyblockjson.IsPlatformId(target), target) + } + }) + + t.Run("an unknown id is reported", func(t *testing.T) { + bad := CheckIndexTargets(index("page-gone"), files) + require.Len(t, bad, 1) + assert.Equal(t, "widgets[0]", bad[0].Property) + assert.Contains(t, bad[0].Reason, "no object with that id") + }) + + // This test used to pin the OPPOSITE: _all_objects and _recent_open were + // real targets in a live space that the importer did not know, so a + // bundle naming one was refused here rather than losing the widget + // silently on install. widget.IsPredefinedWidgetTargetId knows the whole + // inventory since GO-7383, so the same scenario now pins that every + // reserved listing is importable — the refusal branch stays in + // CheckIndexTargets as the guard for the day the two inventories come + // apart again. + t.Run("every reserved listing survives import", func(t *testing.T) { + for _, target := range anyblockjson.ReservedWidgetTargets() { + assert.True(t, anyblockjson.IsImportableWidgetTarget(target), target) + assert.Empty(t, CheckIndexTargets(index(target), files), target) + } + }) + + // the auto-widget ledger's entries are target-shaped references (§2c): + // a reserved listing or a bundle object, and nothing else + t.Run("auto_widget_targets entries are checked like targets", func(t *testing.T) { + idx := index("page-home") + idx.AutoWidgetTargets = []string{"_bin", "page-home", "page-gone"} + bad := CheckIndexTargets(idx, files) + require.Len(t, bad, 1) + assert.Equal(t, "auto_widget_targets[2]", bad[0].Property) + assert.Equal(t, "page-gone", bad[0].Target) + }) +} + +// CheckBundleIds is what makes the reserved `_` namespace hold. Without it the +// rename only moves the shadowing target: an object with id `_set` captures +// the Sets widget exactly the way one with id `set` used to, because the +// importer still resolves through the bundle's ids first. +func TestCheckBundleIds(t *testing.T) { + t.Run("an ordinary bundle passes", func(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, + "objects/home.json": `{"version": 2, "type": "wikiPage", "id": "page-home"}`, + "objects/under.json": `{"version": 2, "type": "wikiPage", "id": "my_page_2"}`, + }) + bad, err := CheckBundleIds(files) + require.NoError(t, err) + assert.Empty(t, bad, "`_` is only reserved as a PREFIX") + }) + + // every reserved name, so adding a listing without adding a ban is not a + // thing that can happen + t.Run("no reserved listing or screen can be a bundle id", func(t *testing.T) { + reserved := append(anyblockjson.ReservedWidgetTargets(), + anyblockjson.HomepageWidgets, anyblockjson.HomepageGraph) + // …and neither can what those are called on the WIRE. This is the half + // the prefix does not cover and the half that decides whether the + // rename accomplished anything: anyblockconvert translates a widget + // target `_set` to `set` before writing the link, and + // common.handleLinkBlock then resolves `set` through the bundle's own + // id map BEFORE asking widget.IsPredefinedWidgetTargetId. Ban only the + // prefix and an object with id `set` captures the Sets widget exactly + // as it did before the rename — the collision moved, nothing else. + for _, r := range reserved { + if wire := anyblockjson.WireWidgetTarget(r); wire != r { + reserved = append(reserved, wire) + } + if wire := anyblockjson.WireHomepage(r); wire != r { + reserved = append(reserved, wire) + } + } + require.Contains(t, reserved, "set", "the wire spellings must be in this list, or it proves nothing") + require.Contains(t, reserved, "graph") + for _, id := range reserved { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, + "objects/shadow.json": `{"version": 2, "type": "wikiPage", "id": "` + id + `"}`, + }) + bad, err := CheckBundleIds(files) + require.NoError(t, err, id) + require.Len(t, bad, 1, id) + assert.Equal(t, id, bad[0].Target) + assert.Equal(t, "id", bad[0].Property) + } + }) + + // the prefix is the rule, not the six words: a bundled object's own + // address is just as unmintable + t.Run("a bundled platform address cannot be a bundle id", func(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, + "objects/page.json": `{"version": 2, "type": "wikiPage", "id": "_otpage"}`, + }) + bad, err := CheckBundleIds(files) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Contains(t, bad[0].Reason, "platform") + }) +} + +// --- CheckTargetTypes: the arms must be ordered the way the converter orders +// them. batch.objectTypeIds asks typeIds FIRST and only falls through to the +// bundled url, so a bundle that defines a document with a bundled key takes +// the local arm — whatever the bundle table says. + +// Fail-OPEN: `page` is bundled AND defined here without an id. Asking the +// bundle first short-circuits and reports clean, while objectTypeIds takes the +// local arm, finds the empty id, and appends an EMPTY STRING to +// relationFormatObjectTypes — a target that names nothing, is invisible in +// every UI, and re-exports as a shorter list than it went in as. +// (cmd/anyblockconvert TestBatch_IdlessLocalTypeYieldsAnEmptyTargetId pins the +// converter half of this.) +func TestCheckTargetTypes_BundledKeyDefinedLocallyWithoutIdIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/page.type.json": `{"version": 2, "kind": "object_type", "internal_key": "page"}`, + "types/person.type.json": `{"version": 2, "kind": "object_type", "internal_key": "person", "id": "type-person", + "type_settings": {"property_definitions": [{"property": "assignee", "format": "objects", "object_types": ["page"]}]}}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + require.Contains(t, typeIds, "page", + "the fixture only bites if the id-less type really is registered — that is what makes the converter take the local arm") + + bad, err := CheckTargetTypes(files, typeIds) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Equal(t, "page", bad[0].Target) + assert.Contains(t, bad[0].Reason, `no "id"`) + assert.Contains(t, bad[0].Reason, "prefers it over the bundled type", + "the author needs to know why a bundled key is being complained about") +} + +// The same shadowing with an id is fine: the converter uses the local id, and +// so it has something real to point at. +func TestCheckTargetTypes_BundledKeyDefinedLocallyWithIdPasses(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/page.type.json": `{"version": 2, "kind": "object_type", "internal_key": "page", "id": "type-page"}`, + "types/person.type.json": `{"version": 2, "kind": "object_type", "internal_key": "person", "id": "type-person", + "type_settings": {"property_definitions": [{"property": "assignee", "format": "objects", "object_types": ["page"]}]}}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + bad, err := CheckTargetTypes(files, typeIds) + require.NoError(t, err) + assert.Empty(t, bad, "%s", ReportTargets(bad)) +} + +// An id-less local type that shadows nothing bundled is reported the same way +// — the reorder must not lose the arm that already worked. +func TestCheckTargetTypes_IdlessLocalTypeIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": `{"version": 2, "kind": "object_type", "internal_key": "wikiPage"}`, + "types/person.type.json": `{"version": 2, "kind": "object_type", "internal_key": "person", "id": "type-person", + "type_settings": {"property_definitions": [{"property": "assignee", "format": "objects", "object_types": ["wikiPage"]}]}}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + bad, err := CheckTargetTypes(files, typeIds) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Contains(t, bad[0].Reason, `no "id"`) + assert.NotContains(t, bad[0].Reason, "prefers it over the bundled type") +} + +// The manifest `files` map binds a file document to its bytes (§2c, v0.47), +// and every failure it can have is silent at import — the file object +// arrives, its content does not. Refusals cover what the map STATES: a key +// naming no document, a path escaping the bundle, a promised blob that is +// not there. A bundle whose map omits a file document passes — whether an +// absent blob is intended is the thin-bundle marker's future question +// (SPEC §15 #20), and refusing it today would refuse every pre-v0.47 +// bundle. +// +// How this can fail: stat the blob against the WALK root instead of the +// index's own directory (a nested bundle's bindings all read missing); +// stop cleaning the path before the escape check (`a/../../x` reads as +// contained); or require an entry per file document (every legacy bundle +// turns invalid overnight). +func TestCheckManifestFiles(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "files"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "files", "file-1.anyblock.json"), + []byte(`{"version": 2, "kind": "file_object", "id": "file-1"}`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "files", "file-1.png"), []byte("bytes"), 0o644)) + files, err := DiscoverJSONFiles(dir) + require.NoError(t, err) + + manifest := func(entries map[string]string) *anyblockjson.Index { + return &anyblockjson.Index{Manifest: &anyblockjson.Manifest{Files: entries}} + } + + t.Run("a binding whose document and blob both exist passes", func(t *testing.T) { + assert.Empty(t, CheckManifestFiles(manifest(map[string]string{"file-1": "files/file-1.png"}), dir, files)) + }) + t.Run("no map, nothing to check", func(t *testing.T) { + assert.Empty(t, CheckManifestFiles(&anyblockjson.Index{}, dir, files)) + }) + t.Run("a key naming no document is a binding for nothing", func(t *testing.T) { + got := CheckManifestFiles(manifest(map[string]string{"ghost": "files/file-1.png"}), dir, files) + require.Len(t, got, 1) + assert.Contains(t, got[0].Reason, "no document") + }) + t.Run("a path that climbs out of the bundle is refused", func(t *testing.T) { + got := CheckManifestFiles(manifest(map[string]string{"file-1": "../outside.png"}), dir, files) + require.Len(t, got, 1) + assert.Contains(t, got[0].Reason, "outside the bundle") + got = CheckManifestFiles(manifest(map[string]string{"file-1": "files/../../outside.png"}), dir, files) + require.Len(t, got, 1) + assert.Contains(t, got[0].Reason, "outside the bundle") + }) + t.Run("a promised blob that is not there is refused", func(t *testing.T) { + got := CheckManifestFiles(manifest(map[string]string{"file-1": "files/file-1.pdf"}), dir, files) + require.Len(t, got, 1) + assert.Contains(t, got[0].Reason, "missing") + }) +} + +// A FAT bundle's manifest-bound .json BLOB is content, not a document +// (§2c, v0.47): 12 corpus file objects carry file_ext == "json", and only +// the `files` map can tell their bytes from an authored bare-.json +// document. Discovery excludes exactly what the map binds — nothing more: +// an unbound .json stays a document, because authored bundles legitimately +// name documents with a bare extension. +// +// How this can fail: skip by extension convention instead of the map +// (every authored bare-.json document vanishes from every batch); or fail +// discovery on a broken index (a bundle with one bad index.json loses its +// whole document set instead of its blob exclusion). +func TestDiscoverJSONFiles_SkipsManifestBoundBlobs(t *testing.T) { + dir := t.TempDir() + write := func(rel, body string) { + path := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) + } + write("objects/home.json", `{"version": 2, "id": "home"}`) + write("files/data.anyblock.json", `{"version": 2, "kind": "file_object", "id": "data"}`) + write("files/data.json", `{"whatever": "a JSON blob that is file CONTENT"}`) + write("index.json", `{"version": 2, "manifest": {"files": {"data": "files/data.json"}}}`) + + files, err := DiscoverJSONFiles(dir) + require.NoError(t, err) + rels := make([]string, 0, len(files)) + for _, f := range files { + rel, _ := filepath.Rel(dir, f) + rels = append(rels, filepath.ToSlash(rel)) + } + assert.ElementsMatch(t, []string{"objects/home.json", "files/data.anyblock.json"}, rels, + "the bound blob is skipped; the documents — double-extended or bare — stay") +} + +// A file document a PRESENT manifest map does not bind is warned about — +// bytes that did not travel, the signature of a partially failed export. +// A bundle with NO map stays silent: metadata-only exports (files off, +// pre-v0.47) are a legitimate mode, and warning on every one of them would +// train readers to ignore the warning. +func TestUnboundFileDocuments(t *testing.T) { + files := writeDocs(t, map[string]string{ + "files/bound.anyblock.json": `{"version": 2, "kind": "file_object", "id": "bound"}`, + "files/unbound.anyblock.json": `{"version": 2, "kind": "file_object", "id": "unbound"}`, + "objects/page.json": `{"version": 2, "id": "page"}`, + }) + + withMap := &anyblockjson.Index{Manifest: &anyblockjson.Manifest{ + Files: map[string]string{"bound": "files/bound.png"}}} + assert.Equal(t, []string{"unbound"}, UnboundFileDocuments(withMap, files), + "only the file document the map skips; pages are not file documents") + + assert.Empty(t, UnboundFileDocuments(&anyblockjson.Index{}, files), + "no map, no warning — metadata-only bundles are a mode, not a defect") +} diff --git a/cmd/internal/anyblockbatch/typeterm.go b/cmd/internal/anyblockbatch/typeterm.go new file mode 100644 index 0000000000..a6e6296fdd --- /dev/null +++ b/cmd/internal/anyblockbatch/typeterm.go @@ -0,0 +1,80 @@ +package anyblockbatch + +// typeterm.go — the one place the batch binds a TYPE term to a stored key. +// +// The envelope `key` a type document carries is the raw STORED key and is +// never translated (SPEC.md §2). Every other type slot IS translated: the +// envelope `type` and `template_for`, and `type_settings.property_definitions[].object_types` +// carry a term that resolves through the §3 chain — the document's own +// `type_internal_keys` legend, then the bundled name table (with its +// forgiving fold), then verbatim. +// +// The lints below compare those slots against `TypeIds`, a map keyed by the +// untranslated envelope `key`. Comparing an untranslated map against a +// translated slot fails both ways: +// +// - fail-closed: a bundle whose `template_for` is a spelling its `type_internal_keys` +// legend binds to a stored key is rejected though the converter resolves +// it perfectly well — and anyblockconvert turns the lint's finding into a +// hard error, so a correct bundle cannot be converted; +// - fail-open, and worse: when the term happens to equal some OTHER type's +// stored key, the lint reports nothing while the converter resolves +// elsewhere and falls through to `_ot` — a bundled url for a +// type that is not bundled, matching nothing on import. That silent +// dangling reference is precisely what these lints exist to catch. +// +// So the lint has to run the codec's own chain before it looks anything up. + +import "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + +// typeLegend is the `type_internal_keys` envelope legend (§2), decoded alongside +// whatever slots a lint reads. It is per-document: the legend is the +// statement THIS document makes about ITS spellings, so it must be decoded +// from the same file as the slot it resolves. +type typeLegend map[string]string + +// resolveTypeTerm binds one type term to the stored key it names, running the +// §3 chain in the type namespace exactly as the codec runs it here. +// +// "Here" is a package-only reader: anyblockconvert and anyblockvalidate pass +// no anyblockjson.Options.Keys, so anyblockjson resolves every type slot +// through the document's legend and then BundledKeyVocabulary — which is the +// same two steps below. The legend lookup is this file's only original line; +// the rest of the chain is one call into the package's own exported +// vocabulary, which answers the bundled name table when it knows the term, +// the forgiving fold when exactly one candidate remains, and hands the term +// back untouched otherwise (that pass-through IS chain step 5, verbatim). +// +// TestLintResolvesTypeTermsLikeTheCodec pins the composition against what +// anyblockjson.Unmarshal actually stores, so the two cannot drift apart +// silently. +func resolveTypeTerm(legend typeLegend, term string) string { + if term == "" { + return "" + } + if key, ok := legend[term]; ok && key != "" { + return key + } + key, _ := anyblockjson.BundledKeyVocabulary{}.TypeKey(term) + return key +} + +// templateKind is what a template's `kind` says (§2). Since v0.22 that is the +// only thing that makes a document a template: the type term `template` used +// to carry the meaning as a reserved spelling, resolved through the document's +// own chain, which is why the gate used to resolve rather than compare. The +// kind is a fixed vocabulary name, so it is compared. +const templateKind = "template" + +// resolvedNote annotates a finding whose slot spelling differs from the +// stored key it resolves to, so the reported term stays the one the author +// can find in the file while the reason names what the converter will +// actually look for. +func resolvedNote(term, resolved string) string { + if resolved == term { + return "" + } + return " (this document's type_internal_keys legend binds " + quote(term) + " to the stored key " + quote(resolved) + ")" +} + +func quote(s string) string { return `"` + s + `"` } diff --git a/cmd/internal/anyblockbatch/typeterm_test.go b/cmd/internal/anyblockbatch/typeterm_test.go new file mode 100644 index 0000000000..13c92655e1 --- /dev/null +++ b/cmd/internal/anyblockbatch/typeterm_test.go @@ -0,0 +1,246 @@ +package anyblockbatch + +// The lints read TRANSLATED type slots (`type`, `template_for`, +// `type_properties[].object_types`) and look them up in TypeIds, which is +// keyed by the UNTRANSLATED envelope `key` (SPEC §2). Every test below is a +// bundle where those two spellings differ, which is the only way the defect +// can show: a slot spelled as a term its own `type_internal_keys` legend binds +// elsewhere. +// +// Each test states which way it fails without the fix — fail-closed (a valid +// bundle rejected, which anyblockconvert turns into a hard error) or +// fail-open (a dangling reference the lint waves through, the case the lint +// exists to catch). + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// customTypeKey is a space-minted (bson) type key, the shape a real space +// gives a user-created type — the stored key a legend entry binds a spelling +// to. +const customTypeKey = "69bbfc78877a91b1d12d1a7c" + +// customType is that type's own document, so the bundle can address it. +const customType = `{"version": 2, "kind": "object_type", "internal_key": "` + customTypeKey + `", "id": "type-custom"}` + +// --- template_for ---------------------------------------------------------- + +// Fail-closed: `template_for` spells a term the document's legend binds to a +// stored key the bundle DOES define. Without resolution the lookup misses and +// the bundle is rejected — anyblockconvert exits non-zero on a bundle it +// converts correctly. +func TestCheckTemplateTargets_LegendBackedTargetPasses(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/custom.type.json": customType, + "templates/article.json": `{"version": 2, "kind": "template", "type": "template", "template_for": "wiki_page", + "type_internal_keys": {"wiki_page": "` + customTypeKey + `"}}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + + bad, err := CheckTemplateTargets(files, typeIds) + require.NoError(t, err) + assert.Empty(t, bad) +} + +// Fail-OPEN, the important one: `template_for` spells `wikiPage`, which is +// ANOTHER type document's stored key, while this document's legend binds that +// spelling to a type the bundle does not define. The raw lookup hits the other +// type and reports nothing; the converter resolves to the bson key, finds no +// local type, and leaves targetObjectType unset — the template imports +// belonging to no type, silently. +func TestCheckTemplateTargets_TermCollidingWithAnotherTypesKeyIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, // key "wikiPage", id "type-wiki-page" + "templates/article.json": `{"version": 2, "kind": "template", "type": "template", "template_for": "wikiPage", + "type_internal_keys": {"wikiPage": "` + customTypeKey + `"}}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + require.Equal(t, "type-wiki-page", typeIds["wikiPage"], + "the fixture only bites if the colliding spelling really is another type's stored key") + + bad, err := CheckTemplateTargets(files, typeIds) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Equal(t, "wikiPage", bad[0].Target) + assert.Contains(t, bad[0].Reason, "no such type") + assert.Contains(t, bad[0].Reason, customTypeKey, "the reason must name the key the converter will look for") +} + +// Fail-open on the GATE: the type term is an ordinary custom key, so nothing +// about the document LOOKS like a template — but `kind` says it is one, the +// codec builds a Template, and the lint has to check it. This used to be +// decided by resolving the type term through the legend, and a document whose +// gate the lint got wrong was skipped whole, its missing target unreported. +func TestCheckTemplateTargets_TheKindMakesADocumentATemplate(t *testing.T) { + doc := `{"version": 2, "kind": "template", "type": "wiki_page", + "type_internal_keys": {"wiki_page": "` + customTypeKey + `"}}` + requireCodecSeesATemplate(t, doc, true) + + files := writeDocs(t, map[string]string{"templates/orphan.json": doc}) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Empty(t, bad[0].Target) + assert.Contains(t, bad[0].Reason, `no "template_for"`) +} + +// The mirror, fail-closed: `type` IS spelled `template` — this is a page +// whose object type is the Template type, which is legal — and the kind says +// page, so the document is not a template. Validation agrees and refuses +// `/template_for` on it. The lint must not demand a target the format forbids. +func TestCheckTemplateTargets_TheTypeTermDoesNotMakeATemplate(t *testing.T) { + for name, doc := range map[string]string{ + "the literal spelling": `{"version": 2, "kind": "page", "type": "template"}`, + "a legend onto the key": `{"version": 2, "kind": "page", "type": "wiki_page", "type_internal_keys": {"wiki_page": "template"}}`, + "no kind, ordinary object": `{"version": 2, "type": "wikiPage"}`, + } { + t.Run(name, func(t *testing.T) { + requireCodecSeesATemplate(t, doc, false) + + files := writeDocs(t, map[string]string{"objects/not-a-template.json": doc}) + bad, err := CheckTemplateTargets(files, map[string]string{}) + require.NoError(t, err) + assert.Empty(t, bad) + }) + } +} + +// requireCodecSeesATemplate pins the fixture to the codec's own verdict, so a +// test claiming "the converter builds a template here" cannot quietly stop +// being true. +func requireCodecSeesATemplate(t *testing.T, doc string, want bool) { + t.Helper() + require.NoError(t, anyblockjson.Validate([]byte(doc)), + "the fixture must be a document the codec accepts, or the lint would never see it") + sbType, _, err := anyblockjson.Unmarshal([]byte(doc), anyblockjson.Options{}) + require.NoError(t, err) + assert.Equal(t, want, sbType == model.SmartBlockType_Template) +} + +// --- object_types ---------------------------------------------------------- + +// Fail-closed: object_types names a term the legend backs with a stored key +// the bundle defines. +func TestCheckTargetTypes_LegendBackedTargetPasses(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/custom.type.json": customType, + "types/person.type.json": `{"version": 2, "kind": "object_type", "internal_key": "person", "id": "type-person", + "type_internal_keys": {"wiki_page": "` + customTypeKey + `"}, + "type_settings": {"property_definitions": [{"property": "assignee", "format": "objects", "object_types": ["wiki_page"]}]}}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + + bad, err := CheckTargetTypes(files, typeIds) + require.NoError(t, err) + assert.Empty(t, bad) +} + +// Fail-OPEN: object_types spells `wikiPage`, another type's stored key, while +// the legend binds it to a type the bundle does not define. The raw lookup +// hits the other type and reports nothing; the converter resolves to the bson +// key, misses typeIDs, and emits `_ot69bbfc…` — a bundled url for a type that +// is not bundled, which matches nothing on import. +func TestCheckTargetTypes_TermCollidingWithAnotherTypesKeyIsReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/wiki-page.type.json": wikiPageType, // key "wikiPage", id "type-wiki-page" + "types/person.type.json": `{"version": 2, "kind": "object_type", "internal_key": "person", "id": "type-person", + "type_internal_keys": {"wikiPage": "` + customTypeKey + `"}, + "type_settings": {"property_definitions": [{"property": "assignee", "format": "objects", "object_types": ["wikiPage"]}]}}`, + }) + typeIds, err := TypeIds(files) + require.NoError(t, err) + require.Equal(t, "type-wiki-page", typeIds["wikiPage"], + "the fixture only bites if the colliding spelling really is another type's stored key") + + bad, err := CheckTargetTypes(files, typeIds) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Equal(t, "wikiPage", bad[0].Target) + assert.Equal(t, "assignee", bad[0].Property) + assert.Contains(t, bad[0].Reason, "no such type") + assert.Contains(t, ReportTargets(bad), customTypeKey) +} + +// A target the bundle neither defines nor the bundle table knows is still +// reported — resolution must not turn every miss into a pass. +func TestCheckTargetTypes_UnknownTargetIsStillReported(t *testing.T) { + files := writeDocs(t, map[string]string{ + "types/person.type.json": `{"version": 2, "kind": "object_type", "internal_key": "person", "id": "type-person", + "type_settings": {"property_definitions": [{"property": "assignee", "format": "objects", "object_types": ["wiki_page"]}]}}`, + }) + bad, err := CheckTargetTypes(files, map[string]string{}) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Equal(t, "wiki_page", bad[0].Target) +} + +// A bundled target still passes, whichever spelling is used — the stored key +// verbatim, or a term the bundled table (with its fold) resolves — because +// resolution folds both arms into one lookup. +func TestCheckTargetTypes_BundledTargetPassesInBothSpellings(t *testing.T) { + for _, target := range []string{"object_type", "objectType", "page", "task"} { + files := writeDocs(t, map[string]string{ + "types/person.type.json": `{"version": 2, "kind": "object_type", "internal_key": "person", "id": "type-person", + "type_settings": {"property_definitions": [{"property": "assignee", "format": "objects", "object_types": ["` + target + `"]}]}}`, + }) + bad, err := CheckTargetTypes(files, map[string]string{}) + require.NoError(t, err) + assert.Empty(t, bad, target) + } +} + +// --- anti-drift ------------------------------------------------------------ + +// The lint's chain is a composition — the document's legend, then the +// package's exported bundled vocabulary — and a composition can drift from +// the codec it models. This pins it against what anyblockjson.Unmarshal +// actually STORES for the same slots, so a future change to the chain that the +// lint does not follow fails here rather than in a production bundle. +func TestLintResolvesTypeTermsLikeTheCodec(t *testing.T) { + cases := []struct { + name string + legend string + typeTerm string + templateFor string + }{ + {"verbatim custom key", ``, "template", "wikiPage"}, + {"bundled stored key spelled verbatim", ``, "template", "task"}, + {"bundled stored key that is nobody's spelling", ``, "template", "objectType"}, + {"legend-backed spelling", `"type_internal_keys": {"wiki_page": "` + customTypeKey + `"},`, "template", "wiki_page"}, + {"legend outranks the bundled table", `"type_internal_keys": {"task": "` + customTypeKey + `"},`, "template", "task"}, + {"legend on the type slot itself", `"type_internal_keys": {"wiki_page": "template"},`, "wiki_page", "task"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + doc := `{"version": 2, "kind": "template", ` + c.legend + + `"type": "` + c.typeTerm + `", "template_for": "` + c.templateFor + `"}` + require.NoError(t, anyblockjson.Validate([]byte(doc))) + + _, snap, err := anyblockjson.Unmarshal([]byte(doc), anyblockjson.Options{}) + require.NoError(t, err) + require.Len(t, snap.ObjectTypes, 2, "both type slots must reach the snapshot, or the case proves nothing") + + var probe struct { + TypeKeys typeLegend `json:"type_internal_keys"` + } + require.NoError(t, json.Unmarshal([]byte(doc), &probe)) + + assert.Equal(t, strings.TrimPrefix(snap.ObjectTypes[0], "ot-"), + resolveTypeTerm(probe.TypeKeys, c.typeTerm), "type") + assert.Equal(t, strings.TrimPrefix(snap.ObjectTypes[1], "ot-"), + resolveTypeTerm(probe.TypeKeys, c.templateFor), "template_for") + }) + } +} diff --git a/cmd/internal/anyblockbatch/viewproperties_test.go b/cmd/internal/anyblockbatch/viewproperties_test.go new file mode 100644 index 0000000000..54454e3502 --- /dev/null +++ b/cmd/internal/anyblockbatch/viewproperties_test.go @@ -0,0 +1,113 @@ +package anyblockbatch + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeDoc(t *testing.T, dir, name, body string) string { + t.Helper() + p := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(p, []byte(body), 0o644)) + return p +} + +// A view naming a property nothing declares is the silent class: the filter +// matches nothing, so the view shows everything it was meant to narrow, and +// the document validates, imports and round-trips byte-stably. Six of nine +// agents asked to "also exclude archived items" produced exactly that. +// +// It is a BATCH check because a per-document reader cannot judge it. A custom +// property whose stored key is already a legal spelling — `aroma_notes`, and +// 112 more in a 77-space corpus — binds no legend entry, because the spelling +// IS the key; inside one document that is indistinguishable from a typo. The +// codec-level version of this check fired on exactly that case, which is how +// the distinction was found. +// +// How this can fail: judge a spelling the dataview's own properties list +// declares, or one the bundle declares elsewhere, and a correct bundle is +// refused. +func TestCheckViewProperties(t *testing.T) { + dir := t.TempDir() + f := writeDoc(t, dir, "board.anyblock.json", `{"version":2,"type":"page", + "property_internal_keys":{"assignee":"6a32d4856761631534b22f85"}, + "blocks":[{"type":"dataview","object_id":"b1", + "properties":[{"property":"local_only","format":"checkbox"}], + "views":[{"id":"v","filters":[ + {"property":"status","condition":"equal","value":"Done"}, + {"property":"is_archived","condition":"not_equal","value":true}, + {"property":"assignee","condition":"not_empty"}, + {"property":"local_only","condition":"equal","value":true}, + {"operator":"or","filters":[{"property":"nowhere","condition":"not_empty"}]}], + "sorts":[{"property":"whenever"}], + "columns":[{"property":"status"}]}]}]}`) + + // what the bundle declares: the dictionary's own entries, plus the stored + // key the legend binds `assignee` to — a real bundle declares the key it + // binds, and the case where it does not is its own finding below + declared := map[string]bool{"status": true, "6a32d4856761631534b22f85": true} + + bad, err := CheckViewProperties([]string{f}, declared) + require.NoError(t, err) + + got := map[string]string{} + for _, b := range bad { + got[b.Property] = b.Slot + } + assert.Equal(t, map[string]string{"nowhere": "filter", "whenever": "sort"}, got, + "only the two that nothing declares, and the group node names no property itself") + + t.Run("what must NOT be flagged", func(t *testing.T) { + for _, prop := range []string{ + "status", // the property dictionary declares it + "is_archived", // a bundled property + "assignee", // bound by this document's legend + "local_only", // the dataview's own properties list declares it + } { + assert.NotContainsf(t, got, prop, "%q is declared and must pass", prop) + } + }) + + // A legend says which stored key a spelling MEANS. It does not make that + // key exist: bind a spelling to a key nothing declares and the filter is + // still a no-op, so the binding must not excuse it. + t.Run("a legend binding to a key nothing declares is still a finding", func(t *testing.T) { + bad, err := CheckViewProperties([]string{f}, map[string]bool{"status": true}) + require.NoError(t, err) + found := false + for _, b := range bad { + if b.Property == "assignee" { + found = true + } + } + assert.True(t, found, "the legend points at 6a32d485…, which nothing declares") + }) + + t.Run("a verbatim custom key the bundle declares is not a typo", func(t *testing.T) { + // the case the codec cannot tell apart: no legend entry, because the + // spelling is the stored key + f := writeDoc(t, dir, "note.anyblock.json", `{"version":2,"type":"page", + "blocks":[{"type":"dataview","object_id":"b2","views":[{"id":"v", + "filters":[{"property":"aroma_notes","condition":"not_empty"}]}]}]}`) + bad, err := CheckViewProperties([]string{f}, map[string]bool{"aroma_notes": true}) + require.NoError(t, err) + assert.Empty(t, bad) + + t.Run("and is a typo when the bundle declares nothing", func(t *testing.T) { + bad, err := CheckViewProperties([]string{f}, map[string]bool{}) + require.NoError(t, err) + require.Len(t, bad, 1) + assert.Equal(t, "aroma_notes", bad[0].Property) + }) + }) + + t.Run("the report says what the slot does instead", func(t *testing.T) { + out := ReportViewProperties(bad) + assert.Contains(t, out, "narrows nothing") + assert.Contains(t, out, "leaves the order untouched") + }) +} diff --git a/core/api/service/property.go b/core/api/service/property.go index 5f7c801f00..3296427663 100644 --- a/core/api/service/property.go +++ b/core/api/service/property.go @@ -9,7 +9,6 @@ import ( "time" "github.com/gogo/protobuf/types" - "github.com/iancoleman/strcase" apimodel "github.com/anyproto/anytype-heart/core/api/model" "github.com/anyproto/anytype-heart/core/api/pagination" @@ -215,7 +214,10 @@ func (s *Service) CreateProperty(ctx context.Context, spaceId string, request ap } if request.Key != "" { - apiKey := strcase.ToSnake(s.sanitizedString(request.Key)) + apiKey, ok := util.MintApiObjectKey(request.Key) + if !ok { + return nil, util.ErrInvalidApiObjectKey("property", request.Key) + } if s.cache.getProperties(spaceId)[apiKey] != nil { return nil, util.ErrBadInput(fmt.Sprintf("property key %q already exists", apiKey)) } @@ -270,7 +272,10 @@ func (s *Service) UpdateProperty(ctx context.Context, spaceId string, propertyId }) } if request.Key != nil { - apiKey := strcase.ToSnake(s.sanitizedString(*request.Key)) + apiKey, ok := util.MintApiObjectKey(*request.Key) + if !ok { + return nil, util.ErrInvalidApiObjectKey("property", *request.Key) + } if apiKey != prop.Key { if existing, exists := s.cache.getProperties(spaceId)[apiKey]; exists && existing.Id != propertyId { return nil, util.ErrBadInput(fmt.Sprintf("property key %q already exists", apiKey)) diff --git a/core/api/service/property_test.go b/core/api/service/property_test.go index 3fb7be00b6..5dbff9582d 100644 --- a/core/api/service/property_test.go +++ b/core/api/service/property_test.go @@ -930,6 +930,123 @@ func TestProcessProperties(t *testing.T) { }) } +// The key a caller supplies is MINTED before it is stored: what lands in +// apiObjectKey is the address every later request must use, and snake-casing +// alone leaves punctuation in place. Measured over a 38,123-object account, +// 27 of 1,530 stored api keys sat outside the key grammar the api +// advertises. +func TestService_CreateProperty_KeyIsMinted(t *testing.T) { + t.Run("a key outside the grammar is converted, and the response says so", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + request := apimodel.CreatePropertyRequest{ + Key: "Lists [in work]", + Name: "Lists", + Format: apimodel.PropertyFormatText, + } + want := "lists_in_work" + + fx.mwMock.On("ObjectCreateRelation", mock.Anything, mock.MatchedBy(func(req *pb.RpcObjectCreateRelationRequest) bool { + return req.SpaceId == mockedSpaceId && + req.Details.Fields[bundle.RelationKeyApiObjectKey.String()].GetStringValue() == want + })).Return(&pb.RpcObjectCreateRelationResponse{ + Error: &pb.RpcObjectCreateRelationResponseError{Code: pb.RpcObjectCreateRelationResponseError_NULL}, + ObjectId: "new-property-id", + }).Once() + + fx.mwMock.On("ObjectShow", mock.Anything, &pb.RpcObjectShowRequest{ + SpaceId: mockedSpaceId, + ObjectId: "new-property-id", + }).Return(&pb.RpcObjectShowResponse{ + Error: &pb.RpcObjectShowResponseError{Code: pb.RpcObjectShowResponseError_NULL}, + ObjectView: &model.ObjectView{ + Details: []*model.ObjectViewDetailsSet{ + { + Details: &types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("new-property-id"), + bundle.RelationKeyName.String(): pbtypes.String("Lists"), + bundle.RelationKeyRelationKey.String(): pbtypes.String("67b0d3e3cda913b84c1299b1"), + bundle.RelationKeyApiObjectKey.String(): pbtypes.String(want), + }, + }, + }, + }, + }, + }).Once() + + // when + property, err := fx.service.CreateProperty(ctx, mockedSpaceId, request) + + // then + require.NoError(t, err) + assert.Equal(t, want, property.Key, "the caller is told the key it must address") + }) + + t.Run("a key with nothing the grammar admits is refused", func(t *testing.T) { + // nothing is left to convert to, and storing no key at all would drop + // a key the caller explicitly asked for — the one case worth a 400 + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + request := apimodel.CreatePropertyRequest{ + Key: "➡️", + Name: "Medium", + Format: apimodel.PropertyFormatText, + } + + // when + property, err := fx.service.CreateProperty(ctx, mockedSpaceId, request) + + // then + require.ErrorIs(t, err, util.ErrBad) + assert.Nil(t, property) + }) +} + +func TestService_UpdateProperty_KeyIsMinted(t *testing.T) { + t.Run("a key with nothing the grammar admits is refused", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + key := "!!!" + request := apimodel.UpdatePropertyRequest{Key: &key} + + fx.mwMock.On("ObjectShow", mock.Anything, &pb.RpcObjectShowRequest{ + SpaceId: mockedSpaceId, + ObjectId: mockedPropertyId, + }).Return(&pb.RpcObjectShowResponse{ + Error: &pb.RpcObjectShowResponseError{Code: pb.RpcObjectShowResponseError_NULL}, + ObjectView: &model.ObjectView{ + Details: []*model.ObjectViewDetailsSet{ + { + Details: &types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(mockedPropertyId), + bundle.RelationKeyRelationKey.String(): pbtypes.String("67b0d3e3cda913b84c1299b1"), + bundle.RelationKeyApiObjectKey.String(): pbtypes.String("custom_key"), + }, + }, + }, + }, + }, + }).Once() + + // when + property, err := fx.service.UpdateProperty(ctx, mockedSpaceId, mockedPropertyId, request) + + // then + require.ErrorIs(t, err, util.ErrBad) + assert.Nil(t, property) + }) +} + func TestSanitizeAndValidatePropertyValueTypeObject(t *testing.T) { setupObjectLayoutMock := func(fx *fixture, objectId string, layout model.ObjectTypeLayout, found bool) { response := &pb.RpcObjectSearchResponse{ diff --git a/core/api/service/tag.go b/core/api/service/tag.go index d7d8a48570..7add1aced2 100644 --- a/core/api/service/tag.go +++ b/core/api/service/tag.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/gogo/protobuf/types" - "github.com/iancoleman/strcase" apimodel "github.com/anyproto/anytype-heart/core/api/model" "github.com/anyproto/anytype-heart/core/api/pagination" @@ -116,7 +115,10 @@ func (s *Service) CreateTag(ctx context.Context, spaceId string, propertyId stri } if request.Key != "" { - apiKey := strcase.ToSnake(s.sanitizedString(request.Key)) + apiKey, ok := util.MintApiObjectKey(request.Key) + if !ok { + return nil, util.ErrInvalidApiObjectKey("tag", request.Key) + } if s.cache.getTags(spaceId)[apiKey] != nil { return nil, util.ErrBadInput(fmt.Sprintf("tag key %q already exists", apiKey)) } @@ -165,7 +167,10 @@ func (s *Service) UpdateTag(ctx context.Context, spaceId string, propertyId stri }) } if request.Key != nil { - apiKey := strcase.ToSnake(s.sanitizedString(*request.Key)) + apiKey, ok := util.MintApiObjectKey(*request.Key) + if !ok { + return nil, util.ErrInvalidApiObjectKey("tag", *request.Key) + } if apiKey != tag.Key { if existing, exists := s.cache.getTags(spaceId)[apiKey]; exists && existing.Id != tagId { return nil, util.ErrBadInput(fmt.Sprintf("tag key %q already exists", apiKey)) diff --git a/core/api/service/tag_test.go b/core/api/service/tag_test.go index d823d3b471..4171adb450 100644 --- a/core/api/service/tag_test.go +++ b/core/api/service/tag_test.go @@ -814,6 +814,146 @@ func TestService_DeleteTag(t *testing.T) { }) } +// The key a caller supplies is MINTED before it is stored: what lands in +// apiObjectKey is the address every later request must use, and snake-casing +// alone leaves punctuation in place. Measured over a 38,123-object account, +// 27 of 1,530 stored api keys sat outside the key grammar the api +// advertises — all but four of them on options like these. +func TestService_Tag_KeyIsMinted(t *testing.T) { + propertyShow := func(fx *fixture) { + fx.mwMock.On("ObjectShow", mock.Anything, &pb.RpcObjectShowRequest{ + SpaceId: mockedSpaceId, + ObjectId: mockedPropertyId, + }).Return(&pb.RpcObjectShowResponse{ + Error: &pb.RpcObjectShowResponseError{Code: pb.RpcObjectShowResponseError_NULL}, + ObjectView: &model.ObjectView{ + Details: []*model.ObjectViewDetailsSet{ + { + Details: &types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyUniqueKey.String(): pbtypes.String("unique-key"), + bundle.RelationKeyRelationKey.String(): pbtypes.String(mockedPropertyKey), + }, + }, + }, + }, + }, + }).Once() + } + + t.Run("a key outside the grammar is converted, and the response says so", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + propertyShow(fx) + + request := apimodel.CreateTagRequest{ + Key: "➡️ Medium", + Name: "➡️ Medium", + Color: apimodel.ColorBlue, + } + want := "medium" + + fx.mwMock.On("ObjectCreateRelationOption", mock.Anything, mock.MatchedBy(func(req *pb.RpcObjectCreateRelationOptionRequest) bool { + return req.SpaceId == mockedSpaceId && + req.Details.Fields[bundle.RelationKeyApiObjectKey.String()].GetStringValue() == want + })).Return(&pb.RpcObjectCreateRelationOptionResponse{ + Error: &pb.RpcObjectCreateRelationOptionResponseError{Code: pb.RpcObjectCreateRelationOptionResponseError_NULL}, + ObjectId: "new-tag-id", + }).Once() + + fx.mwMock.On("ObjectShow", mock.Anything, &pb.RpcObjectShowRequest{ + SpaceId: mockedSpaceId, + ObjectId: "new-tag-id", + }).Return(&pb.RpcObjectShowResponse{ + Error: &pb.RpcObjectShowResponseError{Code: pb.RpcObjectShowResponseError_NULL}, + ObjectView: &model.ObjectView{ + Details: []*model.ObjectViewDetailsSet{ + { + Details: &types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("new-tag-id"), + bundle.RelationKeyName.String(): pbtypes.String("➡️ Medium"), + bundle.RelationKeyUniqueKey.String(): pbtypes.String("unique_new_tag"), + bundle.RelationKeyApiObjectKey.String(): pbtypes.String(want), + bundle.RelationKeyRelationOptionColor.String(): pbtypes.String("blue"), + }, + }, + }, + }, + }, + }).Once() + + // when + tag, err := fx.service.CreateTag(ctx, mockedSpaceId, mockedPropertyId, request) + + // then + require.NoError(t, err) + assert.Equal(t, want, tag.Key, "the caller is told the key it must address") + }) + + t.Run("a key with nothing the grammar admits is refused on create", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + propertyShow(fx) + + request := apimodel.CreateTagRequest{ + Key: "➡️", + Name: "Medium", + Color: apimodel.ColorBlue, + } + + // when + tag, err := fx.service.CreateTag(ctx, mockedSpaceId, mockedPropertyId, request) + + // then + require.ErrorIs(t, err, util.ErrBad) + assert.Nil(t, tag) + }) + + t.Run("a key with nothing the grammar admits is refused on update", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + newKey := "!!!" + request := apimodel.UpdateTagRequest{Key: &newKey} + + fx.mwMock.On("ObjectShow", mock.Anything, &pb.RpcObjectShowRequest{ + SpaceId: mockedSpaceId, + ObjectId: mockedTagId, + }).Return(&pb.RpcObjectShowResponse{ + Error: &pb.RpcObjectShowResponseError{Code: pb.RpcObjectShowResponseError_NULL}, + ObjectView: &model.ObjectView{ + Details: []*model.ObjectViewDetailsSet{ + { + Details: &types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(mockedTagId), + bundle.RelationKeyName.String(): pbtypes.String(mockedTagName), + bundle.RelationKeyUniqueKey.String(): pbtypes.String(mockedTagUniqueKey), + bundle.RelationKeyApiObjectKey.String(): pbtypes.String("old_key"), + bundle.RelationKeyRelationOptionColor.String(): pbtypes.String(mockedTagColor), + }, + }, + }, + }, + }, + }).Once() + + // when + tag, err := fx.service.UpdateTag(ctx, mockedSpaceId, mockedPropertyId, mockedTagId, request) + + // then + require.ErrorIs(t, err, util.ErrBad) + assert.Nil(t, tag) + }) +} + func TestService_getTagFromStruct(t *testing.T) { fx := newFixture(t) diff --git a/core/api/service/type.go b/core/api/service/type.go index ff3e33954e..e28139b825 100644 --- a/core/api/service/type.go +++ b/core/api/service/type.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/gogo/protobuf/types" - "github.com/iancoleman/strcase" apimodel "github.com/anyproto/anytype-heart/core/api/model" "github.com/anyproto/anytype-heart/core/api/pagination" @@ -222,7 +221,10 @@ func (s *Service) buildTypeDetails(ctx context.Context, spaceId string, request } if request.Key != "" { - apiKey := strcase.ToSnake(s.sanitizedString(request.Key)) + apiKey, ok := util.MintApiObjectKey(request.Key) + if !ok { + return nil, util.ErrInvalidApiObjectKey("type", request.Key) + } if _, exists := s.cache.getTypes(spaceId)[apiKey]; exists { return nil, util.ErrBadInput(fmt.Sprintf("type key %q already exists", apiKey)) } @@ -286,7 +288,10 @@ func (s *Service) buildUpdatedTypeDetails(ctx context.Context, spaceId string, t fields[bundle.RelationKeyRecommendedLayout.String()] = pbtypes.Int64(int64(s.typeLayoutToObjectTypeLayout(*request.Layout))) } if request.Key != nil { - apiKey := strcase.ToSnake(s.sanitizedString(*request.Key)) + apiKey, ok := util.MintApiObjectKey(*request.Key) + if !ok { + return nil, util.ErrInvalidApiObjectKey("type", *request.Key) + } if apiKey != t.Key { if existing, exists := s.cache.getTypes(spaceId)[apiKey]; exists && existing.Id != t.Id { return nil, util.ErrBadInput(fmt.Sprintf("type key %q already exists", apiKey)) diff --git a/core/api/service/type_test.go b/core/api/service/type_test.go index b1c98632d8..642a3f55d0 100644 --- a/core/api/service/type_test.go +++ b/core/api/service/type_test.go @@ -2,13 +2,16 @@ package service import ( "context" + "strings" "testing" "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" apimodel "github.com/anyproto/anytype-heart/core/api/model" + "github.com/anyproto/anytype-heart/core/api/util" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" @@ -146,3 +149,109 @@ func TestObjectService_GetType(t *testing.T) { require.Empty(t, ot) }) } + +// The key a caller supplies is MINTED before it is stored: what lands in +// apiObjectKey is the address every later request must use, and snake-casing +// alone leaves punctuation in place. Measured over a 38,123-object account, +// 27 of 1,530 stored api keys sat outside the key grammar the api +// advertises. +func TestService_buildTypeDetails_KeyIsMinted(t *testing.T) { + t.Run("a key outside the grammar is converted", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + request := apimodel.CreateTypeRequest{ + Key: "Manual export & import", + Name: "Manual export", + } + + // when + details, err := fx.service.buildTypeDetails(ctx, mockedSpaceId, request) + + // then + require.NoError(t, err) + assert.Equal(t, "manual_export_import", details.Fields[bundle.RelationKeyApiObjectKey.String()].GetStringValue()) + }) + + t.Run("a key longer than a key may be is bounded", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + request := apimodel.CreateTypeRequest{ + Key: strings.Repeat("k", 300), + Name: "Long", + } + + // when + details, err := fx.service.buildTypeDetails(ctx, mockedSpaceId, request) + + // then + require.NoError(t, err) + assert.Len(t, details.Fields[bundle.RelationKeyApiObjectKey.String()].GetStringValue(), bundle.MaxApiSlugLen) + }) + + t.Run("a key with nothing the grammar admits is refused", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + request := apimodel.CreateTypeRequest{ + Key: "Задача", + Name: "Task", + } + + // when + details, err := fx.service.buildTypeDetails(ctx, mockedSpaceId, request) + + // then + require.ErrorIs(t, err, util.ErrBad) + assert.Nil(t, details) + }) +} + +func TestService_buildUpdatedTypeDetails_KeyIsMinted(t *testing.T) { + existing := &apimodel.Type{ + Id: mockedTypeId, + Key: "custom_type_key", + UniqueKey: "ot-67b0d3e3cda913b84c1299b1", + } + + t.Run("a key outside the grammar is converted", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + key := "Lists [in work]" + request := apimodel.UpdateTypeRequest{Key: &key} + + // when + details, err := fx.service.buildUpdatedTypeDetails(ctx, mockedSpaceId, existing, request) + + // then + require.NoError(t, err) + assert.Equal(t, "lists_in_work", details.Fields[bundle.RelationKeyApiObjectKey.String()].GetStringValue()) + }) + + t.Run("a key with nothing the grammar admits is refused", func(t *testing.T) { + // given + ctx := context.Background() + fx := newFixture(t) + fx.populateCache(mockedSpaceId) + + key := "➡️" + request := apimodel.UpdateTypeRequest{Key: &key} + + // when + details, err := fx.service.buildUpdatedTypeDetails(ctx, mockedSpaceId, existing, request) + + // then + require.ErrorIs(t, err, util.ErrBad) + assert.Nil(t, details) + }) +} diff --git a/core/api/util/key.go b/core/api/util/key.go index a8e33b9267..090ab816bd 100644 --- a/core/api/util/key.go +++ b/core/api/util/key.go @@ -1,10 +1,13 @@ package util import ( + "fmt" "regexp" "strings" "github.com/iancoleman/strcase" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" ) // Key transformation from Internal to API format: @@ -72,3 +75,41 @@ func ToTagApiKey(internalKey string) (apiKey string) { func IsBsonId(key string) bool { return len(key) == bsonIdLength && bsonIdPattern.MatchString(key) && digitPattern.MatchString(key) } + +// MintApiObjectKey mints the apiObjectKey to store for a key an API caller +// supplied. It is the same mint objectcreator applies to a derived key, so a +// property created through the api and one created from a display name land +// in the same grammar and are addressed the same way. +// +// CONVERSION, not refusal, for anything that survives the grammar. Three +// reasons, in order of weight: +// +// - Conversion is the advertised contract. The key field on every create +// and update request documents that a key "should always be snake_case, +// otherwise it will be converted to snake_case". Refusing would break +// that for input that works today: a caller sending `Due Date` and +// receiving `due_date` is not a caller in error. +// - The caller is told. Create and update answer with the object, and its +// key field carries the minted key, so a caller never has to guess the +// spelling it must address. That is what refusal would have bought. +// - Today's alternative is worse than either. Snake-casing alone stores the +// caller's punctuation verbatim, so `Lists [in work]` becomes the stored +// key `lists_[in_work]` — accepted, and then not the snake_case spelling +// the same endpoint promised to store. 27 of 1,530 keys in a measured +// 38,123-object account sit on that path, and nobody was ever told. +// +// ok is false only when NOTHING of the key survives the grammar (`"➡️"`, +// `"!!!"`). That is not a conversion — there is no spelling to convert to — +// and storing no key at all would silently drop a key the caller explicitly +// asked for, so it is the one case the caller must hear about. +func MintApiObjectKey(key string) (apiKey string, ok bool) { + apiKey = bundle.MintApiSlug(key) + return apiKey, apiKey != "" +} + +// ErrInvalidApiObjectKey is the bad-input error for a key MintApiObjectKey +// could mint nothing from. kind names the object ("property", "type", "tag") +// so the message reads like the key errors beside it. +func ErrInvalidApiObjectKey(kind string, key string) error { + return ErrBadInput(fmt.Sprintf("%s key %q holds no character a key may contain; keys are made of letters, digits and underscores", kind, key)) +} diff --git a/core/api/util/key_test.go b/core/api/util/key_test.go index 2ceaa16db2..bf561761ec 100644 --- a/core/api/util/key_test.go +++ b/core/api/util/key_test.go @@ -1,9 +1,13 @@ package util import ( + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" ) func TestToPropertyApiKey(t *testing.T) { @@ -245,6 +249,96 @@ func TestEdgeCases(t *testing.T) { }) } +// MintApiObjectKey is the mint for a key an API CALLER supplied. The fixtures +// are the shapes measured in a 38,123-object account, where 27 of 1,530 +// stored api keys sat outside the key grammar the api advertises. +func TestMintApiObjectKey(t *testing.T) { + tests := []struct { + name string + key string + expected string + ok bool + }{ + { + name: "already a key", + key: "my_custom_key", + expected: "my_custom_key", + ok: true, + }, + { + name: "spelled with spaces", + key: "due date", + expected: "due_date", + ok: true, + }, + { + name: "brackets are dropped, not stored", + key: "Lists [in work]", + expected: "lists_in_work", + ok: true, + }, + { + name: "ampersand is dropped, not stored", + key: "Manual export & import", + expected: "manual_export_import", + ok: true, + }, + { + name: "an emoji is dropped and the word beside it survives", + key: "➡️ Medium", + expected: "medium", + ok: true, + }, + { + name: "surrounding whitespace never reaches the key", + key: " spaced key ", + expected: "spaced_key", + ok: true, + }, + { + name: "a key of only emoji mints nothing", + key: "➡️", + expected: "", + ok: false, + }, + { + name: "a key of only punctuation mints nothing", + key: "!!!", + expected: "", + ok: false, + }, + { + name: "a key in a script the grammar has no letters for mints nothing", + // unlike a display name, a supplied key is not transliterated: + // answering "Задача" with "zadacha" would name the property + // something its author never wrote, so the caller is told instead + key: "Задача", + expected: "", + ok: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, ok := MintApiObjectKey(tt.key) + assert.Equal(t, tt.expected, result) + assert.Equal(t, tt.ok, ok) + }) + } + + t.Run("length is bounded", func(t *testing.T) { + result, ok := MintApiObjectKey(strings.Repeat("k", 300)) + assert.True(t, ok) + assert.Len(t, result, bundle.MaxApiSlugLen) + }) + + t.Run("the refusal is bad input, carrying the key the caller sent", func(t *testing.T) { + err := ErrInvalidApiObjectKey("property", "➡️") + require.ErrorIs(t, err, ErrBad) + assert.Contains(t, err.Error(), `property key "➡️"`) + }) +} + func BenchmarkToPropertyApiKey(b *testing.B) { b.Run("CamelCase", func(b *testing.B) { for i := 0; i < b.N; i++ { diff --git a/core/block/editor/smartblock/detailsprotect.go b/core/block/editor/smartblock/detailsprotect.go new file mode 100644 index 0000000000..51bfa89d13 --- /dev/null +++ b/core/block/editor/smartblock/detailsprotect.go @@ -0,0 +1,93 @@ +package smartblock + +import ( + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" +) + +// identityDetails say what a relation, an option of one, or a type object IS, as opposed to how it +// looks. relationKey is the key every value of the relation is stored under, and on an option the +// relation it belongs to; relationFormat is how those values are read back; sourceObject is the +// bundled definition the object was installed from - the key InstallBundledObjects matches an +// installed object on. Repointing one of them on an object that already exists reinterprets or +// orphans every value ever written under it, and no editor, migration or importer has a reason to, +// so the stored value wins over whatever a writer brings. +// +// One list serves all three kinds rather than a set per type. Options never carry relationFormat or +// sourceObject, and nothing writes either one onto an existing option, so guarding them there costs +// nothing and keeps the rule one sentence long. +// +// Two neighbouring keys are deliberately not here: +// - relationFormatObjectTypes, which SystemObjectReviser extends when a bundled definition gains +// an allowed type (see checkRelationFormatObjectTypes); +// - uniqueKey, whose data source is derived, so it never reaches the tree and injectDerivedDetails +// rebuilds it from the object's own header on every Apply. +var identityDetails = []domain.RelationKey{ + bundle.RelationKeyRelationKey, + bundle.RelationKeyRelationFormat, + bundle.RelationKeySourceObject, +} + +// hasIdentityDetails reports whether identityDetails describe the identity of this object. +// The bundled counterparts live in the read-only marketplace space and are rebuilt from the bundle +// on every read, so there is nothing to preserve for them. +func hasIdentityDetails(sbType smartblock.SmartBlockType) bool { + switch sbType { + case smartblock.SmartBlockTypeRelation, + smartblock.SmartBlockTypeRelationOption, + smartblock.SmartBlockTypeObjectType: + return true + default: + return false + } +} + +// preserveIdentityDetails restores every identityDetails value that the incoming state changes or +// drops on an object that already carries one. It runs as HookBeforeApply, so the restored value is +// the one that reaches the tree and the writer's attempt leaves nothing behind but a log line. +// +// It never returns an error on purpose. Apply discards the whole state when HookBeforeApply fails, +// and reports success to the caller while doing so, so erroring here would turn one bad detail into +// a lost document - during an import, into a lost import. +func (sb *smartBlock) preserveIdentityDetails(info ApplyInfo) error { + stored := committedState(info.State) + for _, key := range identityDetails { + storedValue := stored.Details().Get(key) + if isDetailUnset(storedValue) { + // the first write wins, so both creation and a backfill of a missing value pass through + continue + } + newValue := info.State.Details().Get(key) + if newValue.Equal(storedValue) { + continue + } + info.State.SetDetail(key, storedValue) + log.With("objectId", sb.Id(), "spaceId", sb.SpaceID(), "sbType", sb.Type().String(), + "detail", key.String(), "stored", storedValue.Raw(), "rejected", newValue.Raw()). + Warnf("identity detail of an existing object can not be changed, keeping the stored value") + } + return nil +} + +// committedState returns the state Apply is about to merge into. Writers may stack several states +// on top of it, and the identity to compare against is the one already in the tree, not one an +// intermediate state introduced along the way. A state with no parent is its own committed state, +// which turns every comparison into a no-op: there is nothing yet to preserve. +func committedState(s *state.State) *state.State { + for s.ParentState() != nil { + s = s.ParentState() + } + return s +} + +// isDetailUnset reports whether a detail carries no value yet. A missing key and an empty string +// both count, a zero number does not: 0 is the longtext relation format, not the absence of one. +func isDetailUnset(v domain.Value) bool { + if !v.Ok() || v.IsNull() { + return true + } + str, isString := v.TryString() + return isString && str == "" +} diff --git a/core/block/editor/smartblock/detailsprotect_test.go b/core/block/editor/smartblock/detailsprotect_test.go new file mode 100644 index 0000000000..5229bac19c --- /dev/null +++ b/core/block/editor/smartblock/detailsprotect_test.go @@ -0,0 +1,313 @@ +package smartblock + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/logging" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +const testRelationId = "relationObjectId" + +// newIdentityFixture builds a smartblock of the given type and runs the real Init, so the guard is +// registered exactly the way production registers it. Every assertion below goes through the real +// Apply: if the hook were not reached, the incoming value would simply land in the doc. +func newIdentityFixture(t *testing.T, sbType smartblock.SmartBlockType) *fixture { + fx := newFixture(testRelationId, t) + fx.source.sbType = sbType + fx.init(t, []*model.Block{{Id: testRelationId}}) + fx.indexer.EXPECT().Index(mock.Anything, mock.Anything).Return(nil).Maybe() + fx.eventSender.EXPECT().SendToSession(mock.Anything, mock.Anything).Maybe() + return fx +} + +// captureLog swaps the package logger for one writing into an observer, so a test can tell a +// silent no-op apart from a rejected write. Other warnings of the package are filtered out by +// rejections, so a count is a count of rejections and nothing else. +func captureLog(t *testing.T) func() []observer.LoggedEntry { + core, logs := observer.New(zapcore.WarnLevel) + previous := log + log = &logging.Sugared{SugaredLogger: zap.New(core).Sugar()} + t.Cleanup(func() { log = previous }) + return func() []observer.LoggedEntry { + return logs.FilterMessageSnippet("identity detail of an existing object").All() + } +} + +func TestPreserveIdentityDetails_Relation(t *testing.T) { + t.Run("creation sets the identity details", func(t *testing.T) { + // given + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelation) + rejections := captureLog(t) + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String("author")) + s.SetDetail(bundle.RelationKeyRelationFormat, domain.Int64(int64(model.RelationFormat_object))) + s.SetDetail(bundle.RelationKeySourceObject, domain.String("_brauthor")) + + // when + require.NoError(t, fx.Apply(s)) + + // then + details := fx.Details() + assert.Equal(t, "author", details.GetString(bundle.RelationKeyRelationKey)) + assert.Equal(t, int64(model.RelationFormat_object), details.GetInt64(bundle.RelationKeyRelationFormat)) + assert.Equal(t, "_brauthor", details.GetString(bundle.RelationKeySourceObject)) + assert.Empty(t, rejections()) + }) + + t.Run("change to another value is ignored and reported", func(t *testing.T) { + // given + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelation) + applyIdentity(t, fx, "author", model.RelationFormat_object, "_brauthor") + rejections := captureLog(t) + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String("hijacked")) + s.SetDetail(bundle.RelationKeyRelationFormat, domain.Int64(int64(model.RelationFormat_longtext))) + s.SetDetail(bundle.RelationKeySourceObject, domain.String("_brhijacked")) + s.SetDetail(bundle.RelationKeyName, domain.String("Author")) + + // when + require.NoError(t, fx.Apply(s)) + + // then + details := fx.Details() + assert.Equal(t, "author", details.GetString(bundle.RelationKeyRelationKey)) + assert.Equal(t, int64(model.RelationFormat_object), details.GetInt64(bundle.RelationKeyRelationFormat)) + assert.Equal(t, "_brauthor", details.GetString(bundle.RelationKeySourceObject)) + // the rest of the state still lands, one bad detail must not cost the whole apply + assert.Equal(t, "Author", details.GetString(bundle.RelationKeyName)) + assert.Len(t, rejections(), len(identityDetails)) + }) + + t.Run("rewriting the same value is a silent no-op", func(t *testing.T) { + // given + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelation) + applyIdentity(t, fx, "author", model.RelationFormat_object, "_brauthor") + rejections := captureLog(t) + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String("author")) + s.SetDetail(bundle.RelationKeyRelationFormat, domain.Int64(int64(model.RelationFormat_object))) + s.SetDetail(bundle.RelationKeySourceObject, domain.String("_brauthor")) + + // when + require.NoError(t, fx.Apply(s)) + + // then + details := fx.Details() + assert.Equal(t, "author", details.GetString(bundle.RelationKeyRelationKey)) + assert.Equal(t, int64(model.RelationFormat_object), details.GetInt64(bundle.RelationKeyRelationFormat)) + assert.Empty(t, rejections()) + }) + + t.Run("backfill of a missing value is allowed", func(t *testing.T) { + // given - a relation that never got a sourceObject + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelation) + applyIdentity(t, fx, "author", model.RelationFormat_object, "") + rejections := captureLog(t) + s := fx.NewState() + s.SetDetail(bundle.RelationKeySourceObject, domain.String("_brauthor")) + + // when + require.NoError(t, fx.Apply(s)) + + // then + assert.Equal(t, "_brauthor", fx.Details().GetString(bundle.RelationKeySourceObject)) + assert.Empty(t, rejections()) + }) + + t.Run("longtext format is a value, not an absence", func(t *testing.T) { + // given - format 0 is longtext, so it must be defended like any other format + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelation) + applyIdentity(t, fx, "author", model.RelationFormat_longtext, "_brauthor") + rejections := captureLog(t) + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationFormat, domain.Int64(int64(model.RelationFormat_number))) + + // when + require.NoError(t, fx.Apply(s)) + + // then + assert.Equal(t, int64(model.RelationFormat_longtext), fx.Details().GetInt64(bundle.RelationKeyRelationFormat)) + assert.Len(t, rejections(), 1) + }) + + t.Run("clearing an identity detail is ignored", func(t *testing.T) { + // given + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelation) + applyIdentity(t, fx, "author", model.RelationFormat_object, "_brauthor") + rejections := captureLog(t) + s := fx.NewState() + s.RemoveDetail(bundle.RelationKeyRelationKey, bundle.RelationKeySourceObject) + + // when + require.NoError(t, fx.Apply(s)) + + // then + details := fx.Details() + assert.Equal(t, "author", details.GetString(bundle.RelationKeyRelationKey)) + assert.Equal(t, "_brauthor", details.GetString(bundle.RelationKeySourceObject)) + assert.Len(t, rejections(), 2) + }) + + t.Run("a state stacked on top of another one is compared against the tree", func(t *testing.T) { + // given - the intermediate state is where the bad value is introduced + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelation) + applyIdentity(t, fx, "author", model.RelationFormat_object, "_brauthor") + rejections := captureLog(t) + intermediate := fx.NewState() + intermediate.SetDetail(bundle.RelationKeyRelationKey, domain.String("hijacked")) + s := intermediate.NewState() + s.SetDetail(bundle.RelationKeyName, domain.String("Author")) + + // when + require.NoError(t, fx.Apply(s)) + + // then + details := fx.Details() + assert.Equal(t, "author", details.GetString(bundle.RelationKeyRelationKey)) + assert.Equal(t, "Author", details.GetString(bundle.RelationKeyName)) + assert.Len(t, rejections(), 1) + }) +} + +func TestPreserveIdentityDetails_ObjectType(t *testing.T) { + t.Run("sourceObject of an installed type can not be repointed", func(t *testing.T) { + // given + fx := newIdentityFixture(t, smartblock.SmartBlockTypeObjectType) + s := fx.NewState() + s.SetDetail(bundle.RelationKeySourceObject, domain.String("_otpage")) + require.NoError(t, fx.Apply(s)) + rejections := captureLog(t) + + // when + s = fx.NewState() + s.SetDetail(bundle.RelationKeySourceObject, domain.String("_ottask")) + require.NoError(t, fx.Apply(s)) + + // then + assert.Equal(t, "_otpage", fx.Details().GetString(bundle.RelationKeySourceObject)) + assert.Len(t, rejections(), 1) + }) +} + +func TestPreserveIdentityDetails_UnrelatedType(t *testing.T) { + t.Run("a page keeps no identity details, so sourceObject stays writable", func(t *testing.T) { + // given - duplication repoints sourceObject on ordinary objects + fx := newIdentityFixture(t, smartblock.SmartBlockTypePage) + s := fx.NewState() + s.SetDetail(bundle.RelationKeySourceObject, domain.String("template1")) + require.NoError(t, fx.Apply(s)) + rejections := captureLog(t) + + // when + s = fx.NewState() + s.SetDetail(bundle.RelationKeySourceObject, domain.String("template2")) + require.NoError(t, fx.Apply(s)) + + // then + assert.Equal(t, "template2", fx.Details().GetString(bundle.RelationKeySourceObject)) + assert.Empty(t, rejections()) + }) +} + +func TestPreserveIdentityDetails_RelationOption(t *testing.T) { + // an option's relationKey is the relation it belongs to: repointing it moves the option to + // another relation and orphans every value already written under it + t.Run("creation sets the relation the option belongs to", func(t *testing.T) { + // given + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelationOption) + rejections := captureLog(t) + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String("tag")) + s.SetDetail(bundle.RelationKeyName, domain.String("Urgent")) + + // when + require.NoError(t, fx.Apply(s)) + + // then + assert.Equal(t, "tag", fx.Details().GetString(bundle.RelationKeyRelationKey)) + assert.Empty(t, rejections()) + }) + + t.Run("moving an option to another relation is ignored", func(t *testing.T) { + // given + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelationOption) + applyOption(t, fx, "tag", "Urgent") + rejections := captureLog(t) + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String("status")) + s.SetDetail(bundle.RelationKeyName, domain.String("Renamed")) + + // when + require.NoError(t, fx.Apply(s)) + + // then - the name is the option's to change, the relation it belongs to is not + details := fx.Details() + assert.Equal(t, "tag", details.GetString(bundle.RelationKeyRelationKey)) + assert.Equal(t, "Renamed", details.GetString(bundle.RelationKeyName)) + assert.Len(t, rejections(), 1) + }) + + t.Run("rewriting the same relation is a silent no-op", func(t *testing.T) { + // given + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelationOption) + applyOption(t, fx, "tag", "Urgent") + rejections := captureLog(t) + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String("tag")) + + // when + require.NoError(t, fx.Apply(s)) + + // then + assert.Equal(t, "tag", fx.Details().GetString(bundle.RelationKeyRelationKey)) + assert.Empty(t, rejections()) + }) + + t.Run("backfill of a missing relation key is allowed", func(t *testing.T) { + // given - an option that never got a relationKey + fx := newIdentityFixture(t, smartblock.SmartBlockTypeRelationOption) + s := fx.NewState() + s.SetDetail(bundle.RelationKeyName, domain.String("Urgent")) + require.NoError(t, fx.Apply(s)) + rejections := captureLog(t) + + // when + s = fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String("tag")) + require.NoError(t, fx.Apply(s)) + + // then + assert.Equal(t, "tag", fx.Details().GetString(bundle.RelationKeyRelationKey)) + assert.Empty(t, rejections()) + }) +} + +func applyOption(t *testing.T, fx *fixture, relationKey, name string) { + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String(relationKey)) + s.SetDetail(bundle.RelationKeyName, domain.String(name)) + require.NoError(t, fx.Apply(s)) + require.Equal(t, relationKey, fx.Details().GetString(bundle.RelationKeyRelationKey)) +} + +func applyIdentity(t *testing.T, fx *fixture, relationKey string, format model.RelationFormat, sourceObject string) { + s := fx.NewState() + s.SetDetail(bundle.RelationKeyRelationKey, domain.String(relationKey)) + s.SetDetail(bundle.RelationKeyRelationFormat, domain.Int64(int64(format))) + if sourceObject != "" { + s.SetDetail(bundle.RelationKeySourceObject, domain.String(sourceObject)) + } + require.NoError(t, fx.Apply(s)) + require.Equal(t, relationKey, fx.Details().GetString(bundle.RelationKeyRelationKey)) +} diff --git a/core/block/editor/smartblock/smartblock.go b/core/block/editor/smartblock/smartblock.go index cec81ac624..97dca28dce 100644 --- a/core/block/editor/smartblock/smartblock.go +++ b/core/block/editor/smartblock/smartblock.go @@ -331,6 +331,11 @@ func (sb *smartBlock) Init(ctx *InitContext) (err error) { } sb.undo = undo.NewHistory(0) sb.restrictions = restriction.GetRestrictions(sb) + if hasIdentityDetails(sb.Type()) { + // registered here rather than in the Page/ObjectType editors so that it holds for every + // writer reaching Apply, including the ones that never go through an editor method + sb.AddHook(sb.preserveIdentityDetails, HookBeforeApply) + } if ctx.State != nil { // need to store file keys in case we have some new files in the state sb.storeFileKeys(ctx.State) diff --git a/core/block/editor/widget/widget.go b/core/block/editor/widget/widget.go index dd8644c6e9..23ce2c8c16 100644 --- a/core/block/editor/widget/widget.go +++ b/core/block/editor/widget/widget.go @@ -18,7 +18,14 @@ const ( DefaultWidgetAll = "allObjects" DefaultWidgetRecentlyOpened = "recentOpen" - widgetWrapperBlockSuffix = "-wrapper" // in case blockId is specifically provided to avoid bad tree merges + + // The chat and bin widgets are minted by the clients (anytype-ts spells + // them through its widgetId table); the ids reach the heart only inside + // stored widget objects, which is why no heart-side code creates them. + DefaultWidgetChat = "chat" + DefaultWidgetBin = "bin" + + widgetWrapperBlockSuffix = "-wrapper" // in case blockId is specifically provided to avoid bad tree merges ) @@ -54,9 +61,22 @@ func FillImportFlags(link *model.BlockContentLink, widgetFlags *ImportWidgetFlag return builtinWidget } +// IsPredefinedWidgetTargetId reports whether targetID names a built-in +// listing rather than an object — the whole inventory a widget link can +// carry, not just the four this function used to know. +// +// The gap was not cosmetic. common.handleLinkBlock leaves a link target +// alone only when this function knows it; anything else it cannot resolve +// becomes addr.MissingObject, and WidgetObject.Init then strips the link and +// its now-empty wrapper. allObjects is created by WidgetObject's own +// migration 3, chat and bin by the clients — so importing an app export +// silently lost exactly those widgets, with a log line as the only trace. +// Measured over a 77-space account: 33 of 218 widget links name a listing, +// and 29 of those named one this function did not know. func IsPredefinedWidgetTargetId(targetID string) bool { switch targetID { - case DefaultWidgetFavorite, DefaultWidgetSet, DefaultWidgetRecentlyEdited, DefaultWidgetCollection: + case DefaultWidgetFavorite, DefaultWidgetSet, DefaultWidgetRecentlyEdited, DefaultWidgetCollection, + DefaultWidgetAll, DefaultWidgetRecentlyOpened, DefaultWidgetChat, DefaultWidgetBin: return true default: return false diff --git a/core/block/editor/widget/widget_test.go b/core/block/editor/widget/widget_test.go new file mode 100644 index 0000000000..d12e3fa022 --- /dev/null +++ b/core/block/editor/widget/widget_test.go @@ -0,0 +1,27 @@ +package widget + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// The predefined-target inventory is import behaviour, not decoration: +// common.handleLinkBlock keeps a widget link target exactly when this +// function knows it, and rewrites anything else it cannot resolve to +// addr.MissingObject — after which WidgetObject.Init strips the link and its +// wrapper, losing the widget with no error. The function used to know four +// of the eight listings live spaces actually hold (allObjects comes from +// WidgetObject's own migration 3, chat and bin from the clients), so an app +// export re-imported without exactly those widgets. +func TestIsPredefinedWidgetTargetId(t *testing.T) { + for _, id := range []string{ + DefaultWidgetFavorite, DefaultWidgetSet, DefaultWidgetRecentlyEdited, DefaultWidgetCollection, + DefaultWidgetAll, DefaultWidgetRecentlyOpened, DefaultWidgetChat, DefaultWidgetBin, + } { + assert.True(t, IsPredefinedWidgetTargetId(id), id) + } + for _, id := range []string{"", "bafyreiamuhvd4f72swuxg6ejudiyfsinp56dkpr7crbnq3ulrdmvu7fryy", "page-home", "_favorite"} { + assert.False(t, IsPredefinedWidgetTargetId(id), id) + } +} diff --git a/core/block/export/anyblock/anyblock.go b/core/block/export/anyblock/anyblock.go new file mode 100644 index 0000000000..7d310c3b34 --- /dev/null +++ b/core/block/export/anyblock/anyblock.go @@ -0,0 +1,607 @@ +// Package anyblock is the native AnyBlock JSON exporter: it writes a bundle +// (SPEC.md §2c) from a live space, wiring store, cache and writer around the +// shared composition (pkg/lib/anyblockjson/compose) on top of the extracted +// collection seam (core/block/export/collect). +// +// The pipeline is the designed collect → plan → emit → finish +// (EXPORTER_DESIGN.md §1.1): collection returns the complete doc set before +// anything is written; the plan fixes every path single-threaded from +// details alone (a pure per-id function, §1.3 — no collision machinery); +// emit runs width-bounded concurrent tasks that load one object each, +// decide omission through the package predicates, marshal, write, stream +// the blob for file objects, and close the object out of the cache; finish +// writes properties.json and index.json, re-read-verified (I1 at bundle +// scope). +// +// The RPC surface is model.Export_AnyBlockJSON (design Q6, settled as its +// recommended option (a) — a new enum value, pbjson untouched): the export +// service routes that format here, hands over the doc set it has already +// collected, and supplies a Runner that turns emit into process.Queue +// tasks, so the format reports progress and answers ProcessCancel exactly +// like the five legacy ones. Driven through the Go API directly — tests, +// cmd tooling — emit runs on this package's own bounded pool instead. +package anyblock + +import ( + "bytes" + "context" + "fmt" + "io" + "path" + "runtime" + "strings" + "sync" + "sync/atomic" + + "github.com/anyproto/anytype-heart/core/block/cache" + "github.com/anyproto/anytype-heart/core/block/editor/fileobject" + sb "github.com/anyproto/anytype-heart/core/block/editor/smartblock" + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/block/editor/template" + "github.com/anyproto/anytype-heart/core/block/export/collect" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/compose" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/storeresolver" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" + "github.com/anyproto/anytype-heart/pkg/lib/logging" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/space/spacecore/typeprovider" +) + +var log = logging.Logger("anytype-mw-export-anyblock") + +// Writer is where the bundle's files land. The legacy exporter's dir and +// zip writers satisfy it structurally; DirWriter (writer.go) is the +// package's own deterministic directory form. +// +// One determinism caveat belongs to the writer, not this package: a DIR +// tree from the same space state is byte-identical file for file (the +// determinism test pins it), but a ZIP archive additionally encodes entry +// ORDER, and emit is concurrent — archive-level byte identity would need an +// ordered writer, which nothing requires yet. +type Writer interface { + WriteFile(filename string, r io.Reader, lastModifiedDate int64) error +} + +// Request describes one space's bundle export. +type Request struct { + SpaceId string + // Ids are the requested roots; empty = the whole space. + Ids []string + + IncludeNested bool + IncludeFiles bool + IncludeArchived bool + IncludeBacklinks bool + IncludeSpace bool + + // SpaceName is index.json's fallback name, used only when the space's + // own settings document states none (§2c). + SpaceName string + + // BundleRoot is a path prefix inside the writer — empty for a + // single-space export (the bundle root IS the writer root), and + // "spaces/" per bundle when a caller exports several spaces + // into one archive (design Q9: the wrapper is load-bearing — the same + // id legitimately recurs across spaces, so flattening collides). + // Manifest paths stay bundle-relative either way. + BundleRoot string + + // StateFilters filter the state of objects collected as links, exactly + // as the legacy exporter's LinksStateFilters do. + StateFilters *state.Filters + + // Runner runs the emit phase's per-document tasks. Nil = this package's + // own width-bounded pool (emitWidthFor), which is what tests and cmd + // tooling get; the export service passes a process.Queue-backed runner + // so an RPC export's progress and cancellation ride the same process + // every other format uses. + Runner EmitRunner +} + +// EmitRunner runs the emit phase's tasks to completion, bounded. A non-nil +// error means the run was ABANDONED (cancelled) rather than finished: the +// exporter then stops without writing index.json or properties.json, +// because a bundle whose index claims documents the emit never wrote is +// worse than no bundle at all. +type EmitRunner interface { + Run(ctx context.Context, tasks []func()) error +} + +// Exporter wires the pipeline's dependencies. Construct it with the same +// components the legacy export service holds; every field is required +// except Collector, which only the collecting door needs. +type Exporter struct { + // Collector runs the collection Export needs. ExportCollected takes the + // doc set from its caller instead and leaves this nil. + Collector collect.Collector + // Picker is typed CachedObjectGetter, not ObjectGetter, ON PURPOSE: + // close-after-write is the memory model (design §1.5/§1.6 — without it + // the export retains throughput × cache TTL of loaded trees), so a + // picker that cannot close is a compile error here, never a silent + // degradation behind a failed type assertion. + Picker cache.CachedObjectGetter + ObjectStore objectstore.ObjectStore + SbtProvider typeprovider.SmartBlockTypeProvider +} + +// emitWidthFor bounds the emit phase's concurrency, following the repo's +// prior art for exactly this problem — the reindex limiter +// (core/indexer/reindexlimiter.go): each task cold-builds one object into +// the space's object cache, work is storage-read-bound, so a small overlap +// keeps throughput while capping the peak; mobile gets half the slots. +// With close-after-write (below) the width IS the resident content set: +// at most this many export-loaded trees exist at any instant (design §1.6). +func emitWidthFor(goos string) int { + if goos == "ios" || goos == "android" { + return 2 + } + return 4 +} + +// EmitWidth is emitWidthFor on the running platform — what a caller that +// supplies its own Runner should bound that runner by, so the resident +// content set stays what §1.6 measured whichever pool actually runs emit. +func EmitWidth() int { + return emitWidthFor(runtime.GOOS) +} + +// boundedRunner is this package's own emit: a fixed-width pool of workers +// over one channel — the default when the caller supplies no Runner. It is +// the only Runner that watches ctx directly; the producer stops FEEDING on +// cancellation rather than only letting tasks skip, so nothing queued +// behind the cancellation is ever handed out. +type boundedRunner struct { + width int +} + +func (r boundedRunner) Run(ctx context.Context, tasks []func()) error { + todo := make(chan func()) + var wg sync.WaitGroup + for range r.width { + wg.Add(1) + go func() { + defer wg.Done() + for t := range todo { + t() + } + }() + } + for _, t := range tasks { + select { + case todo <- t: + case <-ctx.Done(): + // tasks already in flight run to completion: each holds a loaded + // object and possibly a half-written file, and unwinding that is + // strictly worse than the seconds it costs to let them land + close(todo) + wg.Wait() + return ctx.Err() + } + } + close(todo) + wg.Wait() + return ctx.Err() +} + +// resolverPool lends one resolver set to each emit task for that task's +// duration. storeresolver.Resolvers is not safe for concurrent use, and its +// per-instance caches (the space's relation snapshot, participant names, +// referenced-object rows) are what keep emit from re-reading the store for +// every document — so sets are RECYCLED rather than minted per task: the +// pool holds at most one per concurrently running task, each already warm. +// Ownership is per task and not per worker because a Runner owns its own +// goroutines and may run any task on any of them. +type resolverPool struct { + mu sync.Mutex + free []anyblockjson.Options + mint func() anyblockjson.Options +} + +func (p *resolverPool) get() anyblockjson.Options { + p.mu.Lock() + if n := len(p.free); n > 0 { + opts := p.free[n-1] + p.free = p.free[:n-1] + p.mu.Unlock() + return opts + } + p.mu.Unlock() + return p.mint() +} + +func (p *resolverPool) put(opts anyblockjson.Options) { + p.mu.Lock() + p.free = append(p.free, opts) + p.mu.Unlock() +} + +// Result is what one export can say about itself — nothing a caller might +// act on is dropped into a log line alone. +type Result struct { + // Succeed counts the documents accounted for: written, or omitted into + // the bundle files. + Succeed int + // DocErrors counts documents that failed to emit at all (load, marshal + // or write) — logged and skipped, the legacy exporter's own per-doc + // discipline, but COUNTED, so a caller can tell a clean backup from a + // holed one. + DocErrors int + // BlobErrors counts file objects whose DOCUMENT was written but whose + // bytes could not be streamed (node offline, blocks missing). The + // document still travels — metadata is strictly more than the nothing a + // failed doc leaves — and the manifest simply omits the binding, which + // the bundle tooling then surfaces (a file document unbound by a + // present map is a warning, §2c). A FAT bundle with BlobErrors > 0 is + // not the faithful byte carrier it promises to be; the caller decides + // whether that fails the operation. + BlobErrors int +} + +// Export runs the collection this format wants and writes the bundle +// through wr: every collected document at its planned path, blobs beside +// their file documents, then properties.json and index.json at the bundle +// root. +func (e *Exporter) Export(ctx context.Context, req Request, wr Writer) (res Result, err error) { + docs, err := e.Collector.Collect(ctx, CollectRequest(req)) + if err != nil { + return res, fmt.Errorf("collect docs for export: %w", err) + } + return e.ExportCollected(ctx, req, docs, wr) +} + +// CollectRequest is the collection this format runs: the derived closure +// (design §1.1), with the request's own flags. Exported so a caller that +// collects for itself — the export service does, before it knows which +// writer the RPC wants — can ask for the same set instead of guessing at +// it, and so the two spellings cannot drift apart. +func CollectRequest(req Request) collect.Request { + return collect.Request{ + SpaceId: req.SpaceId, + Ids: req.Ids, + Closure: collect.ClosureDerived, + IncludeNested: req.IncludeNested, + IncludeFiles: req.IncludeFiles, + IncludeArchived: req.IncludeArchived, + IncludeBacklinks: req.IncludeBacklinks, + IncludeSpace: req.IncludeSpace, + StateFilters: req.StateFilters, + } +} + +// ExportCollected writes the bundle from a doc set the caller collected — +// docs MUST be what CollectRequest(req) returns, since every phase below +// treats it as the complete, closed set (the plan names its members, the +// collection filter drops references outside it). The export service takes +// this door: it runs the same ClosureDerived collection for every format +// before the writer exists (closureForFormat, export.go), and re-collecting +// here would query the whole space a second time. +func (e *Exporter) ExportCollected(ctx context.Context, req Request, docs collect.Docs, wr Writer) (res Result, err error) { + // plan: details only, single-threaded, before the first emit task + // (design §1.1). Excluded rows are dropped here by the same rule every + // emitter applies. + metas := make([]compose.DocMeta, 0, len(docs)) + emitIds := make([]string, 0, len(docs)) + for id, doc := range docs { + if collect.Excluded(doc.Details) { + continue + } + sbType, sbtErr := e.SbtProvider.Type(req.SpaceId, id) + if sbtErr != nil { + log.With("objectId", id).Errorf("failed to get smartblock type: %v", sbtErr) + continue + } + metas = append(metas, compose.DocMeta{ + Id: id, + SbType: sbType.ToProto(), + FileExt: doc.Details.GetString(bundle.RelationKeyFileExt), + FileMime: doc.Details.GetString(bundle.RelationKeyFileMimeType), + }) + emitIds = append(emitIds, id) + } + plan, err := compose.BuildPlan(req.SpaceId, metas) + if err != nil { + return res, fmt.Errorf("build path plan: %w", err) + } + + // the composer gets a DEDICATED resolver set: storeresolver.Resolvers + // is not safe for concurrent use, and the composer consults its options + // only under its own mutex — sharing an instance with a worker would + // race (compose.NewComposer's contract). + composer := compose.NewComposer(storeresolver.New(e.ObjectStore.SpaceIndex(req.SpaceId)).Options(), req.SpaceName) + + // emit: width-bounded tasks, each holding one resolver set for its + // duration. The output cannot depend on scheduling: every path was fixed + // by the plan, the composer's aggregates are commutative, and finish + // sorts (§1.5). + var succeedAsync, docErrs, blobErrs int64 + pool := &resolverPool{mint: func() anyblockjson.Options { + return storeresolver.New(e.ObjectStore.SpaceIndex(req.SpaceId)).Options() + }} + tasks := make([]func(), 0, len(emitIds)) + for _, id := range emitIds { + tasks = append(tasks, func() { + // a cancelled export must stop COLD-LOADING objects, which is + // the expensive half of emit — an account-sized bundle otherwise + // keeps building trees long after the user said stop. The check + // is per task rather than only in the producer because a Runner + // may already hold every task (the queue-backed one does). + if ctx.Err() != nil { + return + } + opts := pool.get() + defer pool.put(opts) + blobFailed, werr := e.emitDoc(ctx, req, docs, plan, composer, opts, wr, id) + if blobFailed { + atomic.AddInt64(&blobErrs, 1) + } + if werr != nil { + log.With("objectID", id).Warnf("can't export doc: %v", werr) + atomic.AddInt64(&docErrs, 1) + } else { + atomic.AddInt64(&succeedAsync, 1) + } + }) + } + runner := req.Runner + if runner == nil { + runner = boundedRunner{width: EmitWidth()} + } + runErr := runner.Run(ctx, tasks) + if runErr == nil { + // a runner that ran every task can still have been abandoned: the + // queue-backed one does not watch ctx, its tasks simply skip + runErr = ctx.Err() + } + res = Result{Succeed: int(succeedAsync), DocErrors: int(docErrs), BlobErrors: int(blobErrs)} + if runErr != nil { + // no bundle files: index.json states what the bundle holds, and half + // an emit holds something nobody measured + return res, fmt.Errorf("emit documents: %w", runErr) + } + if res.BlobErrors > 0 { + log.Errorf("export %s: %d file blob(s) could not be streamed; their documents travel without bytes and the manifest omits the bindings", req.SpaceId, res.BlobErrors) + } + + // finish: the two bundle files, re-read-verified by the composer (I1 + // at bundle scope). Nil bytes = nothing was written, nothing to state. + index, properties, _, err := composer.Finish() + if err != nil { + return res, fmt.Errorf("compose bundle files: %w", err) + } + if properties != nil { + if err := wr.WriteFile(path.Join(req.BundleRoot, anyblockjson.PropertiesFileName), bytes.NewReader(properties), 0); err != nil { + return res, fmt.Errorf("write property dictionary: %w", err) + } + } + if index != nil { + if err := wr.WriteFile(path.Join(req.BundleRoot, anyblockjson.IndexFileName), bytes.NewReader(index), 0); err != nil { + return res, fmt.Errorf("write index: %w", err) + } + } + return res, nil +} + +// emitDoc is one emit task: load, decide omission, marshal, write, stream +// the blob, observe — then close the object out of the cache. A blob +// stream failure does NOT fail the document: the document is already +// written and carries strictly more than the nothing a failed doc leaves, +// so the failure is reported through blobFailed (and the manifest omits +// the binding) rather than by undoing the doc. +func (e *Exporter) emitDoc(ctx context.Context, req Request, docs collect.Docs, plan *compose.Plan, + composer *compose.Composer, opts anyblockjson.Options, wr Writer, id string) (blobFailed bool, _ error) { + + err := cache.Do(e.Picker, id, func(b sb.SmartBlock) error { + st := b.NewState() + if st.CombinedDetails().GetBool(bundle.RelationKeyIsDeleted) { + return nil + } + st = st.Copy().Filter(stateFilters(req, docs, id)) + if isCollection(st) { + collectionFilterMissing(st, docs) + } + + sbType := b.Type().ToProto() + base := snapshotBase(st) + + // omission is decided HERE, on the loaded snapshot — the predicates + // take the base, so they cannot run at plan time (design §1.1). An + // omitted document's facts were lifted into the composer; a planned + // name going unused does not disturb determinism, since omission is + // itself a deterministic function of state. Issues mean the lift + // failed to account for something — a bug worth logging loudly, but + // the export carries on (the fail-closed predicates keep the + // document in every doubtful case, so an issue here is belt over + // braces). + omitted, issues := composer.Observe(sbType, base) + for _, is := range issues { + log.With("objectID", id).Errorf("bundle composition %s: %s", is.Category, is.Detail) + } + if omitted { + return nil + } + + data, err := anyblockjson.Marshal(sbType, base, opts) + if err != nil { + return fmt.Errorf("marshal document: %w", err) + } + docPath, ok := plan.DocPath(id) + if !ok { + return fmt.Errorf("no planned path for %s", id) + } + lastModifiedDate := st.LocalDetails().GetInt64(bundle.RelationKeyLastModifiedDate) + if err := wr.WriteFile(path.Join(req.BundleRoot, docPath), bytes.NewReader(data), lastModifiedDate); err != nil { + return fmt.Errorf("write document: %w", err) + } + if err := composer.ObserveWritten(sbType, base, data, docPath); err != nil { + return fmt.Errorf("observe written document: %w", err) + } + + // the blob, adjacent to its document (§1.4): streamed, never + // buffered, and bound by the manifest `files` map — the document + // itself carries no path, and `source` keeps meaning what its + // relation says it means (the legacy clobber does not carry over) + if req.IncludeFiles && b.Type() == smartblock.SmartBlockTypeFileObject { + blobPath, ok := plan.BlobPath(id) + if !ok { + return fmt.Errorf("no planned blob path for %s", id) + } + fullBlobPath := path.Join(req.BundleRoot, blobPath) + if err := e.saveBlob(ctx, wr, b, fullBlobPath); err != nil { + blobFailed = true + log.With("objectID", id).Warnf("file blob not streamed, document travels without bytes: %v", err) + // a PARTIAL blob is worse than none — truncated bytes a + // reader may trust — so a writer that can un-write gets the + // chance. A zip writer cannot (entries are streamed); there + // the partial stays, unbound by the manifest, and the + // tooling's orphan check flags it. + if remover, ok := wr.(interface{ RemoveFile(string) error }); ok { + if rerr := remover.RemoveFile(fullBlobPath); rerr != nil { + log.With("objectID", id).Warnf("partial blob not removed: %v", rerr) + } + } + return nil + } + composer.ObserveFileBlob(id, blobPath) + } + return nil + }) + + // Close after write — active, immediate, TTL-independent (design §1.5): + // an object closes iff nobody else has it open, so the resident content + // set stays ≈ the emit width instead of throughput × cache TTL, which + // is the whole memory model (design §1.6). Log-only on failure, the + // fulltext indexer's own discipline (core/indexer/fulltext.go:348) — + // the passive TTL collects whatever this call could not. + // + // RELEASE GATE (design Q11): this path reaches the filed ocache + // TryRemove-on-loading bug GO-7333 — nil-deref/hang/race when another + // caller concurrently re-loads the same entry. The fix is any-sync PR + // https://github.com/anyproto/any-sync/pull/769 (open, green CI): the + // any-sync bump carrying it must land before this exporter ships. Our + // own entry is loaded, not loading (cache.Do returned above), so the + // window arms only on a concurrent re-load — the identical race the + // fulltext indexer has soaked in production on every indexed object. + if _, cerr := e.Picker.TryRemoveFromCache(ctx, id); cerr != nil { + log.With("objectID", id).Warnf("object cache remove: %v", cerr) + } + return blobFailed, err +} + +// snapshotBase builds the snapshot both doors render from: the same shape +// the pb converter feeds the wire (core/converter/pbc), which is also the +// shape the corpus sweep verified the codec against — 38k documents, 34 +// known failures. +func snapshotBase(st *state.State) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: st.BlocksToSave(), + Details: st.CombinedDetails().ToProto(), + ObjectTypes: domain.MarshalTypeKeys(st.ObjectTypeKeys()), + Collections: st.Store(), + Key: st.UniqueKeyInternal(), + FileInfo: st.GetFileInfo().ToModel(), + } +} + +// ExportDocument renders ONE object as a standalone document: no bundle, no +// index.json, no property dictionary. This is what ExportSingleInMemory +// serves for the format (design Q7's position) and what rule 7 of the +// format's principles allows — a document stands alone, carrying its own +// property names and formats, so the two bundle files have nothing to add +// about a single object. +// +// No omission predicate runs here either: those exist to keep documents +// whose content the BUNDLE restates (the space's own object, installed +// bundled relations) out of that bundle. With no bundle to restate +// anything, refusing to render the object a caller named by id would just +// be an empty answer. +// +// Unlike emit, this does not close the object out of the cache afterwards: +// close-after-write pays for itself across thousands of documents (§1.6), +// while a single-document caller is typically exporting the object the user +// is looking at, where an eviction only buys the next reader a cold load. +func (e *Exporter) ExportDocument(ctx context.Context, spaceId, objectId string) ([]byte, error) { + opts := storeresolver.New(e.ObjectStore.SpaceIndex(spaceId)).Options() + var data []byte + err := cache.Do(e.Picker, objectId, func(b sb.SmartBlock) error { + st := b.NewState() + if st.CombinedDetails().GetBool(bundle.RelationKeyIsDeleted) { + return fmt.Errorf("object is deleted") + } + out, err := anyblockjson.Marshal(b.Type().ToProto(), snapshotBase(st), opts) + if err != nil { + return fmt.Errorf("marshal document: %w", err) + } + data = out + return nil + }) + if err != nil { + return nil, fmt.Errorf("render document %s: %w", objectId, err) + } + return data, nil +} + +// saveBlob streams one file object's bytes to the writer — the legacy +// saveFile minus the two things this format deletes: the namer (the blob +// path is the plan's pure function of the id) and the Source-detail clobber +// (the manifest binds instead, §1.4). +func (e *Exporter) saveBlob(ctx context.Context, wr Writer, b sb.SmartBlock, blobPath string) error { + fileObject, ok := b.(fileobject.FileObject) + if !ok { + return fmt.Errorf("object is not a file object") + } + file, err := fileObject.GetFile() + if err != nil { + return fmt.Errorf("get file: %w", err) + } + if strings.HasPrefix(file.MimeType(), "image") { + image, err := fileObject.GetImage() + if err != nil { + return fmt.Errorf("get image: %w", err) + } + file, err = image.GetOriginalFile() + if err != nil { + return fmt.Errorf("get original file: %w", err) + } + } + rd, err := file.Reader(ctx) + if err != nil { + return fmt.Errorf("open file reader: %w", err) + } + if err := wr.WriteFile(blobPath, rd, file.LastModifiedDate()); err != nil { + return fmt.Errorf("write file blob: %w", err) + } + return nil +} + +// stateFilters mirrors the legacy exporter's rule: only objects that +// entered the collection as LINKS render filtered. +func stateFilters(req Request, docs collect.Docs, id string) *state.Filters { + if doc, ok := docs[id]; ok && doc.IsLink { + return req.StateFilters + } + return nil +} + +// collectionFilterMissing drops collection members the export does not +// carry, exactly as the legacy exporter does: a collection referencing an +// object outside the bundle would dangle on import. +func collectionFilterMissing(st *state.State, docs collect.Docs) { + collectionIds := st.GetStoreSlice(template.CollectionStoreKey) + existingIds := make([]string, 0, len(collectionIds)) + for _, item := range collectionIds { + if _, exists := docs[item]; exists { + existingIds = append(existingIds, item) + } + } + if len(existingIds) != len(collectionIds) { + st.UpdateStoreSlice(template.CollectionStoreKey, existingIds) + } +} + +func isCollection(st state.Doc) bool { + return st.CombinedDetails().GetInt64(bundle.RelationKeyResolvedLayout) == int64(model.ObjectType_collection) +} diff --git a/core/block/export/anyblock/anyblock_test.go b/core/block/export/anyblock/anyblock_test.go new file mode 100644 index 0000000000..5a32b213c3 --- /dev/null +++ b/core/block/export/anyblock/anyblock_test.go @@ -0,0 +1,531 @@ +package anyblock_test + +// anyblock_test.go drives the native exporter end to end over the store +// fixture: the real collection layer (export.New's Collect), the real plan, +// emit and composition, into a real directory tree. The headline test is +// determinism — export the same space twice, compare trees byte for byte — +// which is the property the whole §1.3 naming decision exists to guarantee, +// proved rather than asserted. +// +// It is an EXTERNAL test package because it builds the real export service +// for that collection seam, and package export now routes +// model.Export_AnyBlockJSON back into this package — an in-package test +// would close that import cycle. Nothing here needs unexported access. + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/anyproto/any-sync/app" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" + + "github.com/anyproto/anytype-heart/core/anytype/account/mock_account" + "github.com/anyproto/anytype-heart/core/block/cache/mock_cache" + editorsb "github.com/anyproto/anytype-heart/core/block/editor/fileobject" + "github.com/anyproto/anytype-heart/core/block/editor/fileobject/mock_fileobject" + "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/export" + "github.com/anyproto/anytype-heart/core/block/export/anyblock" + "github.com/anyproto/anytype-heart/core/block/process" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/core/event/mock_event" + "github.com/anyproto/anytype-heart/core/files/mock_files" + "github.com/anyproto/anytype-heart/core/notifications/mock_notifications" + "github.com/anyproto/anytype-heart/core/relationutils/mock_relationutils" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/storeresolver" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/space/mock_space" + "github.com/anyproto/anytype-heart/space/spacecore/typeprovider/mock_typeprovider" + "github.com/anyproto/anytype-heart/tests/testutil" +) + +const spaceId = "space1" + +// closingPicker wraps the object-getter mock with the TryRemoveFromCache +// the Exporter's typed Picker demands, recording every close so the tests +// can prove close-after-write actually runs — the memory model (design +// §1.5/§1.6) is a claim about this call being made, and a fixture that +// cannot observe it would leave the whole path uncovered. +type closingPicker struct { + *mock_cache.MockObjectGetterComponent + mu sync.Mutex + removed map[string]int +} + +func (p *closingPicker) TryRemoveFromCache(_ context.Context, objectId string) (bool, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.removed == nil { + p.removed = map[string]int{} + } + p.removed[objectId]++ + return true, nil +} + +func (p *closingPicker) removedIds() map[string]int { + p.mu.Lock() + defer p.mu.Unlock() + out := make(map[string]int, len(p.removed)) + for k, v := range p.removed { + out[k] = v + } + return out +} + +type fixture struct { + exporter *anyblock.Exporter + store *objectstore.StoreFixture + picker *closingPicker + provider *mock_typeprovider.MockSmartBlockTypeProvider +} + +// newFixture builds the REAL export service for its Collect seam (the same +// app wiring core/publish's tests use), so this package's tests run the +// production collection rather than a stub of it. +func newFixture(t *testing.T) *fixture { + storeFixture := objectstore.NewStoreFixture(t) + objectGetter := mock_cache.NewMockObjectGetterComponent(t) + provider := mock_typeprovider.NewMockSmartBlockTypeProvider(t) + + fetcher := mock_relationutils.NewMockRelationFormatFetcher(t) + fetcher.EXPECT().GetRelationFormatByKey(mock.Anything, mock.Anything).RunAndReturn( + func(_ string, key domain.RelationKey) (model.RelationFormat, error) { + rel, err := bundle.GetRelation(key) + if err != nil { + return 0, err + } + return rel.Format, nil + }).Maybe() + + picker := &closingPicker{MockObjectGetterComponent: objectGetter} + + a := &app.App{} + a.Register(storeFixture) + a.Register(testutil.PrepareMock(context.Background(), a, mock_event.NewMockSender(t))) + // the CLOSING picker is what the app holds, not the bare getter mock: + // export.Init resolves cache.CachedObjectGetter (the service's own + // picker field is typed that way now), which only the wrapper answers + testutil.PrepareMock(context.Background(), a, objectGetter) + a.Register(picker) + a.Register(process.New()) + a.Register(testutil.PrepareMock(context.Background(), a, mock_space.NewMockService(t))) + a.Register(testutil.PrepareMock(context.Background(), a, provider)) + a.Register(testutil.PrepareMock(context.Background(), a, mock_files.NewMockService(t))) + a.Register(testutil.PrepareMock(context.Background(), a, mock_account.NewMockService(t))) + a.Register(testutil.PrepareMock(context.Background(), a, mock_notifications.NewMockNotifications(t))) + a.Register(testutil.PrepareMock(context.Background(), a, fetcher)) + + exp := export.New() + require.NoError(t, exp.Init(a)) + + return &fixture{ + exporter: &anyblock.Exporter{ + Collector: exp, + Picker: picker, + ObjectStore: storeFixture, + SbtProvider: provider, + }, + store: storeFixture, + picker: picker, + provider: provider, + } +} + +func setupObject(id, typeId string, sbType smartblock.SmartBlockType, details map[domain.RelationKey]domain.Value) *smarttest.SmartTest { + smartBlockTest := smarttest.New(id) + if details == nil { + details = map[domain.RelationKey]domain.Value{} + } + details[bundle.RelationKeyId] = domain.String(id) + details[bundle.RelationKeyType] = domain.String(typeId) + doc := smartBlockTest.NewState().SetDetails(domain.NewDetailsFromMap(details)) + doc.AddBundledRelationLinks(maps.Keys(details)...) + smartBlockTest.Doc = doc + smartBlockTest.SetType(sbType) + return smartBlockTest +} + +// setupSpace seeds one small space — a named page and its custom type — in +// both the store fixture and the object mocks, and returns the export +// request that covers it. +func setupSpace(t *testing.T, fx *fixture) anyblock.Request { + const ( + objectId = "objectId" + typeId = "customObjectType" + ) + uk, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, typeId) + require.NoError(t, err) + + fx.store.AddObjects(t, spaceId, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String(objectId), + bundle.RelationKeyType: domain.String(typeId), + bundle.RelationKeyName: domain.String("Root page"), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + { + bundle.RelationKeyId: domain.String(typeId), + bundle.RelationKeyUniqueKey: domain.String(uk.Marshal()), + bundle.RelationKeyName: domain.String("Custom type"), + bundle.RelationKeyLayout: domain.Int64(int64(model.ObjectType_objectType)), + bundle.RelationKeySpaceId: domain.String(spaceId), + bundle.RelationKeyType: domain.String(typeId), + }, + }) + + page := setupObject(objectId, typeId, smartblock.SmartBlockTypePage, map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("Root page"), + }) + objectType := setupObject(typeId, typeId, smartblock.SmartBlockTypeObjectType, map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("Custom type"), + bundle.RelationKeyUniqueKey: domain.String(uk.Marshal()), + }) + fx.picker.EXPECT().GetObject(mock.Anything, objectId).Return(page, nil) + fx.picker.EXPECT().GetObject(mock.Anything, typeId).Return(objectType, nil) + + fx.provider.EXPECT().Type(spaceId, objectId).Return(smartblock.SmartBlockTypePage, nil) + fx.provider.EXPECT().Type(spaceId, typeId).Return(smartblock.SmartBlockTypeObjectType, nil) + + return anyblock.Request{SpaceId: spaceId, SpaceName: "Fixture space", IncludeArchived: true} +} + +// readTree reads every file below root into path → content bytes. +func readTree(t *testing.T, root string) map[string]string { + out := map[string]string{} + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + out[filepath.ToSlash(rel)] = string(data) + return nil + }) + require.NoError(t, err) + return out +} + +// The bundle a space exports to: every document at its planned id path, +// index.json carrying the manifest, properties.json beside it — and every +// file the exporter wrote reads back through the package's own Unmarshal +// (I1, at both scopes). +// +// How this can fail: route a kind to the wrong directory (the path +// assertions go red); let the manifest carry an absolute or writer-rooted +// path (the manifest assertion catches the bundle-relative contract +// breaking); or write a document the codec refuses (the Unmarshal loop +// finds it at export time, which is the whole point of I1). +func TestExporter_WritesABundle(t *testing.T) { + // given + fx := newFixture(t) + req := setupSpace(t, fx) + dir := t.TempDir() + wr, err := anyblock.NewDirWriter(dir) + require.NoError(t, err) + + // when + result, err := fx.exporter.Export(context.Background(), req, wr) + + // then + require.NoError(t, err) + assert.Equal(t, anyblock.Result{Succeed: 2}, result) + + // close-after-write ran for every emitted object — the memory model is + // this call being made (design §1.5), proved rather than assumed + removed := fx.picker.removedIds() + assert.Contains(t, removed, "objectId") + assert.Contains(t, removed, "customObjectType") + + tree := readTree(t, dir) + require.Contains(t, tree, "objects/objectId.anyblock.json") + require.Contains(t, tree, "types/customObjectType.anyblock.json") + require.Contains(t, tree, anyblockjson.IndexFileName) + require.Contains(t, tree, anyblockjson.PropertiesFileName) + + idx, err := anyblockjson.UnmarshalIndex([]byte(tree[anyblockjson.IndexFileName])) + require.NoError(t, err) + assert.Equal(t, "Fixture space", idx.Name, "no space document in this fixture, so the request name is the fallback") + require.NotNil(t, idx.Manifest) + assert.Equal(t, map[string]string{"customObjectType": "types/customObjectType.anyblock.json"}, idx.Manifest.Types) + assert.Equal(t, anyblockjson.PropertiesFileName, idx.Manifest.Properties) + + _, err = anyblockjson.UnmarshalPropertyDictionary([]byte(tree[anyblockjson.PropertiesFileName])) + require.NoError(t, err) + + opts := storeresolver.New(fx.store.SpaceIndex(spaceId)).Options() + for path, content := range tree { + if path == anyblockjson.IndexFileName || path == anyblockjson.PropertiesFileName { + continue + } + _, _, err := anyblockjson.Unmarshal([]byte(content), opts) + require.NoError(t, err, "document %s must read back through the codec (I1)", path) + } +} + +// Composing the same space twice produces byte-identical trees. This is the +// property the §1.3 naming decision exists to guarantee — every path a pure +// function of the id, no collision machinery, nothing first-writer-wins — +// and the §1.5 concurrency design promises it survives the width-bounded +// emit. Proved here with a test, not asserted in a comment. +// +// How this can fail: reintroduce anything ordering-sensitive — a namer with +// a dedup counter (the legacy namer's rand.Int63n suffix is the exhibit), a +// composer aggregate the finish does not sort, a timestamp in file CONTENT +// — and the two trees diverge. +func TestExporter_SameSpaceTwiceIsByteIdentical(t *testing.T) { + // given + fx := newFixture(t) + req := setupSpace(t, fx) + + runExport := func(t *testing.T) map[string]string { + dir := t.TempDir() + wr, err := anyblock.NewDirWriter(dir) + require.NoError(t, err) + result, err := fx.exporter.Export(context.Background(), req, wr) + require.NoError(t, err) + require.Equal(t, anyblock.Result{Succeed: 2}, result) + return readTree(t, dir) + } + + // when + first := runExport(t) + second := runExport(t) + + // then + require.ElementsMatch(t, maps.Keys(first), maps.Keys(second), "same file set") + for path, content := range first { + assert.Equal(t, content, second[path], "file %s must be byte-identical across runs", path) + } +} + +// The blob path plan and the writer's containment guard, at the wiring +// level: a path the plan did not mint may not escape the root. +func TestDirWriter_RefusesEscape(t *testing.T) { + dir := t.TempDir() + wr, err := anyblock.NewDirWriter(filepath.Join(dir, "bundle")) + require.NoError(t, err) + err = wr.WriteFile("../outside.txt", bytesReader("x"), 0) + require.Error(t, err) +} + +func bytesReader(s string) *os.File { + f, _ := os.CreateTemp("", "anyblocktest") + f.WriteString(s) + f.Seek(0, 0) + return f +} + +// fileObjectWrapper marries a smarttest object with a mocked file +// component, the same pattern core/publish's tests use. +type fileObjectWrapper struct { + *smarttest.SmartTest + editorsb.FileObject +} + +// A file object exports as BOTH halves, adjacent in files/ — the document +// at its id path, the bytes beside it under the sanitized extension — and +// the manifest `files` map is what binds them (§2c, v0.47). The stored +// file_ext here is deliberately EMPTY (431 corpus files), so the extension +// falls back to the mime table; and the document must NOT grow a path +// member — the Source clobber does not carry over. +// +// How this can fail: write the blob before it streams cleanly and observe +// it anyway (the manifest points at bytes that are not there); reintroduce +// the Source detail (the marshalled document diff shows an archive path in +// a user relation); or plan the blob from the sanitized extension but bind +// nothing (the blob is orphaned the moment the layout convention changes). +func TestExporter_StreamsBlobsAndBindsThemInTheManifest(t *testing.T) { + // given + fx := newFixture(t) + const fileId = "fileObjectId" + fx.store.AddObjects(t, spaceId, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String(fileId), + bundle.RelationKeyName: domain.String("notes"), + bundle.RelationKeyFileExt: domain.String(""), // the corpus's commonest dirt: no extension at all + bundle.RelationKeyFileMimeType: domain.String("text/plain"), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + }) + + fileSb := setupObject(fileId, "", smartblock.SmartBlockTypeFileObject, map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("notes"), + }) + blob, err := os.CreateTemp(t.TempDir(), "blob") + require.NoError(t, err) + _, err = blob.WriteString("file bytes travel") + require.NoError(t, err) + _, err = blob.Seek(0, 0) + require.NoError(t, err) + + fileData := mock_files.NewMockFile(t) + fileData.EXPECT().MimeType().Return("text/plain") + fileData.EXPECT().Reader(mock.Anything).Return(blob, nil) + fileData.EXPECT().LastModifiedDate().Return(int64(1700000000)) + fileComponent := mock_fileobject.NewMockFileObject(t) + fileComponent.EXPECT().GetFile().Return(fileData, nil) + + fx.picker.EXPECT().GetObject(mock.Anything, fileId). + Return(&fileObjectWrapper{SmartTest: fileSb, FileObject: fileComponent}, nil) + fx.provider.EXPECT().Type(spaceId, fileId).Return(smartblock.SmartBlockTypeFileObject, nil) + + dir := t.TempDir() + wr, err := anyblock.NewDirWriter(dir) + require.NoError(t, err) + + // when + result, err := fx.exporter.Export(context.Background(), + anyblock.Request{SpaceId: spaceId, SpaceName: "Fixture space", IncludeArchived: true, IncludeFiles: true}, wr) + + // then + require.NoError(t, err) + assert.Equal(t, anyblock.Result{Succeed: 1}, result) + + tree := readTree(t, dir) + require.Contains(t, tree, "files/fileObjectId.anyblock.json", "the document half") + require.Contains(t, tree, "files/fileObjectId.txt", "the bytes, same stem, mime-derived extension") + assert.Equal(t, "file bytes travel", tree["files/fileObjectId.txt"]) + assert.NotContains(t, tree["files/fileObjectId.anyblock.json"], "files/fileObjectId.txt", + "a document member is not a slot for archive bookkeeping (§1.4)") + + idx, err := anyblockjson.UnmarshalIndex([]byte(tree[anyblockjson.IndexFileName])) + require.NoError(t, err) + require.NotNil(t, idx.Manifest) + assert.Equal(t, map[string]string{fileId: "files/fileObjectId.txt"}, idx.Manifest.Files, + "the manifest map is the binding a reader may rely on") +} + +// A blob the node cannot serve does not fail the document: the document is +// already written and carries strictly more than the nothing a failed doc +// leaves. The failure is COUNTED — Result.BlobErrors — and the manifest +// omits the binding, which is what the bundle tooling's unbound-file +// warning then surfaces (§2c). +// +// How this can fail: return the stream error from the emit task (the doc +// counts as failed while sitting written in the bundle — the old +// behaviour); or bind the blob before the stream succeeds (the manifest +// points at bytes that are not there, the exact promise CheckManifestFiles +// refuses). +func TestExporter_ABlobFailureIsCountedNotFatal(t *testing.T) { + // given — a file object whose file component cannot serve bytes + fx := newFixture(t) + const fileId = "brokenFileId" + fx.store.AddObjects(t, spaceId, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String(fileId), + bundle.RelationKeyName: domain.String("gone"), + bundle.RelationKeyFileMimeType: domain.String("image/png"), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + }) + fileSb := setupObject(fileId, "", smartblock.SmartBlockTypeFileObject, map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("gone"), + }) + fileComponent := mock_fileobject.NewMockFileObject(t) + fileComponent.EXPECT().GetFile().Return(nil, os.ErrNotExist) + fx.picker.EXPECT().GetObject(mock.Anything, fileId). + Return(&fileObjectWrapper{SmartTest: fileSb, FileObject: fileComponent}, nil) + fx.provider.EXPECT().Type(spaceId, fileId).Return(smartblock.SmartBlockTypeFileObject, nil) + + dir := t.TempDir() + wr, err := anyblock.NewDirWriter(dir) + require.NoError(t, err) + + // when + result, err := fx.exporter.Export(context.Background(), + anyblock.Request{SpaceId: spaceId, SpaceName: "Fixture space", IncludeArchived: true, IncludeFiles: true}, wr) + + // then — the document travels, the failure is counted, nothing binds + require.NoError(t, err) + assert.Equal(t, anyblock.Result{Succeed: 1, BlobErrors: 1}, result) + tree := readTree(t, dir) + require.Contains(t, tree, "files/brokenFileId.anyblock.json", "the document half still travels") + idx, err := anyblockjson.UnmarshalIndex([]byte(tree[anyblockjson.IndexFileName])) + require.NoError(t, err) + require.NotNil(t, idx.Manifest) + assert.Empty(t, idx.Manifest.Files, "no binding for bytes that did not travel") +} + +// failAfterReader serves a few bytes and then errors — the shape of a +// node that stops serving blocks mid-file ("failed to fetch all nodes", +// seen live on the corpus sweep). +type failAfterReader struct{ served bool } + +func (r *failAfterReader) Read(p []byte) (int, error) { + if r.served { + return 0, os.ErrDeadlineExceeded + } + r.served = true + return copy(p, []byte("truncated")), nil +} +func (r *failAfterReader) Seek(offset int64, whence int) (int64, error) { return 0, nil } +func (r *failAfterReader) Close() error { return nil } + +// A stream that dies MID-COPY leaves no partial blob behind: truncated +// bytes a reader may trust are worse than absent bytes, so the writer's +// cleanup hook removes what the failed copy wrote. Caught live: the corpus +// files-on sweep left files/…jpg truncated on disk, flagged only by the +// orphan check. +// +// How this can fail: skip the RemoveFile hook (the truncated blob ships, +// unbound, and every downstream reader that ignores the manifest trusts +// it); or bind the blob before the copy finishes (the manifest points at +// truncated bytes, which is strictly worse). +func TestExporter_AMidStreamFailureLeavesNoPartialBlob(t *testing.T) { + fx := newFixture(t) + const fileId = "truncatedFileId" + fx.store.AddObjects(t, spaceId, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String(fileId), + bundle.RelationKeyName: domain.String("cut"), + bundle.RelationKeyFileExt: domain.String("jpg"), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + }) + fileSb := setupObject(fileId, "", smartblock.SmartBlockTypeFileObject, map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("cut"), + }) + fileData := mock_files.NewMockFile(t) + fileData.EXPECT().MimeType().Return("application/octet-stream") + fileData.EXPECT().Reader(mock.Anything).Return(&failAfterReader{}, nil) + fileData.EXPECT().LastModifiedDate().Return(int64(1700000000)).Maybe() + fileComponent := mock_fileobject.NewMockFileObject(t) + fileComponent.EXPECT().GetFile().Return(fileData, nil) + fx.picker.EXPECT().GetObject(mock.Anything, fileId). + Return(&fileObjectWrapper{SmartTest: fileSb, FileObject: fileComponent}, nil) + fx.provider.EXPECT().Type(spaceId, fileId).Return(smartblock.SmartBlockTypeFileObject, nil) + + dir := t.TempDir() + wr, err := anyblock.NewDirWriter(dir) + require.NoError(t, err) + + result, err := fx.exporter.Export(context.Background(), + anyblock.Request{SpaceId: spaceId, SpaceName: "Fixture space", IncludeArchived: true, IncludeFiles: true}, wr) + + require.NoError(t, err) + assert.Equal(t, anyblock.Result{Succeed: 1, BlobErrors: 1}, result) + tree := readTree(t, dir) + require.Contains(t, tree, "files/truncatedFileId.anyblock.json") + assert.NotContains(t, tree, "files/truncatedFileId.bin", "the partial blob must be cleaned up") +} diff --git a/core/block/export/anyblock/writer.go b/core/block/export/anyblock/writer.go new file mode 100644 index 0000000000..461062c0c1 --- /dev/null +++ b/core/block/export/anyblock/writer.go @@ -0,0 +1,74 @@ +package anyblock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +// DirWriter writes bundle files under one fixed directory — the +// deterministic Writer for tests and cmd tooling: no timestamped wrapper +// (the legacy dir writer bakes the clock into its root name), no state +// beyond the root. File mtimes follow each document's own +// lastModifiedDate, like the legacy writers, so a bundle browses by real +// dates; content, not mtimes, is what determinism is measured on. +type DirWriter struct { + root string +} + +// NewDirWriter creates root (and parents) and writes everything below it. +func NewDirWriter(root string) (*DirWriter, error) { + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, fmt.Errorf("create export directory: %w", err) + } + return &DirWriter{root: root}, nil +} + +// Path is the directory the bundle lands in. +func (w *DirWriter) Path() string { + return w.root +} + +// WriteFile writes one file at its bundle-relative, slash-separated path. +// The path must stay inside the root — the plan guarantees it for every +// path it mints, and this guards the invariant against any other caller. +func (w *DirWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int64) error { + full := filepath.Join(w.root, filepath.FromSlash(filename)) + if rel, err := filepath.Rel(w.root, full); err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("path %q escapes the bundle root", filename) + } + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return fmt.Errorf("create subdirectory: %w", err) + } + f, err := os.Create(full) + if err != nil { + return fmt.Errorf("create file: %w", err) + } + defer f.Close() + if _, err := io.Copy(f, r); err != nil { + return fmt.Errorf("copy content to file: %w", err) + } + if lastModifiedDate > 0 { + if err := os.Chtimes(full, time.Now(), time.Unix(lastModifiedDate, 0)); err != nil { + return fmt.Errorf("set modified date: %w", err) + } + } + return nil +} + +// RemoveFile deletes one file below the root — the exporter's cleanup hook +// for a partially streamed blob, guarded by the same containment rule as +// WriteFile. A file that is already gone is not an error. +func (w *DirWriter) RemoveFile(filename string) error { + full := filepath.Join(w.root, filepath.FromSlash(filename)) + if rel, err := filepath.Rel(w.root, full); err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("path %q escapes the bundle root", filename) + } + if err := os.Remove(full); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove file: %w", err) + } + return nil +} diff --git a/core/block/export/anyblockjson.go b/core/block/export/anyblockjson.go new file mode 100644 index 0000000000..050d598719 --- /dev/null +++ b/core/block/export/anyblockjson.go @@ -0,0 +1,113 @@ +package export + +// anyblockjson.go routes model.Export_AnyBlockJSON to the native bundle +// exporter (core/block/export/anyblock). The format's own pipeline — plan, +// emit, compose, finish — lives there; what belongs HERE is only what the +// export service owns: the writer, the collected doc set, and the +// process.Queue every export reports progress and cancellation through. + +import ( + "context" + "errors" + "fmt" + + "github.com/anyproto/anytype-heart/core/block/export/anyblock" + "github.com/anyproto/anytype-heart/core/block/process" +) + +// exportAnyBlockJSON writes the native bundle through wr and returns the +// number of documents it accounted for. +// +// The collection is NOT run again: exportObjects ran it before the writer +// existed, and closureForFormat gives this format the same ClosureDerived +// set anyblock.CollectRequest asks for, so a second pass would query the +// whole space twice for one export. +// +// How this can fail: if closureForFormat ever stops mapping this format to +// ClosureDerived, the bundle quietly loses every derived document — types, +// options, templates — instead of failing. The end-to-end test's directory +// assertions are what catch that, since types/ and options/ exist only +// under the derived closure. +func (e *exportContext) exportAnyBlockJSON(ctx context.Context, wr writer, queue process.Queue) (int, error) { + exporter := &anyblock.Exporter{ + Picker: e.picker, + ObjectStore: e.objectStore, + SbtProvider: e.sbtProvider, + } + res, err := exporter.ExportCollected(ctx, anyblock.Request{ + SpaceId: e.spaceId, + Ids: e.reqIds, + IncludeNested: e.includeNested, + IncludeFiles: e.includeFiles, + IncludeArchived: e.includeArchive, + IncludeBacklinks: e.includeBackLinks, + IncludeSpace: e.includeSpace, + StateFilters: e.linkStateFilters, + // SpaceName stays empty on purpose: it is only index.json's fallback + // for a space whose OWN document states no name (§2c), and that + // document travels in the collected set, so the composer already + // holds the better answer. + Runner: queueRunner{queue: queue}, + }, e.docs, wr) + if err != nil { + if errors.Is(err, process.ErrQueueCanceled) || errors.Is(err, context.Canceled) { + // the cancel shape the legacy branch of exportByFormat uses: + // nothing succeeded, the half-written output goes away, and the + // RPC reports no error for the stop the user asked for + cleanupFile(wr) + return 0, nil + } + return 0, fmt.Errorf("export anyblock json bundle: %w", err) + } + return res.Succeed, nil +} + +// queueRunner runs the native exporter's emit tasks on the export queue — +// the same process.Queue the legacy formats hand their per-document tasks +// to. Routing emit through it is what keeps this format's progress +// (Total/Done) and its answer to ProcessCancel identical to every other +// format's, with one process per export rather than two. +type queueRunner struct { + queue process.Queue +} + +// Run hands every task to the queue and blocks until they are all done, or +// until the queue is cancelled. The queue's own worker count bounds how +// many run at once (exportWorkers), and the tasks themselves watch ctx — +// so this does not, and takes ctx only to satisfy anyblock.EmitRunner. +func (r queueRunner) Run(_ context.Context, tasks []func()) error { + queued := make([]process.Task, 0, len(tasks)) + for _, task := range tasks { + queued = append(queued, task) + } + if err := r.queue.Wait(queued...); err != nil { + return fmt.Errorf("run emit tasks on export queue: %w", err) + } + return nil +} + +// exportSingleAnyBlockDocument serves ExportSingleInMemory for the native +// format: ONE document, no bundle files. That is the design's position +// (Q7) resting on rule 7 of the format's principles — a document stands +// alone, carrying its own property names and formats — so index.json and +// properties.json have nothing to add about a single object, and an +// in-memory string could not carry two more files anyway. +func (e *exportContext) exportSingleAnyBlockDocument(ctx context.Context, objectId string) (string, error) { + details, err := e.objectStore.SpaceIndex(e.spaceId).GetDetails(objectId) + if err != nil { + return "", fmt.Errorf("get object details: %w", err) + } + if err := refuseInMemoryFileObject(details); err != nil { + return "", err + } + exporter := &anyblock.Exporter{ + Picker: e.picker, + ObjectStore: e.objectStore, + SbtProvider: e.sbtProvider, + } + data, err := exporter.ExportDocument(ctx, e.spaceId, objectId) + if err != nil { + return "", fmt.Errorf("export anyblock json document: %w", err) + } + return string(data), nil +} diff --git a/core/block/export/anyblockjson_test.go b/core/block/export/anyblockjson_test.go new file mode 100644 index 0000000000..7ca1754996 --- /dev/null +++ b/core/block/export/anyblockjson_test.go @@ -0,0 +1,356 @@ +package export + +// anyblockjson_test.go covers the RPC route only: that +// model.Export_AnyBlockJSON reaches the native exporter through the export +// service with the right collection behind it, that its emit still reports +// progress and answers cancellation through the export queue, and that the +// single-object door returns one document. The bundle's CONTENT — layout +// rules, blob binding, determinism — belongs to core/block/export/anyblock +// and pkg/lib/anyblockjson and is proved there, over the corpus. + +import ( + "archive/zip" + "context" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/anyproto/any-sync/util/crypto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/block/export/collect" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// anyblockSpace seeds one object of every kind that owns a bundle +// directory, in the store and in the mocks, and returns the ids by the +// directory each must land in. Seven kinds, seven directories: the layout +// is the format's own vocabulary (design §1.2), and a format value that +// collected the wrong closure would lose most of them. +func anyblockSpace(t *testing.T, fx *fixture) map[string]string { + const ( + pageId = "pageId" + typeId = "customObjectType" + templateId = "templateId" + fileId = "fileObjectId" + optionId = "optionId" + ) + const propertyKey = domain.RelationKey("customProperty") + + _, pub, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + identity := pub.Account() + participantId := domain.NewParticipantId(spaceId, identity) + + fx.store.AddObjects(t, spaceId, []spaceindex.TestObject{ + prepareTestObjectForStore(pageId, typeId), + prepareTestObjectTypeForStore(t, typeId, nil), + { + bundle.RelationKeyId: domain.String(templateId), + bundle.RelationKeyName: domain.String("Template"), + bundle.RelationKeyTargetObjectType: domain.String(typeId), + bundle.RelationKeyType: domain.String(typeId), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + prepareTestRelationForStore(t, propertyKey, int64(model.RelationFormat_tag)), + prepareTestOptionForStore(t, propertyKey, optionId), + { + bundle.RelationKeyId: domain.String(fileId), + bundle.RelationKeyName: domain.String("notes"), + bundle.RelationKeyFileExt: domain.String("txt"), + bundle.RelationKeyFileMimeType: domain.String("text/plain"), + bundle.RelationKeyType: domain.String(typeId), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + { + bundle.RelationKeyId: domain.String(participantId), + bundle.RelationKeyIdentity: domain.String(identity), + bundle.RelationKeyName: domain.String("Someone"), + bundle.RelationKeyType: domain.String(typeId), + bundle.RelationKeySpaceId: domain.String(spaceId), + bundle.RelationKeyUniqueKey: domain.String("participant" + identity), + }, + }) + + typeUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, typeId) + require.NoError(t, err) + + objects := []struct { + id string + sbType smartblock.SmartBlockType + details map[domain.RelationKey]domain.Value + }{ + {id: pageId, sbType: smartblock.SmartBlockTypePage}, + // the unique key is what the manifest's type table is keyed by, so + // the type document carries its own + {id: typeId, sbType: smartblock.SmartBlockTypeObjectType, details: map[domain.RelationKey]domain.Value{ + bundle.RelationKeyUniqueKey: domain.String(typeUniqueKey.Marshal()), + }}, + {id: templateId, sbType: smartblock.SmartBlockTypeTemplate}, + {id: propertyKey.String(), sbType: smartblock.SmartBlockTypeRelation}, + {id: optionId, sbType: smartblock.SmartBlockTypeRelationOption}, + {id: fileId, sbType: smartblock.SmartBlockTypeFileObject}, + {id: participantId, sbType: smartblock.SmartBlockTypeParticipant}, + } + for _, object := range objects { + loaded := setupObject(object.id, typeId, object.sbType, object.details) + fx.picker.EXPECT().GetObject(mock.Anything, object.id).Return(loaded, nil).Maybe() + fx.sbtProvider.EXPECT().Type(spaceId, object.id).Return(object.sbType, nil).Maybe() + } + + return map[string]string{ + "objects": pageId, + "types": typeId, + "templates": templateId, + "properties": propertyKey.String(), + "options": optionId, + "files": fileId, + // the STORE id, not the bare identity: the participant fold needs a + // real space id (`.`) to parse, and a fixture's "space1" + // does not — so the envelope keeps the composite, and the filename + // follows the envelope either way (design §1.3). The fold itself is + // covered where it lives, in the codec's own tests. + "participants": participantId, + } +} + +// readExportTree reads every file under root into path -> content, paths +// slash-separated and relative to the export root. +func readExportTree(t *testing.T, root string) map[string]string { + tree := map[string]string{} + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + tree[filepath.ToSlash(rel)] = string(data) + return nil + }) + require.NoError(t, err) + return tree +} + +// The whole route, from the RPC request to the bundle on disk: the format +// reaches the native exporter, the collection behind it is the derived one +// (nothing else would carry types, options and templates into a +// whole-space export), every document lands in its kind directory under +// its own id, and the two bundle files read back through the package that +// wrote them. +// +// How this can fail: route the format to ClosureContent and five of the +// seven directories vanish (only pages and file objects survive that +// closure); send it down the legacy writeDoc path and the extension +// becomes .pb.json in relations/ and relationsOptions/; forget the bundle +// files and a reader has no property dictionary to resolve keys against. +func TestExport_AnyBlockJSONWritesABundle(t *testing.T) { + // given + fx := newFixture(t) + byDirectory := anyblockSpace(t, fx) + fx.picker.EXPECT().TryRemoveFromCache(mock.Anything, mock.Anything).Return(true, nil) + + // when + exportPath, succeed, err := fx.Export(context.Background(), pb.RpcObjectListExportRequest{ + SpaceId: spaceId, + Path: t.TempDir(), + Format: model.Export_AnyBlockJSON, + IncludeArchived: true, + NoProgress: true, + }) + + // then + require.NoError(t, err) + assert.Equal(t, len(byDirectory), succeed) + + tree := readExportTree(t, exportPath) + for dir, id := range byDirectory { + assert.Contains(t, tree, path.Join(dir, id+".anyblock.json"), "%s belongs in %s/", id, dir) + } + require.Contains(t, tree, anyblockjson.IndexFileName) + require.Contains(t, tree, anyblockjson.PropertiesFileName) + for name := range tree { + if name == anyblockjson.IndexFileName || name == anyblockjson.PropertiesFileName { + continue + } + assert.True(t, strings.HasSuffix(name, ".anyblock.json"), "unexpected file %q in the bundle", name) + } + + index, err := anyblockjson.UnmarshalIndex([]byte(tree[anyblockjson.IndexFileName])) + require.NoError(t, err) + require.NotNil(t, index.Manifest) + assert.Equal(t, "types/customObjectType.anyblock.json", index.Manifest.Types["customObjectType"]) + _, err = anyblockjson.UnmarshalPropertyDictionary([]byte(tree[anyblockjson.PropertiesFileName])) + require.NoError(t, err) +} + +// The same bundle into a zip archive, which is what a real backup takes: +// the native exporter writes through the export service's own writers, and +// the zip one is the only writer whose paths are not the filesystem's. +// +// How this can fail: build bundle paths with filepath.Join on a platform +// whose separator is not "/" and the archive grows entries no reader can +// resolve; or write the bundle files after Close and lose them silently. +func TestExport_AnyBlockJSONWritesAZipBundle(t *testing.T) { + // given + fx := newFixture(t) + byDirectory := anyblockSpace(t, fx) + fx.picker.EXPECT().TryRemoveFromCache(mock.Anything, mock.Anything).Return(true, nil) + + // when + archivePath, succeed, err := fx.Export(context.Background(), pb.RpcObjectListExportRequest{ + SpaceId: spaceId, + Path: t.TempDir(), + Format: model.Export_AnyBlockJSON, + IncludeArchived: true, + NoProgress: true, + Zip: true, + }) + + // then + require.NoError(t, err) + assert.Equal(t, len(byDirectory), succeed) + + reader, err := zip.OpenReader(archivePath) + require.NoError(t, err) + defer reader.Close() + entries := make(map[string]bool, len(reader.File)) + for _, file := range reader.File { + entries[file.Name] = true + } + assert.Len(t, entries, len(byDirectory)+2) // the documents, index.json, properties.json + for dir, id := range byDirectory { + assert.True(t, entries[dir+"/"+id+".anyblock.json"], "%s belongs in %s/", id, dir) + } + assert.True(t, entries[anyblockjson.IndexFileName]) + assert.True(t, entries[anyblockjson.PropertiesFileName]) +} + +// Emit runs as queue tasks, so the process the client already watches +// counts this format's documents like every other format's — the native +// exporter's own bounded pool would have left the progress bar at 0/0 for +// the whole export. +// +// How this can fail: give the exporter its internal runner here (Total +// stays 0); or bound the queue somewhere other than exportWorkers, and the +// resident content set stops being the width §1.6 measured. +func TestExport_AnyBlockJSONReportsQueueProgress(t *testing.T) { + // given + fx := newFixture(t) + byDirectory := anyblockSpace(t, fx) + fx.picker.EXPECT().TryRemoveFromCache(mock.Anything, mock.Anything).Return(true, nil) + + queue := fx.processService.NewQueue(pb.ModelProcess{ + Id: "anyblockjson", + Message: &pb.ModelProcessMessageOfExport{Export: &pb.ModelProcessExport{}}, + }, exportWorkers(model.Export_AnyBlockJSON), true, fx.notifications) + require.NoError(t, queue.Start()) + + exportCtx := newExportContext(fx.export, pb.RpcObjectListExportRequest{ + SpaceId: spaceId, + Path: t.TempDir(), + Format: model.Export_AnyBlockJSON, + IncludeArchived: true, + NoProgress: true, + }) + require.NoError(t, exportCtx.docsForExport(context.Background())) + wr, err := newDirWriter(exportCtx.path, false) + require.NoError(t, err) + + // when + succeed, err := exportCtx.exportByFormat(context.Background(), wr, queue) + + // then + require.NoError(t, err) + assert.Equal(t, len(byDirectory), succeed) + require.NoError(t, queue.Finalize()) // waits for the workers, so Done is settled + progress := queue.Info().Progress + assert.Equal(t, int64(len(byDirectory)), progress.Total) + assert.Equal(t, int64(len(byDirectory)), progress.Done) +} + +// A cancelled export stops loading objects. The picker mock is the +// assertion: no TryRemoveFromCache expectation is set, and emit closes +// every object it loads — so a single task that ran would fail the test. +// +// How this can fail: drop the ctx check from the emit task and a cancelled +// account-sized export keeps cold-building trees for minutes; write the +// bundle files anyway and index.json claims documents that were never +// emitted. +func TestExport_AnyBlockJSONStopsWhenCancelled(t *testing.T) { + // given + fx := newFixture(t) + anyblockSpace(t, fx) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + // when + exportPath, succeed, err := fx.Export(ctx, pb.RpcObjectListExportRequest{ + SpaceId: spaceId, + Path: t.TempDir(), + Format: model.Export_AnyBlockJSON, + IncludeArchived: true, + NoProgress: true, + }) + + // then + require.NoError(t, err) // the cancel is the user's own request, not a failure + assert.Equal(t, 0, succeed) + assert.NoDirExists(t, exportPath, "a cancelled export leaves nothing behind") +} + +// ExportSingleInMemory answers with ONE document — no index.json, no +// property dictionary, nothing a bundle would add (design Q7, principle 7: +// a document stands alone). +// +// How this can fail: fall through to the legacy converter switch, where +// this format has no case, and the export panics on a nil converter. +func TestExport_AnyBlockJSONSingleInMemory(t *testing.T) { + // given + fx := newFixture(t) + byDirectory := anyblockSpace(t, fx) + + // when + result, err := fx.ExportSingleInMemory(context.Background(), spaceId, byDirectory["objects"], model.Export_AnyBlockJSON) + + // then + require.NoError(t, err) + sbType, snapshot, err := anyblockjson.Unmarshal([]byte(result), anyblockjson.Options{SpaceId: spaceId}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + assert.Equal(t, byDirectory["objects"], snapshot.GetDetails().GetFields()[bundle.RelationKeyId.String()].GetStringValue()) +} + +// The closure every format collects with. Pinned as a table because the +// predicate this replaced was named isAnyblockExport and meant "protobuf +// or pb.json" — the one place where a wrong answer is invisible until a +// bundle silently ships without its types. +func TestClosureForFormat(t *testing.T) { + for format, want := range map[model.ExportFormat]collect.Closure{ + model.Export_Protobuf: collect.ClosureDerived, + model.Export_JSON: collect.ClosureDerived, + model.Export_AnyBlockJSON: collect.ClosureDerived, + model.Export_Markdown: collect.ClosureContent, + model.Export_DOT: collect.ClosureContent, + model.Export_SVG: collect.ClosureContent, + model.Export_GRAPH_JSON: collect.ClosureContent, + } { + assert.Equal(t, want, closureForFormat(format), "closure for %v", format) + } +} diff --git a/core/block/export/collect/collect.go b/core/block/export/collect/collect.go new file mode 100644 index 0000000000..dd70002c3f --- /dev/null +++ b/core/block/export/collect/collect.go @@ -0,0 +1,108 @@ +// Package collect declares the format-agnostic collection seam of the +// exporter: the dependency closure over an export request, complete before +// anything is written (pkg/lib/anyblockjson/EXPORTER_DESIGN.md §1.1). +// +// Collection decides WHICH objects an export carries and holds details only — +// content loads later, per document, inside each writer's emit task. The +// closure MODE used to travel through the legacy exporter as a bare +// `isProtobuf bool`; here it is an explicit Closure, so a new format states +// which closure it wants instead of impersonating protobuf. The +// implementation lives with the legacy exporter (core/block/export), which +// exposes it through Collector; the native AnyBlock JSON writer consumes the +// interface and nothing behind it. +package collect + +import ( + "context" + + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" +) + +// Closure names which dependency closure a collection runs. +type Closure int + +const ( + // ClosureContent is the md-style closure: the requested objects, their + // linked files, and — behind IncludeNested — the objects their content + // links to. Nothing derived is pulled in. + ClosureContent Closure = iota + // ClosureDerived is the collect-everything-derived closure the snapshot + // formats want: types, relations, relation options, templates, dataview + // dependencies and recommended relations ride along, so the export + // stands alone. + ClosureDerived +) + +// Request describes one collection run. +type Request struct { + SpaceId string + // Ids are the requested roots; empty means every exportable object of + // the space. + Ids []string + Closure Closure + + IncludeNested bool + IncludeFiles bool + IncludeArchived bool + IncludeBacklinks bool + IncludeSpace bool + + // StateFilters filter the state of objects that entered the set as + // links (Doc.IsLink); the requested roots always render unfiltered. + StateFilters *state.Filters +} + +// Doc is one collected object: its details, and whether it entered the set +// as a link rather than as a requested root. +type Doc struct { + Details *domain.Details + IsLink bool +} + +// Docs is a collection result, keyed by object id. +type Docs map[string]*Doc + +// TransformToDetailsMap re-wraps the collected details for converters that +// take a plain id → details map. Same pointers, not copies: the collection +// stays the single resident copy of the details (design §1.6). +func (d Docs) TransformToDetailsMap() map[string]*domain.Details { + details := make(map[string]*domain.Details, len(d)) + for id, doc := range d { + details[id] = doc.Details + } + return details +} + +// Collector runs the requested closure and returns the complete set before +// anything is written. +type Collector interface { + Collect(ctx context.Context, req Request) (Docs, error) +} + +// Excluded reports a collected row no format should emit: empty or id-only +// details, an id-plus-backlinks tombstone, or a legacy raw file id. The +// collection may still hold such rows (they arrive through store queries); +// every emitter skips them by this one rule. +func Excluded(details *domain.Details) bool { + if details == nil { + return true + } + n := details.Len() + // Empty details or containing only id + if n <= 1 { + return true + } + // Details only with id + backlinks should be discarded + if n == 2 && details.Has(bundle.RelationKeyBacklinks) { + return true + } + + id := details.GetString(bundle.RelationKeyId) + if domain.IsFileId(id) { + return true + } + + return false +} diff --git a/core/block/export/collection.go b/core/block/export/collection.go new file mode 100644 index 0000000000..47f6e3d804 --- /dev/null +++ b/core/block/export/collection.go @@ -0,0 +1,802 @@ +package export + +// collection.go is the collection half of the exporter — the dependency +// closure over an export request, complete before anything is written — and +// the implementation behind the format-agnostic collect.Collector seam +// (pkg/lib/anyblockjson/EXPORTER_DESIGN.md §1.1). +// +// The closure mode used to travel as a bare `isProtobuf bool`; it is now an +// explicit collect.Closure, so the native AnyBlock JSON writer asks for the +// derived closure by name instead of impersonating protobuf. Everything here +// reads DETAILS from the store and loads an object only to walk its links +// (getViewDependentObjects, collectDerivedObjects, addNestedObject, +// fillLinkedFiles); content stays out of the collection, which is what keeps +// its resident cost O(all details) rather than O(all content) (design §1.6). + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/samber/lo" + + "github.com/anyproto/anytype-heart/core/block/cache" + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/block/export/collect" + "github.com/anyproto/anytype-heart/core/block/object/objectlink" + "github.com/anyproto/anytype-heart/core/block/simple" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/database" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/addr" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/slice" + + sb "github.com/anyproto/anytype-heart/core/block/editor/smartblock" +) + +// Collect implements collect.Collector: it runs the requested closure and +// returns the complete doc set before anything is written. The legacy +// formats reach the same code through exportContext.docsForExport; the +// native AnyBlock JSON exporter consumes only this method. +func (e *export) Collect(ctx context.Context, req collect.Request) (collect.Docs, error) { + ec := &exportContext{ + spaceId: req.SpaceId, + docs: Docs{}, + includeArchive: req.IncludeArchived, + includeNested: req.IncludeNested, + includeFiles: req.IncludeFiles, + reqIds: req.Ids, + closure: req.Closure, + linkStateFilters: req.StateFilters, + includeBackLinks: req.IncludeBacklinks, + includeSpace: req.IncludeSpace, + setOfList: make(map[string]struct{}), + objectTypes: make(map[string]struct{}), + relations: make(map[string]struct{}), + export: e, + } + if err := ec.collectDocs(ctx); err != nil { + return nil, err + } + return ec.docs, nil +} + +// collectDocs runs the closure the context asks for — the body the legacy +// docsForExport dispatched on `isProtobuf`. +func (e *exportContext) collectDocs(ctx context.Context) (err error) { + if len(e.reqIds) == 0 { + return e.getExistedObjects(e.closure) + } + return e.getObjectsByIDs(ctx, e.closure) +} + +func (e *exportContext) getObjectsByIDs(ctx context.Context, closure collect.Closure) error { + res, err := e.queryAndFilterObjectsByRelation(e.spaceId, e.reqIds, bundle.RelationKeyId) + if err != nil { + return fmt.Errorf("query and filter objects by relation: %w", err) + } + for _, object := range res { + id := object.Details.GetString(bundle.RelationKeyId) + e.docs[id] = &Doc{Details: object.Details} + } + if e.includeSpace { + err = e.addSpaceToDocs(ctx) + if err != nil { + return fmt.Errorf("add space to docs: %w", err) + } + } + if closure == collect.ClosureDerived { + if err := e.processDerived(); err != nil { + return fmt.Errorf("process derived closure: %w", err) + } + return nil + } + if err := e.processContent(); err != nil { + return fmt.Errorf("process content closure: %w", err) + } + return nil +} + +func (e *exportContext) queryAndFilterObjectsByRelation(spaceId string, reqIds []string, relationKey domain.RelationKey) ([]database.Record, error) { + var allObjects []database.Record + const singleBatchCount = 50 + for j := 0; j < len(reqIds); { + if j+singleBatchCount < len(reqIds) { + records, err := e.queryObjectsByRelation(spaceId, reqIds[j:j+singleBatchCount], relationKey) + if err != nil { + return nil, fmt.Errorf("query objects by relation: %w", err) + } + allObjects = append(allObjects, records...) + } else { + records, err := e.queryObjectsByRelation(spaceId, reqIds[j:], relationKey) + if err != nil { + return nil, fmt.Errorf("query objects by relation: %w", err) + } + allObjects = append(allObjects, records...) + } + j += singleBatchCount + } + return allObjects, nil +} + +func (e *exportContext) queryObjectsByRelation(spaceId string, reqIds []string, relationKey domain.RelationKey) ([]database.Record, error) { + return e.objectStore.SpaceIndex(spaceId).Query(database.Query{ + Filters: []database.FilterRequest{ + { + RelationKey: relationKey, + Condition: model.BlockContentDataviewFilter_In, + Value: domain.StringList(reqIds), + }, + }, + }) +} + +func (e *exportContext) addSpaceToDocs(ctx context.Context) error { + space, err := e.spaceService.Get(ctx, e.spaceId) + if err != nil { + return fmt.Errorf("get space: %w", err) + } + workspaceId := space.DerivedIDs().Workspace + records, err := e.objectStore.SpaceIndex(e.spaceId).QueryByIds([]string{workspaceId}) + if err != nil { + return fmt.Errorf("query workspace details: %w", err) + } + if len(records) == 0 { + return fmt.Errorf("no objects found for space %s", workspaceId) + } + e.docs[workspaceId] = &Doc{Details: records[0].Details, IsLink: true} + return nil +} + +// processContent is the ClosureContent closure: linked files and, behind +// IncludeNested, content-linked objects — nothing derived. +func (e *exportContext) processContent() error { + ids := listObjectIds(e.docs) + if e.includeFiles { + fileObjectsIds, err := e.processFiles(ids) + if err != nil { + return fmt.Errorf("process files: %w", err) + } + ids = append(ids, fileObjectsIds...) + } + if e.includeNested { + for _, id := range ids { + e.addNestedObject(id, map[string]*Doc{}) + } + } + return nil +} + +// processDerived is the ClosureDerived closure: types, relations, options, +// templates, dataview dependencies and recommended relations ride along, so +// a snapshot export stands alone. +func (e *exportContext) processDerived() error { + if !e.includeNested { + err := e.addDependentObjectsFromDataview() + if err != nil { + return fmt.Errorf("add dependent objects from dataview: %w", err) + } + } + ids := listObjectIds(e.docs) + if e.includeFiles { + err := e.addFileObjects(ids) + if err != nil { + return fmt.Errorf("add file objects: %w", err) + } + } + + err := e.addDerivedObjects() + if err != nil { + return fmt.Errorf("add derived objects: %w", err) + } + ids = e.listTargetTypesFromTemplates(ids) + if e.includeNested { + err = e.addNestedObjects(ids) + if err != nil { + return fmt.Errorf("add nested objects: %w", err) + } + } + return nil +} + +func (e *exportContext) addDependentObjectsFromDataview() error { + var ( + viewDependentObjectsIds []string + err error + ) + for id, doc := range e.docs { + if isExcludedFromExport(doc.Details) { + continue + } + if isObjectWithDataview(doc.Details) { + viewDependentObjectsIds, err = e.getViewDependentObjects(id, viewDependentObjectsIds) + if err != nil { + return fmt.Errorf("get view dependent objects: %w", err) + } + } + } + viewDependentObjects, err := e.queryAndFilterObjectsByRelation(e.spaceId, viewDependentObjectsIds, bundle.RelationKeyId) + if err != nil { + return fmt.Errorf("query dependent objects: %w", err) + } + templates, err := e.queryAndFilterObjectsByRelation(e.spaceId, viewDependentObjectsIds, bundle.RelationKeyTargetObjectType) + if err != nil { + return fmt.Errorf("query templates: %w", err) + } + for _, object := range append(viewDependentObjects, templates...) { + id := object.Details.GetString(bundle.RelationKeyId) + e.docs[id] = &Doc{ + Details: object.Details, + IsLink: e.isLinkProcess, + } + } + return nil +} + +func (e *exportContext) getViewDependentObjects(id string, viewDependentObjectsIds []string) ([]string, error) { + err := cache.Do(e.picker, id, func(sb sb.SmartBlock) error { + st := sb.NewState().Copy().Filter(e.getStateFilters(id)) + viewDependentObjectsIds = append(viewDependentObjectsIds, + objectlink.DependentObjectIDs(st, sb.Space(), e.formatFetcher, objectlink.Flags{Blocks: true})...) + return nil + }) + if err != nil { + return nil, fmt.Errorf("get object from cache: %w", err) + } + return viewDependentObjectsIds, nil +} + +func (e *exportContext) addFileObjects(ids []string) error { + fileObjectsIds, err := e.processFiles(ids) + if err != nil { + return fmt.Errorf("process files: %w", err) + } + if e.includeNested { + err = e.addNestedObjects(fileObjectsIds) + if err != nil { + return fmt.Errorf("add nested objects: %w", err) + } + } + return nil +} + +func (e *exportContext) processFiles(ids []string) ([]string, error) { + var fileObjectsIds []string + for _, id := range ids { + objectFiles, err := e.fillLinkedFiles(id) + if err != nil { + return nil, fmt.Errorf("fill linked files: %w", err) + } + fileObjectsIds = lo.Union(fileObjectsIds, objectFiles) + } + return fileObjectsIds, nil +} + +func (e *exportContext) addDerivedObjects() error { + processedObjects := make(map[string]struct{}, 0) + err := e.getRelationsAndTypes(e.docs, processedObjects) + if err != nil { + return fmt.Errorf("get relations and types: %w", err) + } + + err = e.getTemplatesRelationsAndTypes(processedObjects) + if err != nil { + return fmt.Errorf("get templates relations and types: %w", err) + } + err = e.addRelationsAndTypes() + if err != nil { + return fmt.Errorf("add relations and types: %w", err) + } + return nil +} + +func (e *exportContext) getRelationsAndTypes(notProcessedObjects map[string]*Doc, processedObjects map[string]struct{}) error { + err := e.collectDerivedObjects(notProcessedObjects) + if err != nil { + return fmt.Errorf("collect derived objects: %w", err) + } + // get derived objects only from types, + // because relations currently have only system relations and object type + if len(e.objectTypes) > 0 || len(e.setOfList) > 0 { + err = e.getDerivedObjectsForTypes(processedObjects) + if err != nil { + return fmt.Errorf("get derived objects for types: %w", err) + } + } + return nil +} + +func (e *exportContext) collectDerivedObjects(objects map[string]*Doc) error { + for id, doc := range objects { + if doc != nil && isExcludedFromExport(doc.Details) { + continue + } + err := cache.Do(e.picker, id, func(b sb.SmartBlock) error { + state := b.NewState().Copy().Filter(e.getStateFilters(id)) + objectRelations := state.AllRelationKeys() + fillObjectsMap(e.relations, slice.IntoStrings(objectRelations)) + details := state.CombinedDetails() + if isObjectWithDataview(details) { + dataviewRelations, err := getDataviewRelations(state) + if err != nil { + return fmt.Errorf("get dataview relations: %w", err) + } + fillObjectsMap(e.relations, dataviewRelations) + } + var objectTypes []string + if details.Has(bundle.RelationKeyType) { + objectTypes = append(objectTypes, details.GetString(bundle.RelationKeyType)) + } + if details.Has(bundle.RelationKeyTargetObjectType) { + objectTypes = append(objectTypes, details.GetString(bundle.RelationKeyTargetObjectType)) + } + fillObjectsMap(e.objectTypes, objectTypes) + setOfList := details.GetStringList(bundle.RelationKeySetOf) + fillObjectsMap(e.setOfList, setOfList) + return nil + }) + if err != nil { + return fmt.Errorf("get object from cache: %w", err) + } + } + return nil +} + +func fillObjectsMap(dst map[string]struct{}, objectsToAdd []string) { + for _, objectId := range objectsToAdd { + dst[objectId] = struct{}{} + } +} + +func isObjectWithDataview(details *domain.Details) bool { + return details.GetInt64(bundle.RelationKeyResolvedLayout) == int64(model.ObjectType_collection) || + details.GetInt64(bundle.RelationKeyResolvedLayout) == int64(model.ObjectType_set) +} + +func getDataviewRelations(state *state.State) ([]string, error) { + var relations []string + err := state.Iterate(func(b simple.Block) (isContinue bool) { + if dataview := b.Model().GetDataview(); dataview != nil { + for _, view := range dataview.Views { + for _, relation := range view.Relations { + relations = append(relations, relation.Key) + } + } + } + return true + }) + if err != nil { + return nil, fmt.Errorf("iterate state blocks: %w", err) + } + return relations, nil +} + +func (e *exportContext) getDerivedObjectsForTypes(processedObjects map[string]struct{}) error { + notProceedTypes := make(map[string]*Doc) + for object := range e.objectTypes { + e.fillNotProcessedTypes(processedObjects, object, notProceedTypes) + } + for object := range e.setOfList { + e.fillNotProcessedTypes(processedObjects, object, notProceedTypes) + } + if len(notProceedTypes) == 0 { + return nil + } + err := e.getRelationsAndTypes(notProceedTypes, processedObjects) + if err != nil { + return fmt.Errorf("get relations and types: %w", err) + } + return nil +} + +func (e *exportContext) fillNotProcessedTypes(processedObjects map[string]struct{}, object string, notProceedTypes map[string]*Doc) { + if _, ok := processedObjects[object]; ok { + return + } + notProceedTypes[object] = nil + processedObjects[object] = struct{}{} +} + +func (e *exportContext) getTemplatesRelationsAndTypes(processedObjects map[string]struct{}) error { + allTypes := lo.MapToSlice(e.objectTypes, func(key string, value struct{}) string { return key }) + templates, err := e.queryAndFilterObjectsByRelation(e.spaceId, allTypes, bundle.RelationKeyTargetObjectType) + if err != nil { + return fmt.Errorf("query templates by target type: %w", err) + } + if len(templates) == 0 { + return nil + } + templatesToProcess := make(map[string]*Doc, len(templates)) + for _, template := range templates { + id := template.Details.GetString(bundle.RelationKeyId) + if _, ok := e.docs[id]; !ok { + templateDoc := &Doc{Details: template.Details, IsLink: e.isLinkProcess} + e.docs[id] = templateDoc + templatesToProcess[id] = templateDoc + } + } + err = e.getRelationsAndTypes(templatesToProcess, processedObjects) + if err != nil { + return fmt.Errorf("get relations and types for templates: %w", err) + } + return nil +} + +func (e *exportContext) addRelationsAndTypes() error { + types := lo.MapToSlice(e.objectTypes, func(key string, value struct{}) string { return key }) + setOfList := lo.MapToSlice(e.setOfList, func(key string, value struct{}) string { return key }) + relations := lo.MapToSlice(e.relations, func(key string, value struct{}) string { return key }) + + err := e.addRelations(relations) + if err != nil { + return fmt.Errorf("add relations: %w", err) + } + err = e.processObjectTypesAndSetOfList(types, setOfList) + if err != nil { + return fmt.Errorf("process object types and set of list: %w", err) + } + return nil +} + +func (e *exportContext) addRelations(relations []string) error { + storeRelations, err := e.getRelationsFromStore(relations) + if err != nil { + return fmt.Errorf("get relations from store: %w", err) + } + for _, storeRelation := range storeRelations { + e.addRelation(storeRelation) + err := e.addOptionIfTag(storeRelation) + if err != nil { + return fmt.Errorf("add option if tag: %w", err) + } + } + return nil +} + +func (e *exportContext) getRelationsFromStore(relations []string) ([]database.Record, error) { + uniqueKeys := make([]string, 0, len(relations)) + for _, relation := range relations { + uniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeRelation, relation) + if err != nil { + return nil, fmt.Errorf("create unique key for relation: %w", err) + } + uniqueKeys = append(uniqueKeys, uniqueKey.Marshal()) + } + storeRelations, err := e.queryAndFilterObjectsByRelation(e.spaceId, uniqueKeys, bundle.RelationKeyUniqueKey) + if err != nil { + return nil, fmt.Errorf("query relations by unique key: %w", err) + } + return storeRelations, nil +} + +func (e *exportContext) addRelation(relation database.Record) { + relationKey := domain.RelationKey(relation.Details.GetString(bundle.RelationKeyRelationKey)) + if relationKey != "" { + id := relation.Details.GetString(bundle.RelationKeyId) + e.docs[id] = &Doc{Details: relation.Details, IsLink: e.isLinkProcess} + } +} + +func (e *exportContext) addOptionIfTag(relation database.Record) error { + format := relation.Details.GetInt64(bundle.RelationKeyRelationFormat) + relationKey := relation.Details.GetString(bundle.RelationKeyRelationKey) + if format == int64(model.RelationFormat_tag) || format == int64(model.RelationFormat_status) { + err := e.addRelationOptions(relationKey) + if err != nil { + return fmt.Errorf("add relation options: %w", err) + } + } + return nil +} + +func (e *exportContext) addRelationOptions(relationKey string) error { + relationOptions, err := e.getRelationOptions(relationKey) + if err != nil { + return fmt.Errorf("get relation options: %w", err) + } + for _, option := range relationOptions { + id := option.Details.GetString(bundle.RelationKeyId) + e.docs[id] = &Doc{Details: option.Details, IsLink: e.isLinkProcess} + } + return nil +} + +func (e *exportContext) getRelationOptions(relationKey string) ([]database.Record, error) { + relationOptionsDetails, err := e.objectStore.SpaceIndex(e.spaceId).Query(database.Query{ + Filters: []database.FilterRequest{ + { + RelationKey: bundle.RelationKeyResolvedLayout, + Condition: model.BlockContentDataviewFilter_Equal, + Value: domain.Int64(model.ObjectType_relationOption), + }, + { + RelationKey: bundle.RelationKeyRelationKey, + Condition: model.BlockContentDataviewFilter_Equal, + Value: domain.String(relationKey), + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("query relation options: %w", err) + } + return relationOptionsDetails, nil +} + +func (e *exportContext) processObjectTypesAndSetOfList(objectTypes, setOfList []string) error { + objectDetails, err := e.queryAndFilterObjectsByRelation(e.spaceId, lo.Union(objectTypes, setOfList), bundle.RelationKeyId) + if err != nil { + return fmt.Errorf("query object types: %w", err) + } + if len(objectDetails) == 0 { + return nil + } + recommendedRelations, err := e.addObjectsAndCollectRecommendedRelations(objectDetails) + if err != nil { + return fmt.Errorf("collect recommended relations: %w", err) + } + err = e.addRecommendedRelations(recommendedRelations) + if err != nil { + return fmt.Errorf("add recommended relations: %w", err) + } + return nil +} + +func (e *exportContext) addObjectsAndCollectRecommendedRelations(objectTypes []database.Record) ([]string, error) { + recommendedRelations := make([]string, 0, len(objectTypes)) + for i := 0; i < len(objectTypes); i++ { + rawUniqueKey := objectTypes[i].Details.GetString(bundle.RelationKeyUniqueKey) + uniqueKey, err := domain.UnmarshalUniqueKey(rawUniqueKey) + if err != nil { + return nil, fmt.Errorf("unmarshal unique key: %w", err) + } + id := objectTypes[i].Details.GetString(bundle.RelationKeyId) + e.docs[id] = &Doc{Details: objectTypes[i].Details, IsLink: e.isLinkProcess} + if uniqueKey.SmartblockType() == smartblock.SmartBlockTypeObjectType { + key, err := domain.GetTypeKeyFromRawUniqueKey(rawUniqueKey) + if err != nil { + return nil, fmt.Errorf("get type key from unique key: %w", err) + } + if bundle.IsInternalType(key) { + continue + } + recommendedRelations = lo.Uniq(slices.Concat(recommendedRelations, + objectTypes[i].Details.GetStringList(bundle.RelationKeyRecommendedRelations), + objectTypes[i].Details.GetStringList(bundle.RelationKeyRecommendedHiddenRelations), + objectTypes[i].Details.GetStringList(bundle.RelationKeyRecommendedFeaturedRelations), + objectTypes[i].Details.GetStringList(bundle.RelationKeyRecommendedFileRelations), + )) + } + } + return recommendedRelations, nil +} + +func (e *exportContext) addRecommendedRelations(recommendedRelations []string) error { + relations, err := e.queryAndFilterObjectsByRelation(e.spaceId, recommendedRelations, bundle.RelationKeyId) + if err != nil { + return fmt.Errorf("query recommended relations: %w", err) + } + for _, relation := range relations { + id := relation.Details.GetString(bundle.RelationKeyId) + if id == addr.MissingObject { + continue + } + + relationKey := relation.Details.GetString(bundle.RelationKeyUniqueKey) + uniqueKey, err := domain.UnmarshalUniqueKey(relationKey) + if err != nil { + return fmt.Errorf("unmarshal relation unique key: %w", err) + } + if bundle.IsSystemRelation(domain.RelationKey(uniqueKey.InternalKey())) { + continue + } + e.docs[id] = &Doc{Details: relation.Details, IsLink: e.isLinkProcess} + } + return nil +} + +func (e *exportContext) addNestedObjects(ids []string) error { + nestedDocs := make(map[string]*Doc, 0) + for _, id := range ids { + e.addNestedObject(id, nestedDocs) + } + if len(nestedDocs) == 0 { + return nil + } + exportCtxChild := e.copy() + exportCtxChild.includeNested = false + exportCtxChild.docs = nestedDocs + exportCtxChild.isLinkProcess = true + err := exportCtxChild.processDerived() + if err != nil { + return fmt.Errorf("process nested derived closure: %w", err) + } + for id, object := range exportCtxChild.docs { + if _, ok := e.docs[id]; !ok { + e.docs[id] = object + } + } + return nil +} + +func (e *exportContext) addNestedObject(id string, nestedDocs map[string]*Doc) { + if doc, ok := e.docs[id]; ok && isExcludedFromExport(doc.Details) { + return + } + var links []string + err := cache.Do(e.picker, id, func(sb sb.SmartBlock) error { + st := sb.NewState().Copy().Filter(e.getStateFilters(id)) + links = objectlink.DependentObjectIDs(st, sb.Space(), e.formatFetcher, objectlink.Flags{ + Blocks: true, + Details: true, + Collection: true, + NoHiddenBundledRelations: true, + NoBackLinks: !e.includeBackLinks, + CreatorModifierWorkspace: true, + }) + return nil + }) + if err != nil { + return + } + for _, link := range links { + if _, exists := e.docs[link]; !exists { + sbt, sbtErr := e.sbtProvider.Type(e.spaceId, link) + if sbtErr != nil { + log.Errorf("failed to get smartblocktype of id %s", link) + continue + } + if !validType(sbt) { + continue + } + rec, qErr := e.objectStore.SpaceIndex(e.spaceId).QueryByIds([]string{link}) + if qErr != nil { + log.Errorf("failed to query id %s, err: %s", qErr, err) + continue + } + if isLinkedObjectExist(rec) { + exportDoc := &Doc{Details: rec[0].Details, IsLink: true} + nestedDocs[link] = exportDoc + e.docs[link] = exportDoc + e.addNestedObject(link, nestedDocs) + } + } + } +} + +func (e *exportContext) fillLinkedFiles(id string) ([]string, error) { + if doc, ok := e.docs[id]; ok && isExcludedFromExport(doc.Details) { + return nil, nil + } + spaceIndex := e.objectStore.SpaceIndex(e.spaceId) + var fileObjectsIds []string + err := cache.Do(e.picker, id, func(b sb.SmartBlock) error { + b.NewState().Copy().Filter(e.getStateFilters(id)).IterateLinkedFiles(e.formatFetcher, func(fileObjectId string) { + res, err := spaceIndex.Query(database.Query{ + Filters: []database.FilterRequest{ + { + RelationKey: bundle.RelationKeyId, + Condition: model.BlockContentDataviewFilter_Equal, + Value: domain.String(fileObjectId), + }, + }, + }) + if err != nil { + log.Errorf("failed to get details for file object id %s: %v", fileObjectId, err) + return + } + if len(res) == 0 { + return + } + e.docs[fileObjectId] = &Doc{Details: res[0].Details, IsLink: e.isLinkProcess} + fileObjectsIds = append(fileObjectsIds, fileObjectId) + }) + return nil + }) + if err != nil { + return nil, fmt.Errorf("get object from cache: %w", err) + } + return fileObjectsIds, nil +} + +func (e *exportContext) getExistedObjects(closure collect.Closure) error { + spaceIndex := e.objectStore.SpaceIndex(e.spaceId) + res, err := spaceIndex.List(false) + if err != nil { + return fmt.Errorf("list objects: %w", err) + } + if e.includeArchive { + archivedObjects, err := spaceIndex.List(true) + if err != nil { + return fmt.Errorf("list archived objects: %w", err) + } + res = append(res, archivedObjects...) + } + e.docs = make(map[string]*Doc, len(res)) + for _, info := range res { + objectSpaceID := e.spaceId + if objectSpaceID == "" { + objectSpaceID = info.Details.GetString(bundle.RelationKeySpaceId) + } + sbType, err := e.sbtProvider.Type(objectSpaceID, info.Id) + if err != nil { + log.With("objectId", info.Id).Errorf("failed to get smartblock type: %v", err) + continue + } + if !objectValid(sbType, info, e.includeArchive, closure) { + continue + } + e.docs[info.Id] = &Doc{Details: info.Details} + } + return nil +} + +func (e *exportContext) listTargetTypesFromTemplates(ids []string) []string { + for id, object := range e.docs { + if object.Details.Has(bundle.RelationKeyTargetObjectType) { + ids = append(ids, id) + } + } + return ids +} + +func isExcludedFromExport(details *domain.Details) bool { + return collect.Excluded(details) +} + +func objectValid(sbType smartblock.SmartBlockType, info *database.ObjectInfo, includeArchived bool, closure collect.Closure) bool { + if info.Id == addr.AnytypeProfileId { + return false + } + if closure == collect.ClosureContent && (!validTypeForContentClosure(sbType) || !validLayoutForContentClosure(info.Details)) { + return false + } + if closure == collect.ClosureDerived && !validType(sbType) { + return false + } + if strings.HasPrefix(info.Id, addr.BundledObjectTypeURLPrefix) || strings.HasPrefix(info.Id, addr.BundledRelationURLPrefix) { + return false + } + if info.Details.GetBool(bundle.RelationKeyIsArchived) && !includeArchived { + return false + } + return true +} + +func validType(sbType smartblock.SmartBlockType) bool { + return sbType == smartblock.SmartBlockTypeProfilePage || + sbType == smartblock.SmartBlockTypePage || + sbType == smartblock.SmartBlockTypeTemplate || + sbType == smartblock.SmartBlockTypeWorkspace || + sbType == smartblock.SmartBlockTypeWidget || + sbType == smartblock.SmartBlockTypeObjectType || + sbType == smartblock.SmartBlockTypeRelation || + sbType == smartblock.SmartBlockTypeRelationOption || + sbType == smartblock.SmartBlockTypeFileObject || + sbType == smartblock.SmartBlockTypeParticipant +} + +func validTypeForContentClosure(sbType smartblock.SmartBlockType) bool { + return sbType == smartblock.SmartBlockTypeProfilePage || + sbType == smartblock.SmartBlockTypePage || + sbType == smartblock.SmartBlockTypeFileObject +} + +func validLayoutForContentClosure(details *domain.Details) bool { + return details.GetInt64(bundle.RelationKeyResolvedLayout) != int64(model.ObjectType_collection) && + details.GetInt64(bundle.RelationKeyResolvedLayout) != int64(model.ObjectType_set) +} + +func listObjectIds(docs map[string]*Doc) []string { + ids := make([]string, 0, len(docs)) + for id := range docs { + ids = append(ids, id) + } + return ids +} + +func isLinkedObjectExist(rec []database.Record) bool { + return len(rec) > 0 && !rec[0].Details.GetBool(bundle.RelationKeyIsDeleted) +} diff --git a/core/block/export/collection_test.go b/core/block/export/collection_test.go new file mode 100644 index 0000000000..9906592fb2 --- /dev/null +++ b/core/block/export/collection_test.go @@ -0,0 +1,144 @@ +package export + +// collection_test.go covers the WHOLE-SPACE collection path — empty +// request ids, getExistedObjects — which every case in export_test.go +// bypasses by passing explicit ObjectIds. This is the branch where the +// closure mode actually gates object admission (objectValid), so an +// inverted closure mapping here would have been invisible to the rest of +// the suite. + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// seedWholeSpace fills the store with one object per admission rule the two +// closures disagree about: a plain page (both take it), a relation object +// (derived only — validType admits it, validTypeForContentClosure does +// not), a collection-layout page (derived only — validLayoutForContentClosure +// refuses the layout), and an archived page (either, but only behind +// IncludeArchived). Returns the sbType each id resolves to. +func seedWholeSpace(t *testing.T, fx *fixture) map[string]smartblock.SmartBlockType { + fx.store.AddObjects(t, spaceId, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String("pageId"), + bundle.RelationKeyName: domain.String("Plain page"), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + prepareTestRelationForStore(t, "customRelation", int64(model.RelationFormat_longtext)), + { + bundle.RelationKeyId: domain.String("collectionId"), + bundle.RelationKeyName: domain.String("A collection"), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_collection)), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + { + bundle.RelationKeyId: domain.String("archivedId"), + bundle.RelationKeyName: domain.String("Archived page"), + bundle.RelationKeyIsArchived: domain.Bool(true), + bundle.RelationKeySpaceId: domain.String(spaceId), + }, + }) + types := map[string]smartblock.SmartBlockType{ + "pageId": smartblock.SmartBlockTypePage, + "customRelation": smartblock.SmartBlockTypeRelation, + "collectionId": smartblock.SmartBlockTypePage, + "archivedId": smartblock.SmartBlockTypePage, + } + fx.sbtProvider.EXPECT().Type(spaceId, mock.Anything).RunAndReturn( + func(_ string, id string) (smartblock.SmartBlockType, error) { + return types[id], nil + }).Maybe() + return types +} + +// The content closure (md-style) admits pages and nothing derived; the +// derived closure admits everything validType lists. Both run through the +// same docsForExport the formats call, with no request ids — the branch +// nothing else covers. +// +// How this can fail: swap the closure arms in objectValid (the content run +// collects the relation and the derived run loses it — both assertions go +// red); drop the layout gate (the collection leaks into the content +// closure); or invert the IncludeArchived branch (the archived page +// appears without the flag). +func Test_docsForExport_WholeSpaceClosureRules(t *testing.T) { + t.Run("content closure: pages only, no derived, no collection layouts", func(t *testing.T) { + // given + fx := newFixture(t) + seedWholeSpace(t, fx) + expCtx := newExportContext(fx.export, pb.RpcObjectListExportRequest{ + SpaceId: spaceId, + Format: model.Export_Markdown, + }) + + // when + err := expCtx.docsForExport(context.Background()) + + // then + require.NoError(t, err) + assert.Contains(t, expCtx.docs, "pageId") + assert.NotContains(t, expCtx.docs, "customRelation", "a relation object is derived-closure only") + assert.NotContains(t, expCtx.docs, "collectionId", "a collection layout is derived-closure only") + assert.NotContains(t, expCtx.docs, "archivedId", "archived needs the flag") + assert.Len(t, expCtx.docs, 1) + }) + + t.Run("derived closure: relations and collections ride along, archived behind the flag", func(t *testing.T) { + // given + fx := newFixture(t) + types := seedWholeSpace(t, fx) + for id, sbType := range types { + fx.picker.EXPECT().GetObject(context.Background(), id). + Return(setupObject(id, "someType", sbType, nil), nil).Maybe() + } + expCtx := newExportContext(fx.export, pb.RpcObjectListExportRequest{ + SpaceId: spaceId, + Format: model.Export_Protobuf, + IncludeArchived: true, + }) + + // when + err := expCtx.docsForExport(context.Background()) + + // then + require.NoError(t, err) + assert.Contains(t, expCtx.docs, "pageId") + assert.Contains(t, expCtx.docs, "customRelation") + assert.Contains(t, expCtx.docs, "collectionId") + assert.Contains(t, expCtx.docs, "archivedId") + }) + + t.Run("derived closure without the flag drops the archived page", func(t *testing.T) { + // given + fx := newFixture(t) + types := seedWholeSpace(t, fx) + for id, sbType := range types { + fx.picker.EXPECT().GetObject(context.Background(), id). + Return(setupObject(id, "someType", sbType, nil), nil).Maybe() + } + expCtx := newExportContext(fx.export, pb.RpcObjectListExportRequest{ + SpaceId: spaceId, + Format: model.Export_Protobuf, + }) + + // when + err := expCtx.docsForExport(context.Background()) + + // then + require.NoError(t, err) + assert.NotContains(t, expCtx.docs, "archivedId") + assert.Contains(t, expCtx.docs, "pageId") + }) +} diff --git a/core/block/export/export.go b/core/block/export/export.go index 633f1dba61..09faa9206a 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -39,7 +39,6 @@ import ( "net/url" "os" "path/filepath" - "slices" "strconv" "strings" "sync" @@ -58,9 +57,9 @@ import ( sb "github.com/anyproto/anytype-heart/core/block/editor/smartblock" "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/core/block/editor/template" - "github.com/anyproto/anytype-heart/core/block/object/objectlink" + "github.com/anyproto/anytype-heart/core/block/export/anyblock" + "github.com/anyproto/anytype-heart/core/block/export/collect" "github.com/anyproto/anytype-heart/core/block/process" - "github.com/anyproto/anytype-heart/core/block/simple" "github.com/anyproto/anytype-heart/core/converter" "github.com/anyproto/anytype-heart/core/converter/dot" "github.com/anyproto/anytype-heart/core/converter/graphjson" @@ -74,9 +73,7 @@ import ( "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" - "github.com/anyproto/anytype-heart/pkg/lib/database" "github.com/anyproto/anytype-heart/pkg/lib/gateway" - "github.com/anyproto/anytype-heart/pkg/lib/localstore/addr" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" @@ -84,7 +81,6 @@ import ( "github.com/anyproto/anytype-heart/space/spacecore/typeprovider" "github.com/anyproto/anytype-heart/util/anyerror" "github.com/anyproto/anytype-heart/util/constant" - "github.com/anyproto/anytype-heart/util/slice" "github.com/anyproto/anytype-heart/util/text" ) @@ -110,11 +106,22 @@ var log = logging.Logger("anytype-mw-export") type Export interface { Export(ctx context.Context, req pb.RpcObjectListExportRequest) (path string, succeed int, err error) ExportSingleInMemory(ctx context.Context, spaceId string, objectId string, format model.ExportFormat) (res string, err error) + // Collector is the format-agnostic collection seam (collection.go): + // the native AnyBlock JSON exporter consumes it and nothing behind it. + collect.Collector app.Component } type export struct { - picker cache.ObjectGetter + // picker is typed CachedObjectGetter rather than ObjectGetter because + // the native AnyBlock JSON exporter closes every object it loads out of + // the cache — that is its memory model, not an optimisation + // (EXPORTER_DESIGN §1.5/§1.6), and anyblock.Exporter takes the wider + // type so a picker that cannot close is a compile error there. In + // production this resolves to the same component the narrow type did: + // core/block.Service is the only ObjectGetter the app registers, and it + // already answers the wider interface for the indexer. + picker cache.CachedObjectGetter objectStore objectstore.ObjectStore sbtProvider typeprovider.SmartBlockTypeProvider fileService files.Service @@ -134,7 +141,7 @@ func (e *export) Init(a *app.App) (err error) { e.processService = app.MustComponent[process.Service](a) e.objectStore = app.MustComponent[objectstore.ObjectStore](a) e.fileService = app.MustComponent[files.Service](a) - e.picker = app.MustComponent[cache.ObjectGetter](a) + e.picker = app.MustComponent[cache.CachedObjectGetter](a) e.sbtProvider = app.MustComponent[typeprovider.SmartBlockTypeProvider](a) e.spaceService = app.MustComponent[space.Service](a) e.accountService = app.MustComponent[account.Service](a) @@ -153,7 +160,7 @@ func (e *export) Export(ctx context.Context, req pb.RpcObjectListExportRequest) Id: bson.NewObjectId().Hex(), State: 0, Message: &pb.ModelProcessMessageOfExport{Export: &pb.ModelProcessExport{}}, - }, 4, req.NoProgress, e.notificationService) + }, exportWorkers(req.Format), req.NoProgress, e.notificationService) queue.SetMessage("prepare") if err = queue.Start(); err != nil { @@ -197,42 +204,12 @@ func (e *export) finishWithNotification(spaceId string, exportFormat model.Expor }, nil) } -type Doc struct { - Details *domain.Details - isLink bool -} - -func isExcludedFromExport(details *domain.Details) bool { - if details == nil { - return true - } - n := details.Len() - // Empty details or containing only id - if n <= 1 { - return true - } - // Details only with id + backlinks should be discarded - if n == 2 && details.Has(bundle.RelationKeyBacklinks) { - return true - } +// Doc and Docs alias the collection layer's types +// (core/block/export/collect), so the legacy exporter, its tests and the +// collect seam share one set of pointers with no conversion at the boundary. +type Doc = collect.Doc - id := details.GetString(bundle.RelationKeyId) - if domain.IsFileId(id) { - return true - } - - return false -} - -type Docs map[string]*Doc - -func (d Docs) transformToDetailsMap() map[string]*domain.Details { - details := make(map[string]*domain.Details, len(d)) - for id, doc := range d { - details[id] = doc.Details - } - return details -} +type Docs = collect.Docs type exportContext struct { spaceId string @@ -241,6 +218,7 @@ type exportContext struct { includeNested bool includeFiles bool format model.ExportFormat + closure collect.Closure isJson bool reqIds []string zip bool @@ -266,6 +244,7 @@ func newExportContext(e *export, req pb.RpcObjectListExportRequest) *exportConte includeNested: req.IncludeNested, includeFiles: req.IncludeFiles, format: req.Format, + closure: closureForFormat(req.Format), isJson: req.IsJson, reqIds: req.ObjectIds, zip: req.Zip, @@ -292,6 +271,7 @@ func (e *exportContext) copy() *exportContext { includeNested: e.includeNested, includeFiles: e.includeFiles, format: e.format, + closure: e.closure, isJson: e.isJson, reqIds: e.reqIds, export: e.export, @@ -306,7 +286,7 @@ func (e *exportContext) copy() *exportContext { } func (e *exportContext) getStateFilters(id string) *state.Filters { - if doc, ok := e.docs[id]; ok && doc.isLink { + if doc, ok := e.docs[id]; ok && doc.IsLink { return e.linkStateFilters } return nil @@ -314,6 +294,11 @@ func (e *exportContext) getStateFilters(id string) *state.Filters { // exportObject synchronously exports a single object and return the bytes slice func (e *exportContext) exportObject(ctx context.Context, objectId string) (string, error) { + if e.format == model.Export_AnyBlockJSON { + // one document, no bundle files, no dependency closure to collect + // for it — see exportSingleAnyBlockDocument (anyblockjson.go) + return e.exportSingleAnyBlockDocument(ctx, objectId) + } err := e.docsForExport(ctx) if err != nil { return "", fmt.Errorf("collect docs for export: %w", err) @@ -335,14 +320,11 @@ func (e *exportContext) exportObject(ctx context.Context, objectId string) (stri return "", fmt.Errorf("get object details: %w", err) } - // do not allow file export for in-memory writer - // nolint: gosec - switch model.ObjectTypeLayout(details.GetInt64(bundle.RelationKeyResolvedLayout)) { - case model.ObjectType_file, model.ObjectType_image, model.ObjectType_video, model.ObjectType_audio, model.ObjectType_pdf: - return "", fmt.Errorf("file export is not allowed for in-memory writer") + if err := refuseInMemoryFileObject(details); err != nil { + return "", err } - err = e.writeDoc(ctx, inMemoryWriter, objectId, e.docs.transformToDetailsMap()) + err = e.writeDoc(ctx, inMemoryWriter, objectId, e.docs.TransformToDetailsMap()) if err != nil { return "", fmt.Errorf("write doc: %w", err) } @@ -357,6 +339,18 @@ func (e *exportContext) exportObject(ctx context.Context, objectId string) (stri return "", nil } +// refuseInMemoryFileObject is the refusal every in-memory export owes a +// file object, in any format: the in-memory writer has nowhere to put the +// bytes, so a document that promises them would be a half answer. +func refuseInMemoryFileObject(details *domain.Details) error { + // nolint: gosec + switch model.ObjectTypeLayout(details.GetInt64(bundle.RelationKeyResolvedLayout)) { + case model.ObjectType_file, model.ObjectType_image, model.ObjectType_video, model.ObjectType_audio, model.ObjectType_pdf: + return fmt.Errorf("file export is not allowed for in-memory writer") + } + return nil +} + func (e *exportContext) exportObjects(ctx context.Context, queue process.Queue) (string, int, error) { var ( err error @@ -421,6 +415,10 @@ func (e *exportContext) exportByFormat(ctx context.Context, wr writer, queue pro succeed = e.exportDotAndSVG(ctx, succeed, wr, queue) } else if e.format == model.Export_GRAPH_JSON { succeed = e.exportGraphJson(ctx, succeed, wr, queue) + } else if e.format == model.Export_AnyBlockJSON { + // the native bundle exporter writes the whole tree itself — its own + // plan, emit and bundle files (anyblockjson.go) + return e.exportAnyBlockJSON(ctx, wr, queue) } else { tasks := make([]process.Task, 0, len(e.docs)) var succeedAsync int64 @@ -444,7 +442,7 @@ func (e *exportContext) exportDocs(ctx context.Context, succeed *int64, tasks []process.Task, ) []process.Task { - docsDetails := e.docs.transformToDetailsMap() + docsDetails := e.docs.TransformToDetailsMap() for docId, doc := range e.docs { if isExcludedFromExport(doc.Details) { continue @@ -464,7 +462,7 @@ func (e *exportContext) exportDocs(ctx context.Context, func (e *exportContext) exportGraphJson(ctx context.Context, succeed int, wr writer, queue process.Queue) int { mc := graphjson.NewMultiConverter(e.sbtProvider) - mc.SetKnownDocs(e.docs.transformToDetailsMap()) + mc.SetKnownDocs(e.docs.TransformToDetailsMap()) var werr error if succeed, werr = e.writeMultiDoc(ctx, mc, wr, queue); werr != nil { log.Warnf("can't export docs: %v", werr) @@ -478,7 +476,7 @@ func (e *exportContext) exportDotAndSVG(ctx context.Context, succeed int, wr wri format = dot.ExportFormatSVG } mc := dot.NewMultiConverter(format, e.sbtProvider) - mc.SetKnownDocs(e.docs.transformToDetailsMap()) + mc.SetKnownDocs(e.docs.TransformToDetailsMap()) var werr error if succeed, werr = e.writeMultiDoc(ctx, mc, wr, queue); werr != nil { log.Warnf("can't export docs: %v", werr) @@ -496,686 +494,58 @@ func (e *exportContext) renameZipArchive(wr writer, succeed int) (string, int, e return zipName, succeed, nil } -func isAnyblockExport(format model.ExportFormat) bool { - return format == model.Export_Protobuf || format == model.Export_JSON -} - -func (e *exportContext) docsForExport(ctx context.Context) (err error) { - isProtobuf := isAnyblockExport(e.format) - if len(e.reqIds) == 0 { - return e.getExistedObjects(isProtobuf) - } - - if len(e.reqIds) > 0 { - return e.getObjectsByIDs(ctx, isProtobuf) - } - return -} - -func (e *exportContext) getObjectsByIDs(ctx context.Context, isProtobuf bool) error { - res, err := e.queryAndFilterObjectsByRelation(e.spaceId, e.reqIds, bundle.RelationKeyId) - if err != nil { - return fmt.Errorf("query and filter objects by relation: %w", err) - } - for _, object := range res { - id := object.Details.GetString(bundle.RelationKeyId) - e.docs[id] = &Doc{Details: object.Details} - } - if e.includeSpace { - err = e.addSpaceToDocs(ctx) - if err != nil { - return fmt.Errorf("add space to docs: %w", err) - } - } - if isProtobuf { - if err := e.processProtobuf(); err != nil { - return fmt.Errorf("process protobuf: %w", err) - } - return nil - } - if err := e.processNotProtobuf(); err != nil { - return fmt.Errorf("process non-protobuf: %w", err) - } - return nil -} - -func (e *exportContext) queryAndFilterObjectsByRelation(spaceId string, reqIds []string, relationKey domain.RelationKey) ([]database.Record, error) { - var allObjects []database.Record - const singleBatchCount = 50 - for j := 0; j < len(reqIds); { - if j+singleBatchCount < len(reqIds) { - records, err := e.queryObjectsByRelation(spaceId, reqIds[j:j+singleBatchCount], relationKey) - if err != nil { - return nil, fmt.Errorf("query objects by relation: %w", err) - } - allObjects = append(allObjects, records...) - } else { - records, err := e.queryObjectsByRelation(spaceId, reqIds[j:], relationKey) - if err != nil { - return nil, fmt.Errorf("query objects by relation: %w", err) - } - allObjects = append(allObjects, records...) - } - j += singleBatchCount - } - return allObjects, nil -} - -func (e *exportContext) queryObjectsByRelation(spaceId string, reqIds []string, relationKey domain.RelationKey) ([]database.Record, error) { - return e.objectStore.SpaceIndex(spaceId).Query(database.Query{ - Filters: []database.FilterRequest{ - { - RelationKey: relationKey, - Condition: model.BlockContentDataviewFilter_In, - Value: domain.StringList(reqIds), - }, - }, - }) -} - -func (e *exportContext) addSpaceToDocs(ctx context.Context) error { - space, err := e.spaceService.Get(ctx, e.spaceId) - if err != nil { - return fmt.Errorf("get space: %w", err) - } - workspaceId := space.DerivedIDs().Workspace - records, err := e.objectStore.SpaceIndex(e.spaceId).QueryByIds([]string{workspaceId}) - if err != nil { - return fmt.Errorf("query workspace details: %w", err) - } - if len(records) == 0 { - return fmt.Errorf("no objects found for space %s", workspaceId) - } - e.docs[workspaceId] = &Doc{Details: records[0].Details, isLink: true} - return nil -} - -func (e *exportContext) processNotProtobuf() error { - ids := listObjectIds(e.docs) - if e.includeFiles { - fileObjectsIds, err := e.processFiles(ids) - if err != nil { - return fmt.Errorf("process files: %w", err) - } - ids = append(ids, fileObjectsIds...) - } - if e.includeNested { - for _, id := range ids { - e.addNestedObject(id, map[string]*Doc{}) - } - } - return nil -} - -func (e *exportContext) processProtobuf() error { - if !e.includeNested { - err := e.addDependentObjectsFromDataview() - if err != nil { - return fmt.Errorf("add dependent objects from dataview: %w", err) - } - } - ids := listObjectIds(e.docs) - if e.includeFiles { - err := e.addFileObjects(ids) - if err != nil { - return fmt.Errorf("add file objects: %w", err) - } - } - - err := e.addDerivedObjects() - if err != nil { - return fmt.Errorf("add derived objects: %w", err) - } - ids = e.listTargetTypesFromTemplates(ids) - if e.includeNested { - err = e.addNestedObjects(ids) - if err != nil { - return fmt.Errorf("add nested objects: %w", err) - } - } - return nil -} - -func (e *exportContext) addDependentObjectsFromDataview() error { - var ( - viewDependentObjectsIds []string - err error - ) - for id, doc := range e.docs { - if isExcludedFromExport(doc.Details) { - continue - } - if isObjectWithDataview(doc.Details) { - viewDependentObjectsIds, err = e.getViewDependentObjects(id, viewDependentObjectsIds) - if err != nil { - return fmt.Errorf("get view dependent objects: %w", err) - } - } - } - viewDependentObjects, err := e.queryAndFilterObjectsByRelation(e.spaceId, viewDependentObjectsIds, bundle.RelationKeyId) - if err != nil { - return fmt.Errorf("query dependent objects: %w", err) - } - templates, err := e.queryAndFilterObjectsByRelation(e.spaceId, viewDependentObjectsIds, bundle.RelationKeyTargetObjectType) - if err != nil { - return fmt.Errorf("query templates: %w", err) - } - for _, object := range append(viewDependentObjects, templates...) { - id := object.Details.GetString(bundle.RelationKeyId) - e.docs[id] = &Doc{ - Details: object.Details, - isLink: e.isLinkProcess, - } - } - return nil -} - -func (e *exportContext) getViewDependentObjects(id string, viewDependentObjectsIds []string) ([]string, error) { - err := cache.Do(e.picker, id, func(sb sb.SmartBlock) error { - st := sb.NewState().Copy().Filter(e.getStateFilters(id)) - viewDependentObjectsIds = append(viewDependentObjectsIds, - objectlink.DependentObjectIDs(st, sb.Space(), e.formatFetcher, objectlink.Flags{Blocks: true})...) - return nil - }) - if err != nil { - return nil, fmt.Errorf("get object from cache: %w", err) - } - return viewDependentObjectsIds, nil -} - -func (e *exportContext) addFileObjects(ids []string) error { - fileObjectsIds, err := e.processFiles(ids) - if err != nil { - return fmt.Errorf("process files: %w", err) - } - if e.includeNested { - err = e.addNestedObjects(fileObjectsIds) - if err != nil { - return fmt.Errorf("add nested objects: %w", err) - } - } - return nil -} - -func (e *exportContext) processFiles(ids []string) ([]string, error) { - var fileObjectsIds []string - for _, id := range ids { - objectFiles, err := e.fillLinkedFiles(id) - if err != nil { - return nil, fmt.Errorf("fill linked files: %w", err) - } - fileObjectsIds = lo.Union(fileObjectsIds, objectFiles) - } - return fileObjectsIds, nil -} - -func (e *exportContext) addDerivedObjects() error { - processedObjects := make(map[string]struct{}, 0) - err := e.getRelationsAndTypes(e.docs, processedObjects) - if err != nil { - return fmt.Errorf("get relations and types: %w", err) - } - - err = e.getTemplatesRelationsAndTypes(processedObjects) - if err != nil { - return fmt.Errorf("get templates relations and types: %w", err) - } - err = e.addRelationsAndTypes() - if err != nil { - return fmt.Errorf("add relations and types: %w", err) - } - return nil -} - -func (e *exportContext) getRelationsAndTypes(notProcessedObjects map[string]*Doc, processedObjects map[string]struct{}) error { - err := e.collectDerivedObjects(notProcessedObjects) - if err != nil { - return fmt.Errorf("collect derived objects: %w", err) - } - // get derived objects only from types, - // because relations currently have only system relations and object type - if len(e.objectTypes) > 0 || len(e.setOfList) > 0 { - err = e.getDerivedObjectsForTypes(processedObjects) - if err != nil { - return fmt.Errorf("get derived objects for types: %w", err) - } - } - return nil -} - -func (e *exportContext) collectDerivedObjects(objects map[string]*Doc) error { - for id, doc := range objects { - if doc != nil && isExcludedFromExport(doc.Details) { - continue - } - err := cache.Do(e.picker, id, func(b sb.SmartBlock) error { - state := b.NewState().Copy().Filter(e.getStateFilters(id)) - objectRelations := state.AllRelationKeys() - fillObjectsMap(e.relations, slice.IntoStrings(objectRelations)) - details := state.CombinedDetails() - if isObjectWithDataview(details) { - dataviewRelations, err := getDataviewRelations(state) - if err != nil { - return fmt.Errorf("get dataview relations: %w", err) - } - fillObjectsMap(e.relations, dataviewRelations) - } - var objectTypes []string - if details.Has(bundle.RelationKeyType) { - objectTypes = append(objectTypes, details.GetString(bundle.RelationKeyType)) - } - if details.Has(bundle.RelationKeyTargetObjectType) { - objectTypes = append(objectTypes, details.GetString(bundle.RelationKeyTargetObjectType)) - } - fillObjectsMap(e.objectTypes, objectTypes) - setOfList := details.GetStringList(bundle.RelationKeySetOf) - fillObjectsMap(e.setOfList, setOfList) - return nil - }) - if err != nil { - return fmt.Errorf("get object from cache: %w", err) - } - } - return nil -} - -func fillObjectsMap(dst map[string]struct{}, objectsToAdd []string) { - for _, objectId := range objectsToAdd { - dst[objectId] = struct{}{} - } -} - -func isObjectWithDataview(details *domain.Details) bool { - return details.GetInt64(bundle.RelationKeyResolvedLayout) == int64(model.ObjectType_collection) || - details.GetInt64(bundle.RelationKeyResolvedLayout) == int64(model.ObjectType_set) -} - -func getDataviewRelations(state *state.State) ([]string, error) { - var relations []string - err := state.Iterate(func(b simple.Block) (isContinue bool) { - if dataview := b.Model().GetDataview(); dataview != nil { - for _, view := range dataview.Views { - for _, relation := range view.Relations { - relations = append(relations, relation.Key) - } - } - } - return true - }) - if err != nil { - return nil, fmt.Errorf("iterate state blocks: %w", err) - } - return relations, nil -} - -func (e *exportContext) getDerivedObjectsForTypes(processedObjects map[string]struct{}) error { - notProceedTypes := make(map[string]*Doc) - for object := range e.objectTypes { - e.fillNotProcessedTypes(processedObjects, object, notProceedTypes) - } - for object := range e.setOfList { - e.fillNotProcessedTypes(processedObjects, object, notProceedTypes) - } - if len(notProceedTypes) == 0 { - return nil - } - err := e.getRelationsAndTypes(notProceedTypes, processedObjects) - if err != nil { - return fmt.Errorf("get relations and types: %w", err) - } - return nil -} - -func (e *exportContext) fillNotProcessedTypes(processedObjects map[string]struct{}, object string, notProceedTypes map[string]*Doc) { - if _, ok := processedObjects[object]; ok { - return - } - notProceedTypes[object] = nil - processedObjects[object] = struct{}{} -} - -func (e *exportContext) getTemplatesRelationsAndTypes(processedObjects map[string]struct{}) error { - allTypes := lo.MapToSlice(e.objectTypes, func(key string, value struct{}) string { return key }) - templates, err := e.queryAndFilterObjectsByRelation(e.spaceId, allTypes, bundle.RelationKeyTargetObjectType) - if err != nil { - return fmt.Errorf("query templates by target type: %w", err) - } - if len(templates) == 0 { - return nil - } - templatesToProcess := make(map[string]*Doc, len(templates)) - for _, template := range templates { - id := template.Details.GetString(bundle.RelationKeyId) - if _, ok := e.docs[id]; !ok { - templateDoc := &Doc{Details: template.Details, isLink: e.isLinkProcess} - e.docs[id] = templateDoc - templatesToProcess[id] = templateDoc - } - } - err = e.getRelationsAndTypes(templatesToProcess, processedObjects) - if err != nil { - return fmt.Errorf("get relations and types for templates: %w", err) - } - return nil -} - -func (e *exportContext) addRelationsAndTypes() error { - types := lo.MapToSlice(e.objectTypes, func(key string, value struct{}) string { return key }) - setOfList := lo.MapToSlice(e.setOfList, func(key string, value struct{}) string { return key }) - relations := lo.MapToSlice(e.relations, func(key string, value struct{}) string { return key }) - - err := e.addRelations(relations) - if err != nil { - return fmt.Errorf("add relations: %w", err) - } - err = e.processObjectTypesAndSetOfList(types, setOfList) - if err != nil { - return fmt.Errorf("process object types and set of list: %w", err) - } - return nil -} - -func (e *exportContext) addRelations(relations []string) error { - storeRelations, err := e.getRelationsFromStore(relations) - if err != nil { - return fmt.Errorf("get relations from store: %w", err) - } - for _, storeRelation := range storeRelations { - e.addRelation(storeRelation) - err := e.addOptionIfTag(storeRelation) - if err != nil { - return fmt.Errorf("add option if tag: %w", err) - } - } - return nil -} - -func (e *exportContext) getRelationsFromStore(relations []string) ([]database.Record, error) { - uniqueKeys := make([]string, 0, len(relations)) - for _, relation := range relations { - uniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeRelation, relation) - if err != nil { - return nil, fmt.Errorf("create unique key for relation: %w", err) - } - uniqueKeys = append(uniqueKeys, uniqueKey.Marshal()) - } - storeRelations, err := e.queryAndFilterObjectsByRelation(e.spaceId, uniqueKeys, bundle.RelationKeyUniqueKey) - if err != nil { - return nil, fmt.Errorf("query relations by unique key: %w", err) - } - return storeRelations, nil -} - -func (e *exportContext) addRelation(relation database.Record) { - relationKey := domain.RelationKey(relation.Details.GetString(bundle.RelationKeyRelationKey)) - if relationKey != "" { - id := relation.Details.GetString(bundle.RelationKeyId) - e.docs[id] = &Doc{Details: relation.Details, isLink: e.isLinkProcess} - } -} - -func (e *exportContext) addOptionIfTag(relation database.Record) error { - format := relation.Details.GetInt64(bundle.RelationKeyRelationFormat) - relationKey := relation.Details.GetString(bundle.RelationKeyRelationKey) - if format == int64(model.RelationFormat_tag) || format == int64(model.RelationFormat_status) { - err := e.addRelationOptions(relationKey) - if err != nil { - return fmt.Errorf("add relation options: %w", err) - } - } - return nil -} - -func (e *exportContext) addRelationOptions(relationKey string) error { - relationOptions, err := e.getRelationOptions(relationKey) - if err != nil { - return fmt.Errorf("get relation options: %w", err) - } - for _, option := range relationOptions { - id := option.Details.GetString(bundle.RelationKeyId) - e.docs[id] = &Doc{Details: option.Details, isLink: e.isLinkProcess} - } - return nil -} - -func (e *exportContext) getRelationOptions(relationKey string) ([]database.Record, error) { - relationOptionsDetails, err := e.objectStore.SpaceIndex(e.spaceId).Query(database.Query{ - Filters: []database.FilterRequest{ - { - RelationKey: bundle.RelationKeyResolvedLayout, - Condition: model.BlockContentDataviewFilter_Equal, - Value: domain.Int64(model.ObjectType_relationOption), - }, - { - RelationKey: bundle.RelationKeyRelationKey, - Condition: model.BlockContentDataviewFilter_Equal, - Value: domain.String(relationKey), - }, - }, - }) - if err != nil { - return nil, fmt.Errorf("query relation options: %w", err) - } - return relationOptionsDetails, nil -} - -func (e *exportContext) processObjectTypesAndSetOfList(objectTypes, setOfList []string) error { - objectDetails, err := e.queryAndFilterObjectsByRelation(e.spaceId, lo.Union(objectTypes, setOfList), bundle.RelationKeyId) - if err != nil { - return fmt.Errorf("query object types: %w", err) - } - if len(objectDetails) == 0 { - return nil - } - recommendedRelations, err := e.addObjectsAndCollectRecommendedRelations(objectDetails) - if err != nil { - return fmt.Errorf("collect recommended relations: %w", err) - } - err = e.addRecommendedRelations(recommendedRelations) - if err != nil { - return fmt.Errorf("add recommended relations: %w", err) - } - return nil -} - -func (e *exportContext) addObjectsAndCollectRecommendedRelations(objectTypes []database.Record) ([]string, error) { - recommendedRelations := make([]string, 0, len(objectTypes)) - for i := 0; i < len(objectTypes); i++ { - rawUniqueKey := objectTypes[i].Details.GetString(bundle.RelationKeyUniqueKey) - uniqueKey, err := domain.UnmarshalUniqueKey(rawUniqueKey) - if err != nil { - return nil, fmt.Errorf("unmarshal unique key: %w", err) - } - id := objectTypes[i].Details.GetString(bundle.RelationKeyId) - e.docs[id] = &Doc{Details: objectTypes[i].Details, isLink: e.isLinkProcess} - if uniqueKey.SmartblockType() == smartblock.SmartBlockTypeObjectType { - key, err := domain.GetTypeKeyFromRawUniqueKey(rawUniqueKey) - if err != nil { - return nil, fmt.Errorf("get type key from unique key: %w", err) - } - if bundle.IsInternalType(key) { - continue - } - recommendedRelations = lo.Uniq(slices.Concat(recommendedRelations, - objectTypes[i].Details.GetStringList(bundle.RelationKeyRecommendedRelations), - objectTypes[i].Details.GetStringList(bundle.RelationKeyRecommendedHiddenRelations), - objectTypes[i].Details.GetStringList(bundle.RelationKeyRecommendedFeaturedRelations), - objectTypes[i].Details.GetStringList(bundle.RelationKeyRecommendedFileRelations), - )) - } - } - return recommendedRelations, nil -} - -func (e *exportContext) addRecommendedRelations(recommendedRelations []string) error { - relations, err := e.queryAndFilterObjectsByRelation(e.spaceId, recommendedRelations, bundle.RelationKeyId) - if err != nil { - return fmt.Errorf("query recommended relations: %w", err) - } - for _, relation := range relations { - id := relation.Details.GetString(bundle.RelationKeyId) - if id == addr.MissingObject { - continue - } - - relationKey := relation.Details.GetString(bundle.RelationKeyUniqueKey) - uniqueKey, err := domain.UnmarshalUniqueKey(relationKey) - if err != nil { - return fmt.Errorf("unmarshal relation unique key: %w", err) - } - if bundle.IsSystemRelation(domain.RelationKey(uniqueKey.InternalKey())) { - continue - } - e.docs[id] = &Doc{Details: relation.Details, isLink: e.isLinkProcess} - } - return nil -} - -func (e *exportContext) addNestedObjects(ids []string) error { - nestedDocs := make(map[string]*Doc, 0) - for _, id := range ids { - e.addNestedObject(id, nestedDocs) - } - if len(nestedDocs) == 0 { - return nil - } - exportCtxChild := e.copy() - exportCtxChild.includeNested = false - exportCtxChild.docs = nestedDocs - exportCtxChild.isLinkProcess = true - err := exportCtxChild.processProtobuf() - if err != nil { - return fmt.Errorf("process nested protobuf: %w", err) - } - for id, object := range exportCtxChild.docs { - if _, ok := e.docs[id]; !ok { - e.docs[id] = object - } - } - return nil -} - -func (e *exportContext) addNestedObject(id string, nestedDocs map[string]*Doc) { - if doc, ok := e.docs[id]; ok && isExcludedFromExport(doc.Details) { - return - } - var links []string - err := cache.Do(e.picker, id, func(sb sb.SmartBlock) error { - st := sb.NewState().Copy().Filter(e.getStateFilters(id)) - links = objectlink.DependentObjectIDs(st, sb.Space(), e.formatFetcher, objectlink.Flags{ - Blocks: true, - Details: true, - Collection: true, - NoHiddenBundledRelations: true, - NoBackLinks: !e.includeBackLinks, - CreatorModifierWorkspace: true, - }) - return nil - }) - if err != nil { - return - } - for _, link := range links { - if _, exists := e.docs[link]; !exists { - sbt, sbtErr := e.sbtProvider.Type(e.spaceId, link) - if sbtErr != nil { - log.Errorf("failed to get smartblocktype of id %s", link) - continue - } - if !validType(sbt) { - continue - } - rec, qErr := e.objectStore.SpaceIndex(e.spaceId).QueryByIds([]string{link}) - if qErr != nil { - log.Errorf("failed to query id %s, err: %s", qErr, err) - continue - } - if isLinkedObjectExist(rec) { - exportDoc := &Doc{Details: rec[0].Details, isLink: true} - nestedDocs[link] = exportDoc - e.docs[link] = exportDoc - e.addNestedObject(link, nestedDocs) - } - } +// exportWorkers is the export queue's width. The legacy formats keep the 4 +// they have always run at. The native AnyBlock JSON bundle runs its emit AS +// queue tasks, and each of those cold-builds one object into the space's +// object cache before closing it again, so for that format the queue's +// width IS the resident content set the design measured +// (EXPORTER_DESIGN §1.5/§1.6) — which is half as wide on mobile. +func exportWorkers(format model.ExportFormat) int { + if format == model.Export_AnyBlockJSON { + return anyblock.EmitWidth() + } + return 4 +} + +// closureForFormat maps an export format onto the collection closure it +// runs. The SELF-CONTAINED formats — the two protobuf renderings and the +// native AnyBlock JSON bundle — take the derived closure, so types, +// relations, options and templates travel with the objects; markdown, dot, +// svg and graphjson take content only (collect.Closure, design §1.1). +// +// The predicate this replaced was spelled isAnyblockExport and meant +// "protobuf or pb.json" — a name that predates the AnyBlock JSON format and +// now reads as its exact opposite, five lines from the routing that decides +// what an AnyBlock JSON export collects. Listing the formats here costs one +// switch and cannot be misread. +func closureForFormat(format model.ExportFormat) collect.Closure { + switch format { + case model.Export_Protobuf, model.Export_JSON, model.Export_AnyBlockJSON: + return collect.ClosureDerived + default: + return collect.ClosureContent } } -func (e *exportContext) fillLinkedFiles(id string) ([]string, error) { - if doc, ok := e.docs[id]; ok && isExcludedFromExport(doc.Details) { - return nil, nil - } - spaceIndex := e.objectStore.SpaceIndex(e.spaceId) - var fileObjectsIds []string - err := cache.Do(e.picker, id, func(b sb.SmartBlock) error { - b.NewState().Copy().Filter(e.getStateFilters(id)).IterateLinkedFiles(e.formatFetcher, func(fileObjectId string) { - res, err := spaceIndex.Query(database.Query{ - Filters: []database.FilterRequest{ - { - RelationKey: bundle.RelationKeyId, - Condition: model.BlockContentDataviewFilter_Equal, - Value: domain.String(fileObjectId), - }, - }, - }) - if err != nil { - log.Errorf("failed to get details for file object id %s: %v", fileObjectId, err) - return - } - if len(res) == 0 { - return - } - e.docs[fileObjectId] = &Doc{Details: res[0].Details, isLink: e.isLinkProcess} - fileObjectsIds = append(fileObjectsIds, fileObjectId) - }) - return nil +func (e *exportContext) docsForExport(ctx context.Context) (err error) { + docs, err := e.export.Collect(ctx, collect.Request{ + SpaceId: e.spaceId, + Ids: e.reqIds, + Closure: e.closure, + IncludeNested: e.includeNested, + IncludeFiles: e.includeFiles, + IncludeArchived: e.includeArchive, + IncludeBacklinks: e.includeBackLinks, + IncludeSpace: e.includeSpace, + StateFilters: e.linkStateFilters, }) if err != nil { - return nil, fmt.Errorf("get object from cache: %w", err) - } - return fileObjectsIds, nil -} - -func (e *exportContext) getExistedObjects(isProtobuf bool) error { - spaceIndex := e.objectStore.SpaceIndex(e.spaceId) - res, err := spaceIndex.List(false) - if err != nil { - return fmt.Errorf("list objects: %w", err) - } - if e.includeArchive { - archivedObjects, err := spaceIndex.List(true) - if err != nil { - return fmt.Errorf("list archived objects: %w", err) - } - res = append(res, archivedObjects...) - } - e.docs = make(map[string]*Doc, len(res)) - for _, info := range res { - objectSpaceID := e.spaceId - if objectSpaceID == "" { - objectSpaceID = info.Details.GetString(bundle.RelationKeySpaceId) - } - sbType, err := e.sbtProvider.Type(objectSpaceID, info.Id) - if err != nil { - log.With("objectId", info.Id).Errorf("failed to get smartblock type: %v", err) - continue - } - if !objectValid(sbType, info, e.includeArchive, isProtobuf) { - continue - } - e.docs[info.Id] = &Doc{Details: info.Details} + return err } + e.docs = docs return nil } -func (e *exportContext) listTargetTypesFromTemplates(ids []string) []string { - for id, object := range e.docs { - if object.Details.Has(bundle.RelationKeyTargetObjectType) { - ids = append(ids, id) - } - } - return ids -} - func (e *exportContext) writeMultiDoc(ctx context.Context, mw converter.MultiConverter, wr writer, queue process.Queue) (succeed int, err error) { for did, doc := range e.docs { if isExcludedFromExport(doc.Details) { @@ -1401,25 +771,6 @@ func provideFileDirectory(blockType smartblock.SmartBlockType) string { } } -func objectValid(sbType smartblock.SmartBlockType, info *database.ObjectInfo, includeArchived bool, isProtobuf bool) bool { - if info.Id == addr.AnytypeProfileId { - return false - } - if !isProtobuf && (!validTypeForNonProtobuf(sbType) || !validLayoutForNonProtobuf(info.Details)) { - return false - } - if isProtobuf && !validType(sbType) { - return false - } - if strings.HasPrefix(info.Id, addr.BundledObjectTypeURLPrefix) || strings.HasPrefix(info.Id, addr.BundledRelationURLPrefix) { - return false - } - if info.Details.GetBool(bundle.RelationKeyIsArchived) && !includeArchived { - return false - } - return true -} - func newNamer() *namer { return &namer{ names: make(map[string]string), @@ -1463,30 +814,6 @@ func (fn *namer) Get(path, hash, title, ext string) (name string) { } } -func validType(sbType smartblock.SmartBlockType) bool { - return sbType == smartblock.SmartBlockTypeProfilePage || - sbType == smartblock.SmartBlockTypePage || - sbType == smartblock.SmartBlockTypeTemplate || - sbType == smartblock.SmartBlockTypeWorkspace || - sbType == smartblock.SmartBlockTypeWidget || - sbType == smartblock.SmartBlockTypeObjectType || - sbType == smartblock.SmartBlockTypeRelation || - sbType == smartblock.SmartBlockTypeRelationOption || - sbType == smartblock.SmartBlockTypeFileObject || - sbType == smartblock.SmartBlockTypeParticipant -} - -func validTypeForNonProtobuf(sbType smartblock.SmartBlockType) bool { - return sbType == smartblock.SmartBlockTypeProfilePage || - sbType == smartblock.SmartBlockTypePage || - sbType == smartblock.SmartBlockTypeFileObject -} - -func validLayoutForNonProtobuf(details *domain.Details) bool { - return details.GetInt64(bundle.RelationKeyResolvedLayout) != int64(model.ObjectType_collection) && - details.GetInt64(bundle.RelationKeyResolvedLayout) != int64(model.ObjectType_set) -} - func cleanupFile(wr writer) { if wr == nil { return @@ -1495,18 +822,6 @@ func cleanupFile(wr writer) { os.Remove(wr.Path()) } -func listObjectIds(docs map[string]*Doc) []string { - ids := make([]string, 0, len(docs)) - for id := range docs { - ids = append(ids, id) - } - return ids -} - -func isLinkedObjectExist(rec []database.Record) bool { - return len(rec) > 0 && !rec[0].Details.GetBool(bundle.RelationKeyIsDeleted) -} - func pbFiltersToState(filters *pb.RpcObjectListExportStateFilters) *state.Filters { if filters == nil { return nil @@ -1532,7 +847,7 @@ func (e *exportContext) postProcess(ctx context.Context, wr writer) error { return nil } // Create a lazy object resolver - knownObjects := e.docs.transformToDetailsMap() + knownObjects := e.docs.TransformToDetailsMap() resolver := newLazyObjectResolver(e.objectStore, e.spaceId) // Create markdown post-processor diff --git a/core/block/export/export_test.go b/core/block/export/export_test.go index 2e6082bae3..642371665e 100644 --- a/core/block/export/export_test.go +++ b/core/block/export/export_test.go @@ -46,7 +46,12 @@ const spaceId = "space1" type fixture struct { *export - picker *mock_cache.MockObjectGetter + // the picker mock is the CACHED getter because the service field is: + // the native AnyBlock JSON format closes every object it loads + // (EXPORTER_DESIGN §1.5/§1.6), so the narrower ObjectGetter no longer + // satisfies it. Strictly richer — the legacy formats never call + // TryRemoveFromCache, and the mock would fail the test if they did. + picker *mock_cache.MockCachedObjectGetter store *objectstore.StoreFixture sbtProvider *mock_typeprovider.MockSmartBlockTypeProvider notifications *mock_notifications.MockNotifications @@ -56,7 +61,7 @@ type fixture struct { } func newFixture(t *testing.T) *fixture { - objectGetter := mock_cache.NewMockObjectGetter(t) + objectGetter := mock_cache.NewMockCachedObjectGetter(t) storeFixture := objectstore.NewStoreFixture(t) provider := mock_typeprovider.NewMockSmartBlockTypeProvider(t) notifications := mock_notifications.NewMockNotifications(t) diff --git a/core/block/export/writer.go b/core/block/export/writer.go index c305e2c013..e0afce86a7 100644 --- a/core/block/export/writer.go +++ b/core/block/export/writer.go @@ -89,6 +89,19 @@ func (d *dirWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int return } +// RemoveFile deletes one file below the export root. Nothing in the legacy +// formats calls it: it exists for the native AnyBlock JSON exporter's +// un-write hook, which reaches it through an optional interface assertion +// — a blob whose stream fails half way is worse left truncated than +// missing (core/block/export/anyblock, emitDoc). A zip export cannot offer +// the same, since its entries are already streamed. +func (d *dirWriter) RemoveFile(filename string) error { + if err := os.Remove(filepath.Join(d.path, filepath.FromSlash(filename))); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove file: %w", err) + } + return nil +} + func (d *dirWriter) Close() (err error) { return nil } diff --git a/core/block/import/common/objectcreator/objectcreator.go b/core/block/import/common/objectcreator/objectcreator.go index 9fd7fccbcd..82b0469662 100644 --- a/core/block/import/common/objectcreator/objectcreator.go +++ b/core/block/import/common/objectcreator/objectcreator.go @@ -383,17 +383,45 @@ func (oc *ObjectCreator) setWorkspaceDetails(spaceID string, st *state.State) { } } +// isDerivedFromBundledObject reports whether the incoming snapshot claims to be a copy of a +// bundled object. Revision numbers version bundled definitions, so comparing them is only +// meaningful between two copies of the same bundled object. A user's own object carries no +// revision, and comparing its implicit 0 against a bundled object's revision would silently +// drop the whole imported state. +func isDerivedFromBundledObject(st *state.State) bool { + return st.Details().GetInt64(bundle.RelationKeyRevision) > 0 || + st.Details().GetString(bundle.RelationKeySourceObject) != "" +} + +// preserveBundledIdentity keeps sourceObject and revision of the object we are resetting when the +// incoming snapshot carries neither. Resetting to a version drops details the new state omits, and +// these two tie an installed object back to its bundled definition: sourceObject is what +// InstallBundledObjects matches on to see an object is already installed, and revision is what +// SystemObjectReviser compares against the bundle. A user's snapshot landing on such an object +// should replace its content, not sever that link. +func preserveBundledIdentity(b smartblock.SmartBlock, st *state.State) { + if isDerivedFromBundledObject(st) { + return + } + for _, key := range []domain.RelationKey{bundle.RelationKeySourceObject, bundle.RelationKeyRevision} { + if value := b.Details().Get(key); value.Ok() { + st.SetDetail(key, value) + } + } +} + func (oc *ObjectCreator) resetState(newID string, st *state.State) *domain.Details { var respDetails *domain.Details err := cache.Do(oc.objectGetterDeleter, newID, func(b smartblock.SmartBlock) error { currentRevision := b.Details().GetInt64(bundle.RelationKeyRevision) newRevision := st.Details().GetInt64(bundle.RelationKeyRevision) - if currentRevision > newRevision { + // never update objects with older revision + // we use revision for bundled objects like relations and object types + if isDerivedFromBundledObject(st) && currentRevision > newRevision { log.With(zap.String("object id", newID)).Warnf("skipping object %s, revision %d > %d", st.Details().GetString(bundle.RelationKeyUniqueKey), currentRevision, newRevision) - // never update objects with older revision - // we use revision for bundled objects like relations and object types return nil } + preserveBundledIdentity(b, st) if st.ObjectTypeKey() == bundle.TypeKeyObjectType { template.InitTemplate(st, template.WithDetail(bundle.RelationKeyRecommendedLayout, domain.Int64(model.ObjectType_basic))) } diff --git a/core/block/import/common/objectcreator/objectcreator_test.go b/core/block/import/common/objectcreator/objectcreator_test.go index 1ffa5a2925..dbe005c200 100644 --- a/core/block/import/common/objectcreator/objectcreator_test.go +++ b/core/block/import/common/objectcreator/objectcreator_test.go @@ -9,6 +9,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/treestorage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/anyproto/anytype-heart/core/block/detailservice/mock_detailservice" "github.com/anyproto/anytype-heart/core/block/editor/smartblock" @@ -201,6 +202,160 @@ func (g *dumbObjectGetter) DeleteObject(id string) error { return nil } +// resetRecorder captures the state resetState hands to ResetToVersion, which smarttest ignores. +type resetRecorder struct { + *smarttest.SmartTest + resetTo *state.State +} + +func (r *resetRecorder) ResetToVersion(st *state.State) error { + r.resetTo = st + return nil +} + +func TestObjectCreator_resetState(t *testing.T) { + const objectId = "bundledProjectId" + + // newFixture returns an object already in the space: the bundled Project type at revision 3. + newFixture := func(t *testing.T) (*resetRecorder, ObjectCreator) { + existing := smarttest.New(objectId) + st := existing.NewState() + st.SetDetails(domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyId: domain.String(objectId), + bundle.RelationKeyName: domain.String("Project"), + bundle.RelationKeyUniqueKey: domain.String("ot-project"), + bundle.RelationKeySourceObject: domain.String("_otproject"), + bundle.RelationKeyRevision: domain.Int64(3), + })) + require.NoError(t, existing.Apply(st)) + + recorder := &resetRecorder{SmartTest: existing} + return recorder, ObjectCreator{ + objectGetterDeleter: newDumbObjectGetter(map[string]smartblock.SmartBlock{objectId: recorder}), + } + } + + // newSnapshot builds an incoming object type state; revision 0 and an empty source object + // stand for a type the user authored themselves. + newSnapshot := func(revision int64, sourceObject string) *state.State { + st := state.NewDoc(objectId, nil).(*state.State) + details := domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyId: domain.String(objectId), + bundle.RelationKeyName: domain.String("Project"), + bundle.RelationKeyUniqueKey: domain.String("ot-project"), + bundle.RelationKeyDescription: domain.String("An initiative the team has committed to"), + }) + if revision > 0 { + details.SetInt64(bundle.RelationKeyRevision, revision) + } + if sourceObject != "" { + details.SetString(bundle.RelationKeySourceObject, sourceObject) + } + st.SetDetails(details) + st.SetObjectTypeKey(bundle.TypeKeyObjectType) + return st + } + + t.Run("applies a user's own type over an existing object carrying a revision", func(t *testing.T) { + // given + recorder, oc := newFixture(t) + st := newSnapshot(0, "") + + // when + oc.resetState(objectId, st) + + // then + require.NotNil(t, recorder.resetTo, "an object without a revision is not a bundled object, so it must not be skipped") + assert.Equal(t, "An initiative the team has committed to", recorder.resetTo.Details().GetString(bundle.RelationKeyDescription)) + }) + + t.Run("keeps the bundled identity of the object it resets", func(t *testing.T) { + // given + recorder, oc := newFixture(t) + st := newSnapshot(0, "") + + // when + oc.resetState(objectId, st) + + // then + require.NotNil(t, recorder.resetTo) + assert.Equal(t, "_otproject", recorder.resetTo.Details().GetString(bundle.RelationKeySourceObject)) + assert.Equal(t, int64(3), recorder.resetTo.Details().GetInt64(bundle.RelationKeyRevision)) + }) + + t.Run("does not invent a bundled identity the object never had", func(t *testing.T) { + // given + existing := smarttest.New(objectId) + existingState := existing.NewState() + existingState.SetDetails(domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyId: domain.String(objectId), + bundle.RelationKeyName: domain.String("Project"), + })) + require.NoError(t, existing.Apply(existingState)) + recorder := &resetRecorder{SmartTest: existing} + oc := ObjectCreator{ + objectGetterDeleter: newDumbObjectGetter(map[string]smartblock.SmartBlock{objectId: recorder}), + } + + // when + oc.resetState(objectId, newSnapshot(0, "")) + + // then + require.NotNil(t, recorder.resetTo) + assert.False(t, recorder.resetTo.Details().Has(bundle.RelationKeySourceObject)) + assert.False(t, recorder.resetTo.Details().Has(bundle.RelationKeyRevision)) + }) + + t.Run("skips a bundled object with an older revision", func(t *testing.T) { + // given + recorder, oc := newFixture(t) + st := newSnapshot(2, "_otproject") + + // when + oc.resetState(objectId, st) + + // then + assert.Nil(t, recorder.resetTo) + }) + + t.Run("skips a bundled object whose revision is absent", func(t *testing.T) { + // given + recorder, oc := newFixture(t) + st := newSnapshot(0, "_otproject") + + // when + oc.resetState(objectId, st) + + // then + assert.Nil(t, recorder.resetTo) + }) + + t.Run("applies a bundled object with an equal revision", func(t *testing.T) { + // given + recorder, oc := newFixture(t) + st := newSnapshot(3, "_otproject") + + // when + oc.resetState(objectId, st) + + // then + require.NotNil(t, recorder.resetTo) + assert.Equal(t, "An initiative the team has committed to", recorder.resetTo.Details().GetString(bundle.RelationKeyDescription)) + }) + + t.Run("applies a bundled object with a newer revision", func(t *testing.T) { + // given + recorder, oc := newFixture(t) + st := newSnapshot(4, "_otproject") + + // when + oc.resetState(objectId, st) + + // then + require.NotNil(t, recorder.resetTo) + }) +} + func TestObjectCreator_createNewObject(t *testing.T) { t.Run("collection store IDs are replaced during object creation", func(t *testing.T) { // given diff --git a/core/block/import/common/objectid/deriveobject_test.go b/core/block/import/common/objectid/deriveobject_test.go index 74d161afb2..d47bea8c19 100644 --- a/core/block/import/common/objectid/deriveobject_test.go +++ b/core/block/import/common/objectid/deriveobject_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/anyproto/anytype-heart/core/block/import/common" + "github.com/anyproto/anytype-heart/core/block/object/payloadcreator" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/domain/objectorigin" "github.com/anyproto/anytype-heart/pkg/lib/bundle" @@ -104,3 +105,215 @@ func TestDerivedObject_GetIDAndPayload(t *testing.T) { assert.Equal(t, "oldId", id) }) } + +func TestDerivedObject_GetIDAndPayload_ObjectType(t *testing.T) { + const bundledProjectId = "bundledProjectId" + + // newObjectTypeSnapshot builds an imported object type snapshot with the given unique key + // and name; an empty unique key stands for a legacy export that predates unique keys. + newObjectTypeSnapshot := func(uniqueKey, name string) *common.Snapshot { + details := domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String(name), + }) + if uniqueKey != "" { + details.SetString(bundle.RelationKeyUniqueKey, uniqueKey) + } + return &common.Snapshot{ + Id: "type-project", + Snapshot: &common.SnapshotModel{ + SbType: coresb.SmartBlockTypeObjectType, + Data: &common.StateSnapshot{Key: "pmProject", Details: details}, + }, + } + } + + // newFixture returns a store already holding the bundled Project type, as every space does. + newFixture := func(t *testing.T) (*objectstore.StoreFixture, *derivedObject) { + sf := objectstore.NewStoreFixture(t) + sf.AddObjects(t, "spaceId", []objectstore.TestObject{ + { + bundle.RelationKeyId: domain.String(bundledProjectId), + bundle.RelationKeyUniqueKey: domain.String("ot-project"), + bundle.RelationKeyName: domain.String("Project"), + bundle.RelationKeySourceObject: domain.String("_otproject"), + bundle.RelationKeyRevision: domain.Int64(3), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_objectType)), + bundle.RelationKeySpaceId: domain.String("spaceId"), + }, + }) + service := mock_space.NewMockService(t) + space := mock_clientspace.NewMockSpace(t) + service.EXPECT().Get(mock.Anything, "spaceId").Return(space, nil).Maybe() + space.EXPECT().DeriveTreePayload(mock.Anything, mock.Anything).Return(treestorage.TreeStorageCreatePayload{ + RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: "freshTypeId"}, + }, nil).Maybe() + return sf, newDerivedObject(newExistingObject(sf), service, sf) + } + + t.Run("type with own unique key is not merged into a same-named type", func(t *testing.T) { + // given + _, deriveObject := newFixture(t) + sn := newObjectTypeSnapshot("ot-pmProject", "Project") + + // when + id, payload, err := deriveObject.GetIDAndPayload(context.Background(), "spaceId", sn, time.Now(), false, objectorigin.Import(model.Import_Pb)) + + // then + assert.Nil(t, err) + assert.Equal(t, "freshTypeId", id) + assert.NotNil(t, payload.RootRawChange, "a new type must be created, not merged into the bundled one") + }) + + t.Run("type is merged into an existing type with the same unique key", func(t *testing.T) { + // given + _, deriveObject := newFixture(t) + sn := newObjectTypeSnapshot("ot-project", "Project renamed by the user") + + // when + id, payload, err := deriveObject.GetIDAndPayload(context.Background(), "spaceId", sn, time.Now(), false, objectorigin.Import(model.Import_Pb)) + + // then + assert.Nil(t, err) + assert.Equal(t, bundledProjectId, id) + assert.Nil(t, payload.RootRawChange) + }) + + t.Run("legacy type without unique key is merged by name", func(t *testing.T) { + // given + _, deriveObject := newFixture(t) + sn := newObjectTypeSnapshot("", "Project") + + // when + id, payload, err := deriveObject.GetIDAndPayload(context.Background(), "spaceId", sn, time.Now(), false, objectorigin.Import(model.Import_Pb)) + + // then + assert.Nil(t, err) + assert.Equal(t, bundledProjectId, id) + assert.Nil(t, payload.RootRawChange) + }) + + t.Run("type with neither unique key nor name gets a new id", func(t *testing.T) { + // given + _, deriveObject := newFixture(t) + sn := newObjectTypeSnapshot("", "") + + // when + id, payload, err := deriveObject.GetIDAndPayload(context.Background(), "spaceId", sn, time.Now(), false, objectorigin.Import(model.Import_Pb)) + + // then + assert.Nil(t, err) + assert.Equal(t, "freshTypeId", id) + assert.NotNil(t, payload.RootRawChange) + }) +} + +func TestDerivedObject_GetIDAndPayload_ChatDerived(t *testing.T) { + // newChatSnapshot builds the snapshot cmd/anyblockconvert emits for a + // `kind: "chat"` document: the envelope's key on Data.Key, and no + // uniqueKey detail — an authored bundle has no id to carry one. + newChatSnapshot := func(uniqueKey string) *common.Snapshot { + details := domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("Wiki requests"), + }) + if uniqueKey != "" { + details.SetString(bundle.RelationKeyUniqueKey, uniqueKey) + } + return &common.Snapshot{ + Id: "chat-wiki-requests", + Snapshot: &common.SnapshotModel{ + SbType: coresb.SmartBlockTypeChatDerivedObject, + Data: &common.StateSnapshot{Key: "wikiRequests", Details: details}, + }, + } + } + + newFixture := func(t *testing.T) (*objectstore.StoreFixture, *derivedObject) { + sf := objectstore.NewStoreFixture(t) + service := mock_space.NewMockService(t) + space := mock_clientspace.NewMockSpace(t) + service.EXPECT().Get(mock.Anything, "spaceId").Return(space, nil).Maybe() + space.EXPECT().DeriveTreePayload(mock.Anything, mock.Anything).Return(treestorage.TreeStorageCreatePayload{ + RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: "derivedChatId"}, + }, nil).Maybe() + return sf, newDerivedObject(newExistingObject(sf), service, sf) + } + + t.Run("chat is derived from the key its document carries", func(t *testing.T) { + // given + _, deriveObject := newFixture(t) + sn := newChatSnapshot("") + + // when + id, payload, err := deriveObject.GetIDAndPayload(context.Background(), "spaceId", sn, time.Now(), false, objectorigin.Import(model.Import_Pb)) + + // then + assert.Nil(t, err) + assert.Equal(t, "derivedChatId", id) + assert.NotNil(t, payload.RootRawChange) + }) + + // What keeps a reinstalled bundle from growing a second copy of every chat + // is the derivation key, not a store lookup: existingObject resolves a + // uniqueKey only for relations, options and types, so a chat always reaches + // DeriveTreePayload — and the same key derives the same id. Pin the key. + t.Run("derivation is keyed on the chat's own unique key", func(t *testing.T) { + // given + sf := objectstore.NewStoreFixture(t) + service := mock_space.NewMockService(t) + space := mock_clientspace.NewMockSpace(t) + service.EXPECT().Get(mock.Anything, "spaceId").Return(space, nil).Maybe() + + var derivedWith string + space.EXPECT().DeriveTreePayload(mock.Anything, mock.Anything).RunAndReturn( + func(_ context.Context, params payloadcreator.PayloadDerivationParams) (treestorage.TreeStorageCreatePayload, error) { + derivedWith = params.Key.Marshal() + return treestorage.TreeStorageCreatePayload{ + RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: "derivedChatId"}, + }, nil + }).Maybe() + deriveObject := newDerivedObject(newExistingObject(sf), service, sf) + + // when — the document carries only its key, as an authored bundle does + id, _, err := deriveObject.GetIDAndPayload(context.Background(), "spaceId", newChatSnapshot(""), time.Now(), false, objectorigin.Import(model.Import_Pb)) + + // then + assert.Nil(t, err) + assert.Equal(t, "derivedChatId", id) + assert.Equal(t, "chatDerived-wikiRequests", derivedWith) + }) +} + +func TestProvider_ChatDerivedObjectIsImportable(t *testing.T) { + // The regression this guards: with no provider registered for the type, + // GetIDAndPayload fell through to "unsupported smartblock to import" and + // the whole archive failed — one chat document was enough to sink a + // 60-object bundle, and the error named neither the type nor the document. + sf := objectstore.NewStoreFixture(t) + service := mock_space.NewMockService(t) + space := mock_clientspace.NewMockSpace(t) + service.EXPECT().Get(mock.Anything, "spaceId").Return(space, nil).Maybe() + space.EXPECT().DeriveTreePayload(mock.Anything, mock.Anything).Return(treestorage.TreeStorageCreatePayload{ + RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: "derivedChatId"}, + }, nil).Maybe() + + // built through the real constructor: the point of the test is that the + // wiring in NewIDProvider knows the type, so registering one by hand here + // would assert nothing + p := NewIDProvider(sf, service, nil, nil) + + sn := &common.Snapshot{ + Id: "chat-wiki-requests", + Snapshot: &common.SnapshotModel{ + SbType: coresb.SmartBlockTypeChatDerivedObject, + Data: &common.StateSnapshot{ + Key: "wikiRequests", + Details: domain.NewDetails(), + }, + }, + } + + id, _, err := p.GetIDAndPayload(context.Background(), "spaceId", sn, time.Now(), false, objectorigin.Import(model.Import_Pb)) + + assert.Nil(t, err) + assert.Equal(t, "derivedChatId", id) +} diff --git a/core/block/import/common/objectid/existingobject.go b/core/block/import/common/objectid/existingobject.go index d306a1ee31..29698d0b90 100644 --- a/core/block/import/common/objectid/existingobject.go +++ b/core/block/import/common/objectid/existingobject.go @@ -160,30 +160,18 @@ func (e *existingObject) getExistingRelation(snapshot *common.Snapshot, spaceID } func (e *existingObject) getExistingObjectType(snapshot *common.Snapshot, spaceID string) string { - name := snapshot.Snapshot.Data.Details.GetString(bundle.RelationKeyName) - if name == "" { + identity, ok := objectTypeIdentityFilter(snapshot) + if !ok { return "" } - // Search for existing object type by name or unique key records, err := e.objectStore.SpaceIndex(spaceID).QueryRaw(&database.Filters{FilterObj: database.FiltersAnd{ database.FilterEq{ Key: bundle.RelationKeyResolvedLayout, Cond: model.BlockContentDataviewFilter_Equal, Value: domain.Int64(model.ObjectType_objectType), }, - database.FiltersOr{ - database.FilterEq{ - Key: bundle.RelationKeyName, - Cond: model.BlockContentDataviewFilter_Equal, - Value: snapshot.Snapshot.Data.Details.Get(bundle.RelationKeyName), - }, - database.FilterEq{ - Key: bundle.RelationKeyUniqueKey, - Cond: model.BlockContentDataviewFilter_Equal, - Value: snapshot.Snapshot.Data.Details.Get(bundle.RelationKeyUniqueKey), - }, - }, + identity, }}, 1, 0) if err == nil && len(records) > 0 { return records[0].Details.GetString(bundle.RelationKeyId) @@ -191,3 +179,29 @@ func (e *existingObject) getExistingObjectType(snapshot *common.Snapshot, spaceI return "" } + +// objectTypeIdentityFilter tells which existing type the snapshot should be merged into. +// +// A unique key identifies a type exactly, so when the snapshot carries one we match on it alone. +// Matching such a snapshot by name as well would merge two unrelated types that only share a +// title — an imported ot-pmProject named "Project" into the bundled ot-project, for instance — +// and the merge silently discards the imported dataview, description and recommended relations. +// Name is only used for legacy exports, which predate unique keys and offer nothing else to +// match on. +func objectTypeIdentityFilter(snapshot *common.Snapshot) (database.Filter, bool) { + if uniqueKey := snapshot.Snapshot.Data.Details.GetString(bundle.RelationKeyUniqueKey); uniqueKey != "" { + return database.FilterEq{ + Key: bundle.RelationKeyUniqueKey, + Cond: model.BlockContentDataviewFilter_Equal, + Value: domain.String(uniqueKey), + }, true + } + if name := snapshot.Snapshot.Data.Details.GetString(bundle.RelationKeyName); name != "" { + return database.FilterEq{ + Key: bundle.RelationKeyName, + Cond: model.BlockContentDataviewFilter_Equal, + Value: domain.String(name), + }, true + } + return nil, false +} diff --git a/core/block/import/common/objectid/provider.go b/core/block/import/common/objectid/provider.go index ce586da5b4..064fd21f55 100644 --- a/core/block/import/common/objectid/provider.go +++ b/core/block/import/common/objectid/provider.go @@ -68,6 +68,13 @@ func NewIDProvider( p.idProviderBySmartBlockType[sb.SmartBlockTypeProfilePage] = derivedObject p.idProviderBySmartBlockType[sb.SmartBlockTypeTemplate] = treeObject p.idProviderBySmartBlockType[sb.SmartBlockTypeParticipant] = newParticipant() + // a chat is identified by its unique key like a type is, and is created + // with the same DeriveTreeObject the derived path already models — so an + // imported chat lands on the id its key derives, and a second import of + // the same archive resolves to that object instead of a duplicate. Its + // messages live in the any-store CRDT rather than in the snapshot, so what + // imports is the chat itself: name, icon, description, and no history. + p.idProviderBySmartBlockType[sb.SmartBlockTypeChatDerivedObject] = derivedObject return p } diff --git a/core/block/object/objectcreator/relation_option.go b/core/block/object/objectcreator/relation_option.go index 2726f67533..7a59383be8 100644 --- a/core/block/object/objectcreator/relation_option.go +++ b/core/block/object/objectcreator/relation_option.go @@ -48,8 +48,16 @@ func (s *service) createRelationOption(ctx context.Context, space clientspace.Sp } injectApiObjectKey(object, objectKey) + // injectApiObjectKey minted from the unique key on the arm above; an + // option created WITH a unique key whose slug is empty still has a name to + // derive one from, and a name is what an option is addressed by anyway. + // Minted, not merely transliterated: this used to store the raw + // transliteration, spaces and brackets and all, which is not the + // snake_case spelling the api promises a key is. if strings.TrimSpace(object.GetString(bundle.RelationKeyApiObjectKey)) == "" { - object.SetString(bundle.RelationKeyApiObjectKey, transliterate(object.GetString(bundle.RelationKeyName))) + if slug := bundle.MintApiSlugFromName(object.GetString(bundle.RelationKeyName)); slug != "" { + object.SetString(bundle.RelationKeyApiObjectKey, slug) + } } createState := state.NewDocWithUniqueKey("", nil, uniqueKey).(*state.State) diff --git a/core/block/object/objectcreator/util.go b/core/block/object/objectcreator/util.go index 940b3c41fa..df6ff2e012 100644 --- a/core/block/object/objectcreator/util.go +++ b/core/block/object/objectcreator/util.go @@ -7,26 +7,42 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/bundle" coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/globalsign/mgo/bson" - "github.com/gosimple/unidecode" - "github.com/iancoleman/strcase" ) // injectApiObjectKey sets a value for ApiObjectKey relation in priority: // - User-provided ApiObjectKey // - Key from relationKey/uniqueKey -// - Transliterated Name relation +// - Name relation, transliterated +// +// The derived value is MINTED, not merely snake-cased. What is stored here is +// the spelling callers address the object by, and the api promises that +// spelling is snake_case — but snake-casing leaves every character it does not +// understand in place. Measured over a 38,123-object account, 27 of 1,530 +// stored api keys sit outside the advertised grammar `^[a-zA-Z0-9_]+$`: the +// name `Lists [in work]` had minted `lists_[in_work]`, `Manual export & +// import` had minted `manual_export_&_import`, and `➡️ Medium` had minted +// `[?]_medium`, the emoji arriving as unidecode's literal `[?]`. +// +// Nothing is stored when the mint comes back empty — a name of only emoji, a +// key of only punctuation. An object with no derivable slug is addressed by +// its internal key, which is a real address; an empty apiObjectKey is not. func injectApiObjectKey(object *domain.Details, key string) { - if strings.TrimSpace(object.GetString(bundle.RelationKeyApiObjectKey)) == "" { - if key == "" { - key = transliterate(object.GetString(bundle.RelationKeyName)) - } - key = strcase.ToSnake(key) - object.SetString(bundle.RelationKeyApiObjectKey, key) + if strings.TrimSpace(object.GetString(bundle.RelationKeyApiObjectKey)) != "" { + return } -} - -func transliterate(in string) string { - return unidecode.Unidecode(strings.TrimSpace(in)) + var slug string + if key == "" { + slug = bundle.MintApiSlugFromName(object.GetString(bundle.RelationKeyName)) + } else { + // no transliteration on this arm: the key already IS an internal key, + // and the api derives a slug from a stored key with the same + // transform, so transliterating one side would make the two disagree. + slug = bundle.MintApiSlug(key) + } + if slug == "" { + return + } + object.SetString(bundle.RelationKeyApiObjectKey, slug) } func getUniqueKeyOrGenerate(sbType coresb.SmartBlockType, details *domain.Details) (uk domain.UniqueKey, wasGenerated bool, err error) { diff --git a/core/block/object/objectcreator/util_test.go b/core/block/object/objectcreator/util_test.go new file mode 100644 index 0000000000..7d3d67a38e --- /dev/null +++ b/core/block/object/objectcreator/util_test.go @@ -0,0 +1,114 @@ +package objectcreator + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" +) + +// injectApiObjectKey mints the key every type, property and option created +// outside the api is addressed by. The fixtures are the shapes measured in a +// 38,123-object account, where 27 of 1,530 stored api keys sat outside the key +// grammar `^[a-zA-Z0-9_]+$` — all but four of them on options, whose key comes +// from the name and nothing else. +func TestInjectApiObjectKey(t *testing.T) { + t.Run("derived from a name", func(t *testing.T) { + for _, tc := range []struct { + name string + want string + }{ + {name: "Manual property", want: "manual_property"}, + // the three that used to store an unaddressable key + {name: "Lists [in work]", want: "lists_in_work"}, + {name: "Manual export & import", want: "manual_export_import"}, + {name: "➡️ Medium", want: "medium"}, + // transliteration still carries the word across scripts + {name: "Задача", want: "zadacha"}, + } { + t.Run(tc.name, func(t *testing.T) { + // given + object := domain.NewDetails() + object.SetString(bundle.RelationKeyName, tc.name) + + // when + injectApiObjectKey(object, "") + + // then + assert.Equal(t, tc.want, object.GetString(bundle.RelationKeyApiObjectKey)) + }) + } + }) + + t.Run("derived from a name longer than a key may be", func(t *testing.T) { + // given + object := domain.NewDetails() + object.SetString(bundle.RelationKeyName, strings.Repeat("a", 300)) + + // when + injectApiObjectKey(object, "") + + // then + assert.Len(t, object.GetString(bundle.RelationKeyApiObjectKey), bundle.MaxApiSlugLen) + }) + + t.Run("a name with nothing the grammar admits stores no key", func(t *testing.T) { + // an object with no derivable slug is addressed by its internal key, + // which is a real address; an empty apiObjectKey would not be + for _, name := range []string{"➡️", "☕", " "} { + t.Run(name, func(t *testing.T) { + // given + object := domain.NewDetails() + object.SetString(bundle.RelationKeyName, name) + + // when + injectApiObjectKey(object, "") + + // then + assert.False(t, object.Has(bundle.RelationKeyApiObjectKey)) + }) + } + }) + + t.Run("an internal key outranks the name and is not transliterated", func(t *testing.T) { + // given + object := domain.NewDetails() + object.SetString(bundle.RelationKeyName, "Some Name") + + // when + injectApiObjectKey(object, "dueDate") + + // then + assert.Equal(t, "due_date", object.GetString(bundle.RelationKeyApiObjectKey)) + }) + + t.Run("an internal key still loses what the grammar refuses", func(t *testing.T) { + // given + object := domain.NewDetails() + object.SetString(bundle.RelationKeyName, "Some Name") + + // when + injectApiObjectKey(object, "my key!") + + // then + assert.Equal(t, "my_key", object.GetString(bundle.RelationKeyApiObjectKey)) + }) + + t.Run("a key already on the object is never re-minted", func(t *testing.T) { + // the user-provided apiObjectKey is the top of the priority list, and + // re-minting one would silently respell an address already in use + // somewhere + object := domain.NewDetails() + object.SetString(bundle.RelationKeyName, "Some Name") + object.SetString(bundle.RelationKeyApiObjectKey, "chosen_key") + + // when + injectApiObjectKey(object, "dueDate") + + // then + assert.Equal(t, "chosen_key", object.GetString(bundle.RelationKeyApiObjectKey)) + }) +} diff --git a/core/publish/service_test.go b/core/publish/service_test.go index 9670454082..f4cf13b542 100644 --- a/core/publish/service_test.go +++ b/core/publish/service_test.go @@ -848,7 +848,8 @@ func prepareExporter(t *testing.T, objectTypeId string, spaceService *mock_space mockSender := mock_event.NewMockSender(t) a.Register(storeFixture) a.Register(testutil.PrepareMock(context.Background(), a, mockSender)) - a.Register(testutil.PrepareMock(context.Background(), a, objectGetter)) + testutil.PrepareMock(context.Background(), a, objectGetter) + a.Register(cachedObjectGetter{objectGetter}) a.Register(process.New()) a.Register(testutil.PrepareMock(context.Background(), a, spaceService)) a.Register(testutil.PrepareMock(context.Background(), a, mock_typeprovider.NewMockSmartBlockTypeProvider(t))) @@ -863,6 +864,19 @@ func prepareExporter(t *testing.T, objectTypeId string, spaceService *mock_space return exp } +// cachedObjectGetter is what these fixtures register as the app's picker. +// The export service resolves cache.CachedObjectGetter now — its native +// AnyBlock JSON path closes every object it loads out of the cache — and +// the component mock alone does not answer that interface. Publishing +// never takes that path, so the close is a stub that closes nothing. +type cachedObjectGetter struct { + *mock_cache.MockObjectGetterComponent +} + +func (cachedObjectGetter) TryRemoveFromCache(_ context.Context, _ string) (bool, error) { + return false, nil +} + type fileObjectWrapper struct { editorsb.SmartBlock fileobject.FileObject @@ -999,7 +1013,8 @@ func prepareExporterWithFile(t *testing.T, objectTypeId string, spaceService *mo ctx := context.Background() a.Register(storeFixture) a.Register(testutil.PrepareMock(ctx, a, mockSender)) - a.Register(testutil.PrepareMock(ctx, a, objectGetter)) + testutil.PrepareMock(ctx, a, objectGetter) + a.Register(cachedObjectGetter{objectGetter}) a.Register(process.New()) a.Register(testutil.PrepareMock(ctx, a, spaceService)) a.Register(testutil.PrepareMock(ctx, a, mock_typeprovider.NewMockSmartBlockTypeProvider(t))) diff --git a/docs/Flow.md b/docs/Flow.md index b6532a5647..b60d41870d 100644 --- a/docs/Flow.md +++ b/docs/Flow.md @@ -70,6 +70,23 @@ Anytype will update system objects only if `Revision` of object from marketplace 1. Update description of system object, that is stored in `pkg/lib/bundle` 2. Increase `revision` field of system type/relation or put `"revision":1` if it was empty 3. Generate go-level variables for new version of types and relations using `pkg/lib/bundle/generator` -4. Make sure that new fields are taken into account in [System Object Reviser](../core/block/object/objectcreator/systemobjectreviser.go). - (Right now only these fields are checked: **Revision**, **Name**, **Description**, **IsHidden**, **IsReadonly**) -5. Build and run Anytype. All system objects with lower `Revision` should be updated according your changes in all spaces \ No newline at end of file +4. Make sure that new fields are taken into account in [System Object Reviser](../space/internal/components/migration/systemobjectreviser/systemobjectreviser.go). + (Only the fields listed in `systemObjectFilterKeys` are checked, e.g. **Revision**, **Name**, **IsHidden**, **IsReadonly**, **PluralName**) +5. Build and run Anytype. All system objects with lower `Revision` should be updated according your changes in all spaces + +### How to rename bundled non-system relations + +Bundled relations that are NOT system ones (e.g. **AudioGenre**) can be renamed in `relations.json` too, +but users are allowed to rename such relations in their spaces, so the reviser must not overwrite a user's own name. +The reviser therefore applies a bundled rename only if the installed relation still carries a previous bundled name. + +1. Change the `name` of the relation in `pkg/lib/bundle/relations.json` +2. Increase its `revision` field or put `"revision":1` if it was empty — without this the rename never reaches existing spaces +3. Append the OLD name to `previousBundledRelationNames` in the + [System Object Reviser](../space/internal/components/migration/systemobjectreviser/systemobjectreviser.go) package +4. Regenerate go-level variables using `pkg/lib/bundle/generator` + +Only **Revision** and **Name** are revised on non-system relations, and the name is applied only when the local name +still equals one of the previous bundled names; any other local name is treated as the user's rename and kept. +Renamed system types and relations do not need `previousBundledRelationNames` entries: users cannot rename them, +so the system path applies the bundled name unconditionally (a `revision` bump is still required). \ No newline at end of file diff --git a/docs/proto.md b/docs/proto.md index 1ed41e7fdf..10a6dd6320 100644 --- a/docs/proto.md +++ b/docs/proto.md @@ -36750,6 +36750,7 @@ stored | | DOT | 3 | | | SVG | 4 | | | GRAPH_JSON | 5 | | +| AnyBlockJSON | 6 | AnyBlockJSON is the native AnyBlock JSON bundle (pkg/lib/anyblockjson SPEC.md): a directory of `<id>.anyblock.json` documents beside an index.json and properties.json. Additive — existing values keep their numbers, so a client that does not know it is unaffected. | diff --git a/go.mod b/go.mod index 49fffd5fa6..e18455f2d0 100644 --- a/go.mod +++ b/go.mod @@ -88,6 +88,7 @@ require ( github.com/pseudomuto/protoc-gen-doc v1.5.1 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/samber/lo v1.49.1 + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/sasha-s/go-deadlock v0.3.5 github.com/shirou/gopsutil/v4 v4.26.2 github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c diff --git a/go.sum b/go.sum index b7c92955c4..62121d982a 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/didip/tollbooth/v8 v8.0.1 h1:VAAapTo1t4Bn6bbpcHjuovwoa9u3JH++wgjbpWv+ github.com/didip/tollbooth/v8 v8.0.1/go.mod h1:oEd9l+ep373d7DmvKLc0a5gasPOev2mTewi6KPQBGJ4= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dsoprea/go-exif/v2 v2.0.0-20200321225314-640175a69fe4/go.mod h1:Lm2lMM2zx8p4a34ZemkaUV95AnMl4ZvLbCUbwOvLC2E= github.com/dsoprea/go-exif/v3 v3.0.0-20200717053412-08f1b6708903/go.mod h1:0nsO1ce0mh5czxGeLo4+OCZ/C6Eo6ZlMWsz7rH/Gxv8= github.com/dsoprea/go-exif/v3 v3.0.0-20210428042052-dca55bf8ca15/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk= @@ -827,6 +829,8 @@ github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9t github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU= github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= diff --git a/pkg/lib/anyblockjson/ANOMALIES.md b/pkg/lib/anyblockjson/ANOMALIES.md new file mode 100644 index 0000000000..92692c4a63 --- /dev/null +++ b/pkg/lib/anyblockjson/ANOMALIES.md @@ -0,0 +1,347 @@ +# Real-world data anomalies + +Findings from round-trip testing `pkg/lib/anyblockjson` against a production +account (`cmd/anyblockroundtrip`): four iterations on 2026-07-23 over +~35 400 objects across ~48 spaces, and a later sweep over a 36 808-object +account during the v0.9–v0.11 work. Which figure belongs to which run, and +what a pass rate measures, is the ledger at the end of this file. Each entry +records a shape that real snapshots contain but the clean data model does +not predict, and how the format handles it. Kept separate from SPEC.md: the +spec defines the format, this file explains *why* several of its rules exist +and what future importers and migrations must expect in old accounts. + +Legend: **handling** = what export/import does today; **spec** = where the +rule lives. + +## 1. Content-less blocks (unset content oneof) + +Blocks with `Content == nil` exist in two shapes: + +- every legacy `STRelation` / `STRelationOption` object wraps its "used in" + dataview in a bare content-less block (id `rel-` or a 24-hex legacy + id); +- ordinary pages contain orphaned content-less leaves with no children. + +Volume: 277 objects failed on this before the fix (~67% of all failures in +run 2); 87 distinct block ids. + +**Handling**: the block is dropped either way — it is a transparent +container (§7a) — and a subtree under one (e.g. the relation's dataview) is +lifted into its place. It was once written out as a `group` block, and +on 160 of these objects that wrapper was what kept §7's primary-dataview pin +from firing. **Spec**: §7 "Content-less blocks", §7a. + +## 2. File blocks with real block children + +The editor treats file blocks as leaves, but legacy data nests genuine text +blocks under them — observed: a table cell with File content holding four +paragraph children (real user text, including a mention). Dropping them was +*silent* data loss. + +Volume: 1 object / 4 blocks — rare but the worst class of bug. + +**Handling**: descendants of file blocks are allowed and round-trip verbatim +(in the flat encoding: blocks indented under the file block; file types are +not in the leaf list). Because the observed case lives in a *table cell*, +this anomaly is also the standing evidence for the cell **array form** — +a cell block with descendants serializes as an array of flat blocks (§6.1) +rather than dropping them. **Spec**: §5 file row, §6.1. + +## 3. Recommended-relation lists holding bare property keys + +Type objects predating per-space derived relation ids store bare property +**keys** (`"creator"`, `"createdDate"`) in `recommended_hidden_relations` and +friends, where object ids are expected. + +Volume: ~15 objects. + +**Handling**: export resolves list entries with a fallback chain — property +id → reverse key lookup → bundle (system properties). On import the entry +comes back as a proper object id, i.e. round-tripping *migrates* the legacy +key to an id. **Spec**: §2a. + +## 4. Orphaned pre-CID relation ids in recommended lists + +The same lists also hold Mongo-ObjectId-style 24-hex ids +(e.g. `68cdaa41e9223c9dc7ce5f30`) that resolve to nothing in the space — +relics of deleted relations or pre-migration data. Some repeat verbatim +across 38–46 different objects (template cloning preserves source block and +detail ids). + +Volume: ~14 objects, concentrated in shared/community spaces. + +**Handling**: dropped on export (unresolvable), by design. Round-tripping +cleans them up. **Spec**: §2a. + +## 5. `_missing_object` sentinels in object-reference details + +Dangling object references are stored as the literal sentinel +`"_missing_object"` (`pkg/lib/localstore/addr`) inside detail lists. + +Volume: ~60 objects. + +**Handling**: dropped on export like any unresolvable reference. Test +tooling must not count their disappearance as loss. **Spec**: §2a; harness: +`cmd/anyblockroundtrip` filters the sentinel in comparisons. + +## 6. Duplicate-named select/multiSelect options + +Spaces contain multiple option objects with the same display name for one +property. Because option values round-trip by **name** (§3), import resolves +to one canonical option — objects referencing the duplicate get their option +id swapped. + +Volume: 7 objects (`tag` property). **Closed on the default shape, accepted +on the id-less one.** The `option_ids` legend (§9a) carries the id beside the +name and is written unconditionally, so a default export read back into a +space that still serves the id lands on the option the object was actually +on. Two readings still resolve by name and still swap: `OmitIds`, which drops +the legend because an id-less document shipping a map of ids is not one (§9), +and any read into a space that does not serve those ids — where the legend is +a hint that fails its liveness check and falls through by design (§3). §15.3 +(names vs `{id, name}` value objects) is settled on the strength of that: the +id rides beside the name rather than inside the value. + +## 7. Default-valued details are semantically present + +Details like `is_hidden: false`, `revision: 0`, `property_format_include_time: +false` (legacy spelling) appear *explicitly* on thousands of objects. Presence of a property +key — even with a default/empty value — records that the property was set on +the object; clients rely on it. + +Volume: run 1 flagged 14 032 issue lines on 5 577 objects before the design +call. + +**Handling**: `properties` values are written verbatim, including `false`, +`0`, `""`, `[]`, and explicit `null`; the omit-empty canon applies only to +block attributes and envelope fields. **Spec**: §3 "Presence is +meaningful". + +## 8. Empty-vs-absent recommended lists + +Type objects store the four `recommended*Relations` keys as explicit empty +lists; hand-authored documents may omit them entirely. The two must not be +conflated: run 3 caught a type with *all four* lists empty round-tripping to +absent keys. + +**Handling**: type documents always carry `type_properties`, even as `[]` — +presence of the array is the trigger to rebuild all four lists (empty +sections become explicit empty lists); a document without the field leaves +the lists untouched. **Spec**: §2a. + +## 9. Point lookups and listings disagree, in both directions + +The spaceindex relation lookups are mutually inconsistent: + +- `GetRelationByKey` misses some relations that `ListAllRelations` still + returns (observed for custom keys like `artist`, `assignee` in old + spaces). +- The inverse (run 4): `GetRelationById` resolves a relation that is absent + from **both** `ListAllRelations` and `GetRelationByKey` — observed for + ordinary user-created relations (bson-id keys, e.g. a "Tag" property), + presumably deleted or partially indexed. A resolver that answers by id + but cannot invert the resulting key breaks the round trip: import passes + the key through and re-export drops the entry. + +Any resolver wired over the object store should prime id↔key maps from the +full listing, use point lookups as fallback, and **cache every point-lookup +hit in both directions** so resolution stays invertible — +`cmd/anyblockroundtrip` does both. Root cause in the index itself is +unverified; worth an issue. + +## 10. Template-cloned block ids repeat across objects + +The same 24-hex block ids recur verbatim in dozens of distinct objects +(template instantiation preserved source ids; `fields.analyticsOriginalId` +often matches). Harmless for this format — ids are document-scoped — but any +tooling assuming account-wide block-id uniqueness will be wrong. + +## 11. Flat encoding: the sweep verifications (v0.6) + +The v0.6 flat-blocks change carried assumptions the flat-encoding sweep +(run 4, 2026-07-23, 35 372 objects) checked; measured results: + +- **Cells with descendants**: counter reported **0** across the account — + the §6.1 array form is implemented losslessly anyway (and #2's historical + case proves the class exists), but it is effectively absent from current + data. +- **Tables nested inside table cells**: none — zero failures, so the + non-recursive `cellBlock` restriction (§12) matches reality. Should such + data ever appear, Marshal now rejects it loudly (`export_error` in the + sweep) instead of emitting a document its own validation rejects; the + depth bound (32) is enforced at export the same way. +- **Depth histogram** (per-object max indent): 0 → 34 057, 1 → 816, + 2 → 323, 3 → 119, 4 → 37, 5 → 5, 6 → 6, 7–8 → 2, and a long-tail of + 7 outliers at 16–26. The "~6 typical max" datum holds for the bulk; the + outliers stay comfortably under the 32 bound (`indent` > 32 is a + validation error). +- **Run 4 result**: 21 failures = 7 accepted duplicate-name option swaps + (#6) + 14 tool-resolver asymmetry (a relation resolvable by id via point + lookup but absent from both the listing and the by-key lookup — the #9 + class; export emitted its key, import could not invert it, re-export + dropped the entry). Fixed in `cmd/anyblockroundtrip` by caching point- + lookup hits in both directions; format and package unaffected. + +## 12. Charset-dirty block ids are no longer laundered by relabeling + +The schema's block-id pattern is `^[A-Za-z0-9_-]{1,64}$`, but stored ids +are not guaranteed to match it (legacy/imported data could carry other +characters; no live producer found). The earlier charset relabel rule +*accidentally laundered* such an id whenever its 5-char tail was clean — +the served document carried the clean label and validated. Under the +minted-shape rule (`isMintedLocalId`, API v2 Wave 0 hardening) a +non-minted id serves verbatim, so a document holding a charset-dirty +block id now fails its own `Validate` on export. **Handling**: closed, not accepted — +the minted-shape rule decides *whether* an id is compacted, and the export +sanitizer decides how whatever comes out is spelled, so a charset-dirty id +is written as `a_b` rather than verbatim and `table1` keeps its name. Both +rules landed on the same branch and compose; `TestExport_BlockIdOutsideCharsetIsSanitized` +pins the first half and the compact golden pins the second. No such id +appeared in the 2026-07-23 sweeps (~35 400 objects) or the later +36 808-object one. **Spec**: §9a (relabel rule), schema `$defs/blockId`. + +## 13. A quarter of all image covers are absolute filesystem paths + +`coverId` is declared `longtext` with no constraint, so nothing in the store +or in the format ever checked that a cover marked `coverType: 1` (an image) +names an image object. `core/block/import/notion/api/commonobjects.go` sets +`coverId` to `cover.External.URL` with `coverType: 1`, expecting a later +pass to download the file and rewrite the reference. On **33 objects** of a +36,966-object account that pass never ran, leaving values like: + +``` +/var/folders/j0/b3km_psx1bd14q06gdpvzk5m0000gn/T/anytype_notion_import/f99972cc….png +``` + +The temp directory is long gone. That is 33 of the 130 `coverType: 1` +covers — **25% of every image cover in the account** — permanently corrupt, +and nothing reported it, because a `longtext` holding a path is a perfectly +good `longtext`. The same importer writes URLs into `iconImage` by the same +mechanism; no leaked icon survived in this account (0 of 12,011). + +**Handling**: refused rather than carried. §2b's `cover.file` is an object +reference (`^[^/]+$`), so the value cannot be written — and carrying it +would make `Marshal` emit what its own `Validate` rejects (I1). Export drops +the cover with a warning naming the value, which turns permanent silent +corruption into a named event, and `snapshotdiff` still reports it as loss: +66 findings over those 33 objects, the only loss the icon/cover collapse +causes. **Spec**: §2b, §11 `N(S)` clause (e). The importer bug is a separate +ticket — this file records the data, not the fix. + +Two smaller ones from the same census, both handled rather than open: +`iconOption` holds 12, 13 and 15 on six objects, because +`core/block/import/pb/converter.go` mints `rand.Intn(16)+1` while +`core/block/import/markdown/schema.go` mints `rand.Intn(10)+1` for the same +ten-colour palette — §2b's integer colour escape carries them. And 54 +objects hold the bundled `iconEmoji` (empty) beside a **space-minted +relation whose own stored key is literally `icon_emoji`**, holding a real +emoji; anything reading "the icon" out of `icon_emoji` in those documents +reads user data. §2b's lift separates the two visibly. + +--- + +## The compact goldens no longer differ from the plain ones + +**Status: open, recorded rather than fixed — and now measured.** With +object-ref compaction deleted, `CompactIds` selects only block-label +relabeling, and the rich fixture's block ids are all short or hand-authored +(`b1`, `dv1`, `v1`, `table1`), none of them minted-shaped, so none relabels. +Two of the four goldens therefore freeze nothing the other two do not. + +**Measured**, not inferred, at `35ec288e6` — `cmp` plus `shasum -a 256` over +`pkg/lib/anyblockjson/testdata`: + +| golden | bytes | sha256 (first 16) | +|---|---|---| +| `rich.json` | 4694 | `3ed93ddf8025c87b` | +| `rich_compact_ids.json` | 4694 | `3ed93ddf8025c87b` | +| `rich_omit_ids.json` | 3669 | `ce96eee96aea3d4b` | +| `rich_compact_omit.json` | 3669 | `ce96eee96aea3d4b` | + +Both pairs are byte-identical — re-verified after the raw-name +regeneration (`cmp`: 4,846 bytes and 3,821 bytes per pair). While that +holds — i.e. while no id in the +rich fixture is minted-shaped — a change to the relabel rule *alone* produces +**zero** golden drift, and zero drift is exactly what reads as "the goldens +saw it and it was fine". That is the misleading part, not the duplication. + +The path is not uncovered — `TestExport_MintedShapeRelabeling` and +`compactsplit_test.go` both pin block relabeling against minted ids, and +`TestExport_CompactIdsIsAnAliasForBlockLabels` pins the alias against a +fixture that does relabel. So this is a redundancy in the golden set, not a +hole in the coverage. + +**Recommendation (not performed — goldens are load-bearing here, and this is +a maintainer's call).** If it is closed, close it with a *new, small* pair — +a two-block fixture carrying one minted-shaped id and one meaningful id, with +its own plain and `CompactIds` goldens — rather than by minting an id into +`rich`. Minting into `rich` moves block ids in all four existing goldens for a +property that has nothing to do with what `rich` is for (a frozen picture of a +hand-authored document exercising every block type), and it produces a golden +diff that has to be justified line by line for lines that carry no meaning. A +dedicated pair differs by exactly the one thing the flag does, which is what a +golden is worth freezing. Leaving it as it stands is also defensible: the +named tests above carry the rule, and this entry is the note that keeps the +zero-drift signal from being read as coverage. **Spec**: §9a. + +--- + +## No golden carried a `property_internal_keys` legend + +**Status: closed by the exhaustive legend.** The four goldens used to hold +bundled or verbatim keys only, so the §3 legend never appeared in a frozen +document and the goldens proved nothing about it — including nothing about +its canonical position relative to `option_ids`, which is keyed by the +spellings `property_internal_keys` inverts. Since the legend became +exhaustive — one entry for every spelling the bundled table does not bind +— the rich fixture's two custom keys earn their entries, all four goldens +now freeze a two-entry `property_internal_keys`, and the two id-bearing +goldens freeze its canonical position before `option_ids` (see +`testdata/rich.json`). `TestOptionRefs_TheLegendFollowsPropertyKeys` still +pins the ordering independently of the fixtures. **Spec**: §2, §4. + +--- + +## Run ledger + +Four object counts circulate in this file and in the package, because the +account grew between sweeps. Each belongs to one run, and none of them is a +statement about the others. + +What a pass rate measures here: `snapshotdiff` compares detail values (up to +the documented normalizations) and the plain text of text blocks as a +multiset, and the harness compares the re-exported bytes with the exported +ones. Marks, block order, table shape, dataview content and file/bookmark +metadata are not compared, so a systematic loss in any of them is +byte-stable and invisible to the number — a blind spot the pre-freeze +review first named, and the comparator now says itself: its findings are +triage input, not proof. + +- **Runs 1–3** (2026-07-23, ~35 400 objects across ~48 spaces, pre-flat). + Run 1 flagged 14 032 issue lines on 5 577 objects, all default-valued + details (#7); run 2 failed 277 objects on content-less blocks, ~67% of its + failures (#1). Final state after those fixes: 35 369 objects, 0 + export/import errors, 99.86% byte-identical, the remainder the accepted + duplicate-name option swaps (#6). That 99.86% is a pre-flat number and + belongs to run 3 alone. +- **Run 4** (2026-07-23, 35 372 objects) is the v0.6 flat-encoding sweep; + its results are §11 — 21 failures = 7 accepted swaps (#6) + 14 from a + resolver asymmetry in the harness (#9), fixed there. This file used to + close by calling that rerun pending — a line added by the same commit that + recorded §11's measured results, so it was stale on arrival. The run + happened: 21 failures in 35 372 objects is 99.94%, above the ≥ 99.86% bar + that line set, with neither failure category new. +- **The 36 808-object sweep** (August 2026, during the v0.9–v0.11 work) has + **no pass rate recorded anywhere** — only the findings it produced: 59 + objects failing their own export on the envelope `key` charset (SPEC §2's + deny rule; `TestValidate_EnvelopeKeyAcceptsRealStoredKeys` carries the + real keys), 12 objects whose dataview came back pointing at another + property (SPEC §3's `property_internal_keys` legend; `storeresolver/keyvocab.go`), + two date-filter documents export emitted and validation rejected + (`datefilter_test.go`), and 10 378 false data-loss issues caused by the + harness's own stale copy of the internal-property list (now + `InternalPropertyKeys`, `export.go`). +- **Nothing since has been measured against real data.** Every v0.9–v0.11 + fix is demonstrated by unit test, not by a sweep — the 12 re-pointed + objects have not been swept again since the guard landed — and no sweep is + recorded against the package as it stands after them. diff --git a/pkg/lib/anyblockjson/EXPORTER_DESIGN.md b/pkg/lib/anyblockjson/EXPORTER_DESIGN.md new file mode 100644 index 0000000000..85b0764d9e --- /dev/null +++ b/pkg/lib/anyblockjson/EXPORTER_DESIGN.md @@ -0,0 +1,986 @@ +# The native AnyBlock JSON exporter — design + +Status: IMPLEMENTED (GO-7383, 2026-08-26). §1's pipeline is live: +collection behind `core/block/export/collect` (Closure replacing +`isProtobuf`), composition in `pkg/lib/anyblockjson/compose` (Q10 option b; +the roundtrip harness runs the same code), the exporter wiring in +`core/block/export/anyblock`, and the manifest `files` map as SPEC §2c +(Q4 option a). Q5 taken as (a), Q8 settled to `.anyblock.json`, Q9 as (a) +via the per-bundle `BundleRoot` prefix. Q11 shipped as close-after-write +with the release gate named at the call site (any-sync PR #769, the +GO-7333 fix, must land first — anyblock.go). Q6 landed as its +recommendation (a): `Export_AnyBlockJSON = 6` on `model.Export.Format`, +routed by `closureForFormat`/`exportByFormat` over the doc set the export +service has already collected, with emit running as the export queue's own +tasks. Still open, deliberately: **Q7** (default-backup / pb retirement, a +product call). One deviation from §1.1: the manifest +type-path table accumulates at EMIT from actually-written documents rather +than being pre-built at plan — provably consistent with the output (a doc +whose emit fails never enters the manifest), and determinism is unaffected +since finish sorts. + +Verified against the corpus by `cmd/anyblockroundtrip -native`, which +drives THIS exporter (not the pb path) over every space and checks layout, +kind classification, filename purity, blob binding, omission accounting, +determinism (every space exported twice, trees byte-compared) and +per-document fidelity against a same-process pb export. First run over +28,542 real documents: layout/classification/naming clean, data loss +byte-for-byte equal to the pb baseline (34 objects / 67 findings, all +codec-level). Real defects caught by real data and review, all fixed: +the participant filename fold (§1.3 demanded the ENVELOPE id); a +non-total option-vocabulary sort (same-name options tied into scheduling +order); the manifest `files` map having validators but no READER +(cmd/anyblockconvert now binds each blob into the archive and writes the +archive-side `source` from the map — the pb importer's own contract); +the §15 #1 skip rule existing only as text (discovery now excludes what +each manifest binds); and a mid-stream blob failure leaving a truncated +file on disk (the writer's cleanup hook removes it; failures are counted +in Export's Result, and a file document unbound by a present map is a +tooling warning). One upstream observation, not an exporter defect: +objects whose root change carries no creation date (participants, +chiefly) get `createdDate` stamped at load (smartblock Apply), so any two +exports separated by a cache eviction differ on that value — the pb +exporter shares this. RESOLVED by the human as a format rule: +participant documents omit `created_date` (`participantProvenanceKeys`, +the type-provenance pattern; snapshotdiff taught in the same change). + +RECORDED OPINION, not acted on (the human's call leans toward leaving +them): after that omission a participant's remaining provenance — +`creator` and `last_modified_by` — reads `_anytype_profile` on 2,492 of +2,492 corpus participants, a 100% placeholder. By the format's own §15 +#12 discipline I would omit those too: "one distinct value in all real +data is the definition of saying nothing" is the exact verdict that +admitted the `fileIndexingStatus` drop, and a value that is always a +placeholder additionally trains consumers to treat `_anytype_profile` as +an identity. The counter-position (the human's) is also real: keeping the +field leaves upstream's bug visible instead of papered over, and is +self-healing — the day heart populates a participant's creator with the +real identity, documents start carrying it with no format change. The +cost asymmetry favours reversibility either way: un-omitting later is one +kind-scoped map entry and needs no version bump (documents never carried +the key while omitted, so nothing existing changes meaning), while +keeping the placeholder costs two fields × 2,492 documents of false claim +per export until upstream fixes it. Leaving them, as decided, is cheap to +reverse; so would omitting have been. + +Scope: the production exporter that writes an AnyBlock JSON bundle (SPEC.md +§2c) from a live space — the replacement for the writing half of +`core/block/export`, sitting on the extracted collection half. The +architecture decision (keep collection, replace writing, target shape +`core/block/export/{collect,writer,anyblock}`) is taken and not re-argued +here; this document decides layout, naming, blob handling, the seam, and +concurrency. + +Evidence base: the code cited by `file:line` throughout, and a 77-space +production corpus sweep (38,105 source objects, 28,542 emitted documents) +measured with Python for this document. Corpus numbers below are from that +sweep unless said otherwise. + +--- + +## 1. Proposed design + +### 1.1 The pipeline: collect → plan → emit → finish + +Four phases, two of them new relative to today's exporter: + +| phase | threading | does | +|---|---|---| +| **collect** | as today | dependency closure over the request: nested objects, dataview-referenced objects, types, relations, options, templates, linked files, recommended relations (`processProtobuf`, export.go:610). Extracted behind a format-agnostic interface; the bare `isProtobuf bool` (export.go:503-504) becomes an explicit closure mode. Output: `map[id]*Doc`, complete before anything is written. | +| **plan** | single-threaded | classify every collected doc (kind → directory, §1.2), compute every filename (§1.3 — a pure per-document function of the id, no collision machinery), and pre-build the manifest type-path table (stored type keys come from the `uniqueKey` detail). **Plan reads details only — id, name, type/layout, uniqueKey — never content**; that invariant is what keeps it O(collected details) in memory and free of object loads (§1.6). Cheap: map passes over details already in memory, no store reads, no marshal. | +| **emit** | width-bounded concurrent queue tasks (§1.5; the queue is already width-4 today, export.go:152-156) | per document: load state, run the omission predicates on the loaded snapshot (`OmittedBundledRelation`, `OmittedSpaceSettings`, `OmittedWidgetObject`, `OmittedProfilePage` — omittedrelation.go:151, spacesettings.go:156, widgetobject.go:359, profilepage.go:40 — they take the snapshot base, so they CANNOT run at plan time), lift-or-`anyblockjson.Marshal`, write to the planned filename, close (§1.5); for file objects, stream the blob (§1.4). Accumulates bundle facts (installed keys, dictionary entries, option vocabularies, index lift, used property keys) into a mutex-guarded composer. A name planned for a document emit then omits simply goes unused — determinism is unaffected, since omission is itself a deterministic function of state. | +| **finish** | single-threaded, at the `postProcess` seam (export.go:1529) | compose and write `properties.json` and `index.json` (with manifest), re-reading both through the package's own `Unmarshal` before writing — the bundle-level I1 discipline the harness already practices (cmd/anyblockroundtrip/main.go:983-1012). | + +The composer is a production re-home of the harness's `spaceComposer` +(cmd/anyblockroundtrip/main.go:711-1030), which already implements the §2f +composition end to end: installed-key census, divergent-entry override, +option vocabulary with `orderId` ordering, index lift from the omitted +space-settings and widget documents, manifest, and the re-read check. What +moves is the code's home and its input source (in-memory states instead of +`.pb` files on disk), not its logic. + +One piece cannot move as-is: `anyblockbatch.UsedPropertyKeys` reads written +files back from disk (cmd/internal/anyblockbatch/scan.go:908). A zip export +cannot re-read its own entries (zipWriter has no read path, writer.go:130), +so the used-key scan must run on the marshalled bytes **before** they are +written. The scan logic should be promoted from `cmd/internal/anyblockbatch` +into a place production code may import (a byte-level +`UsedPropertyKeysFromBytes` in `pkg/lib/anyblockjson` or a small exported +subpackage), keeping the cmd tools on the same single implementation. + +### 1.2 Directory layout (question a) — SETTLED: kind directories, `objects/` flat + +The importer never dispatches on directory names — its only path rule is +skipping `files/` (import/pb/converter.go:38, 338-341); classification is by +the document's own declared kind/type (import/pb/converter.go:337 onward), +and SPEC.md:2319 states outright that "the format defines no folder layout — +`objects/`, `types/`, `relations/` are one exporter's convention". So the +layout is chosen for the human opening the bundle, and for consistency with +the format's own vocabulary — which spells everything it defines +snake_case and never says "relation" (SPEC §1 Naming; PRINCIPLES rule 3 — +the word survives only in recorded stored keys and user-given names, +neither of which a directory name is). + +Proposed layout, one bundle root per space: + +``` +/ + index.json — the bundle index + manifest (SPEC §2c; index.go:30) + properties.json — the property dictionary (SPEC §2f; dictionary.go:47) + objects/ — kind: page (and any kind without a dedicated home, + e.g. the rare fail-closed widget document — 1 in + 28,542 corpus docs). FLAT — no type subdirectories + (settled; type grouping belongs to the later + human-readable mode, §1.3) + types/ — kind: object_type + templates/ — kind: template + properties/ — kind: property — only the KEPT documents (divergent + installed copies and space-minted properties; the + rest are omitted into the dictionary per §2f) + options/ — kind: property_option + participants/ — kind: participant + files/ — kind: file_object documents AND their blobs, + adjacent (§1.4) +``` + +Rationale, against the legacy names (export.go:96-103): + +- **Format vocabulary, not store vocabulary.** `relations` → + `properties/`, `relationsOptions` → `options/`, matching the kinds the + documents themselves declare (`kind: "property"`, `"property_option"`). + The format's own vocabulary never says "relation" (PRINCIPLES rule 3); + the directory a reader sees first should keep that rule too. +- **snake_case / single words.** `filesObjects` and `relationsOptions` are + camelCase compounds in an archive whose every document member is + snake_case. All proposed names are single lowercase words, sidestepping + the case question entirely. +- **`participants/` is new.** 2,492 participant documents are 8.7% of the + corpus and today land in `objects/`, where they bury real pages (the + median space has 78 documents total). They are machine-derived membership + records; giving them their own room keeps `objects/` browsable. +- **`files/` holds both halves of a file** — see §1.4. This deletes the + legacy `filesObjects`/`files` split, which forced a human to correlate + two directories by id. +- **Kind counts justify the split**: file_object 10,254 · page 9,688 · + property_option 2,641 · participant 2,492 · object_type 1,760 · property + 1,215 · template 491 across the corpus. Every proposed directory earns + its place in a real account; none is speculative. +- **No `profile` file.** The raw-protobuf `profile` is an install artifact + of the `ObjectImportExperience` path and is written by `cmd/anyblockconvert` + when preparing an installable experience (SPEC §2c "How it reaches the + space"); a native backup bundle carries the same facts in `index.json`. + The legacy exporter's `createProfileFile` (export.go:1316) does not carry + over. +- **No `index.pb`-style home special case.** Legacy writes the home object + as `index` at the root (export.go:1267-1268) — which for a JSON + format collides head-on with `index.json`. The native bundle records the + homepage in `index.json` (`homepage`, SPEC §2c) and the home object is an + ordinary document under `objects/`. + +Multi-space export keeps the `spaces//` wrapper (export.go:96, +1381-1385), each space directory being a complete self-contained bundle root +with its own `index.json` and `properties.json`. The wrapper is also what +keeps id filenames collision-free across spaces: the same id legitimately +recurs in several bundles — 448 cross-space repeats measured in the +corpus, chiefly participant identities exported into every space the +member belongs to — and each lands in its own bundle root. A reader who +flattens a multi-space export into one directory WILL hit real filename +collisions; the per-space root is load-bearing, not cosmetic. + +**The kind-split tension, resolved.** With id filenames (§1.3), id→path is +a pure function only WITHIN a directory; a reference does not say which +kind its target is, so resolving an arbitrary id against this layout is a +probe over the seven kind directories. Position taken: **the bounded probe +is acceptable, and the rule is stated plainly** — "a document is +`/.anyblock.json` for exactly one of the seven directories; to +resolve an id, check them in order". The probe constant is 7, fixed by this +design, independent of space size. In practice it is not even a probe: a +zip reader holds the archive's entire central directory in memory, so +resolving any path is one map hit regardless of folders; on a filesystem it +is at most 7 stats, and any reader resolving many references builds a full +id→path map in one walk (7 readdirs) — which it must be able to do ANYWAY, +because the layout is one exporter's convention (SPEC.md:2319) and an +authored bundle may put its documents anywhere, so a general reader walks +and indexes regardless (`DiscoverJSONFiles`, +cmd/internal/anyblockbatch/scan.go:266). What the kind split still buys +once filenames are opaque: `files/` blobs separated from documents, kind +tallies visible at a glance when debugging an export, and the enumeration +of each kind without opening anything. The genuinely-flat alternative +(every document in one directory — a strictly purer id→path function) is +recorded in §2; it remains cheap to adopt later precisely because nothing +dispatches on directories. + +Authored bundles (consumer 2, SPEC.md:1543) are unconstrained by all of +this: a hand-written bundle may put documents anywhere, because nothing +resolves by path except through the manifest the author writes. The layout +above is what OUR exporter emits, recorded in SPEC §2c's "one exporter's +convention" slot. + +### 1.3 File naming (question b) — SETTLED: `.anyblock.json` + +**The rule.** Every document is named by its envelope id, verbatim: + +``` +objects/bafyreickzryfg6w3srlo3tlirqkftg7rhhgaxzpnjmuwv5kg7hjebd2j3u.anyblock.json +types/bafyreihayoh64xvkp2rdr34eudnwoht36d5cdmii465v5y7haojvdyu534.anyblock.json +files/bafyreigp3himcyqenxemyyk3iu7qtnmlglu4qgnn3r63b743ytzyp6hpv4.anyblock.json +participants/AAjEaEwPF4nkEh9AWkqEnzcQ8HziBB4ETjiTpvRCQvWnSMDZ.anyblock.json +``` + +This is what the harness already writes (cmd/anyblockroundtrip/main.go:377). + +**The argument that settled it** (human decision, 2026-08-26; a hybrid +`--` scheme was the standing proposal and was overturned): a +reference inside a document carries an **id and nothing else** — a link +block's `object_id`, a mention target, an option id in `option_ids`, every +manifest key. The format addresses everything by id, on every shape (SPEC +§9a: "object references are never compacted"). With any name-bearing +filename there is **no way to get from a reference to its file except +scanning documents**; with `.anyblock.json` the mapping is a pure +function of the reference itself. The human's framing: the format is +already transparent — "we have ids everywhere" — and the bundle's main +consumer here is machine reading with clear rules; an authoring agent +minting a use case can even choose ids that ARE its filenames. Legibility +of the LISTING is deliberately traded away in this mode and comes back +whole in a later mode (below). + +**Why ids are safe, measured.** Corpus ids are exactly two populations: +26,050 ids of 59 chars (lowercase-base32 CIDs) and 2,492 of 48 chars +(base58 participant identities). Their combined character set is +`1-9 A-H J-N P-Z a-k m-z` — no `0`, `I`, `O`, `l`, no path-hostile +characters, no Unicode, no normalization surface, no Windows reserved +stems, no length hazard (59 + 14 = 73 bytes per component maximum, under +the 255-byte limit; worst full path with `spaces/<59-char id>/objects/` +prefixes ≈ 150 chars, under Windows' 260 default). Uniqueness is by +construction (ids are unique per space; measured: zero duplicates within +any of the 77 bundles). Case-insensitive filesystems are covered by two +different arguments, one per population, and the distinction matters: the +59-char CIDs **cannot** case-collide structurally — their alphabet has no +uppercase, so folding is the identity function on them; the 48-char +identities ARE mixed-case, so a fold collision is not structurally +impossible for them — merely astronomically improbable (two distinct +identities would have to differ only in the case of their letters), and +**zero occur across the 2,492 measured** (true case-fold collisions within +a bundle across all 28,542 ids: 0). Determinism is free: the +name is the id, no collision machinery, no global set needed — which also +retires `namer.Get`'s `rand.Int63n` nondeterminism (export.go:1435, 1462) +without replacing it with anything. + +**Two bonuses, both secondary to the argument above:** archives are +rename-stable (a renamed object keeps its path, so backup diffs show only +the content change — the same property SPEC §9 chose for `RefNames`, +default off, "the backup shape stays minimal and rename-stable"); and the +plan phase (§1.1) no longer performs collision resolution at all — the path +is a per-document pure function, and plan's remaining naming job is just +the manifest table. + +**The later human-readable mode — one mode, not designed now.** Readable +output is an export MODE to be added later, and it bundles **both** +readable filenames **and** type-subdirectory grouping under `objects/` into +one switch — one rule per mode, nothing half-legible. The default mode must +not foreclose it, and does not: nothing dispatches on paths +(import/pb/converter.go:338-341; SPEC.md:2319), and the manifest carries +whatever paths the writing mode chose, so the two modes differ only in the +exporter's path function. Facts already in hand for whoever designs it +(measured; do not re-derive): + +- The gain is thin for most spaces: ordinary objects per space median 20 + (max 2,001), distinct types per space median 3 (max 42) — grouping 20 + objects into 3 folders — and 115 of 359 type directories (32%) would + hold ≤ 2 objects. +- Type-name slugs are safe as directory names, unlike object names: zero + within-space type-slug collisions, one type name with a path-hostile + character (object names: 7.5% hostile, §2). +- Directories must be named by the resolved type NAME slug (stored key → + type document → name), never by the `type` wire spelling: **617 ordinary + objects (6.4%), across 52 distinct types, spell their `type` as a bson + key** (e.g. `69346f554c932bae256cbd02`) — 52 opaque hex directories + otherwise. +- The name-hazard table and slug-collision measurements in §2 apply to its + filename half. + +### 1.4 File blobs, and finding them (question c) + +Two problems must be solved together: how the bundle binds a `file_object` +document to its bytes, and how a human finds both. The thing being replaced +is the `source`-clobber: legacy export stuffs the archive-relative blob path +into `bundle.RelationKeySource` (export.go:1236, second site export.go:1196), +overwriting a real, user-facing, editable `url` relation named "Source" +(pkg/lib/bundle/relations.json:930-935) that bookmarks legitimately hold — +the corpus's very first sampled bookmark carries a real URL there — and the +pb importer reads the path back out of the same key +(import/pb/converter.go:404-414). A document member may not be a slot for +archive bookkeeping; that is the lesson, and neither alternative below puts +a path into the document. + +Facts that shape the design: 10,254 file objects (36% of all corpus +documents; median 25 per space, p90 505, max 2,242). Every one carries +`name`, `file_ext`, `file_mime_type`, `size_in_bytes` in `properties` — +but `file_ext` is dirty as a path component: 431 empty, 9 longer than 10 +chars, dozens non-alphanumeric (`0-rc01`, `9-alpha` — shrapnel of versioned +library filenames), and 12 literally `json`. SPEC §15 #20 (SPEC.md:6545) +fixes the bundle as FAT — the bytes travel, no `fileVariantKeys`, no +encryption keys — and this design carries bytes and nothing else; the thin +bundle's future marker slot is left untouched. + +**Alternative A — adjacency convention.** The blob sits beside its document +in `files/`, same stem, real extension: + +``` +files/bafyreigp3himcyqenxemyyk3iu7qtnmlglu4qgnn3r63b743ytzyp6hpv4.anyblock.json ← the document +files/bafyreigp3himcyqenxemyyk3iu7qtnmlglu4qgnn3r63b743ytzyp6hpv4.png ← the bytes +``` + +Binding rule: same stem, the one sibling that is not `*.anyblock.json`. +(Under §1.3's settled id naming the stem is the file object's id, so +adjacency and the pure id→path function coincide: doc and blob are both +direct functions of the id, differing only in extension.) +Human answer: the two files sort adjacent in any listing — nothing to +correlate, nothing to open. Nothing is added to any document or index. +Weaknesses: the rule is a convention a reader must know (exactly what +SPEC.md:2319 says the format refuses to define); the blob's extension must +be sanitized (empty/dirty `file_ext` → derive from `file_mime_type`, else +`.bin` — three-step rule where Alternative B needs none); an authored bundle +is forced into the same layout to be understood; and "exactly one non-doc +sibling per stem" is an invariant only tooling can police. + +**Alternative B — the manifest binds blobs.** `index.json`'s manifest — +whose charter is precisely "where to find what a reader must resolve by key +or id rather than by walking" (SPEC §2c, SPEC.md:2316-2319) — gains a third +member beside `types` and `properties`: + +```json +{ "manifest": { + "types": { "task": "types/bafyreihayoh64….anyblock.json" }, + "properties": "properties.json", + "files": { "bafyreigp3him…": "files/bafyreigp3him….png" } } } +``` + +One entry per file object: object id → archive-relative blob path. This is +the lookup the deleted manifest `options` map never had a reader for +(SPEC §2c, removed) — but blobs have exactly that reader: every +importer holding a file_object document must find its bytes, and every +export tool must enumerate them. The paths are free: an authored bundle +writes `"files": {"logo": "assets/logo.png"}` against its own slug ids and +any layout it likes, which is what makes consumer 2 (SPEC.md:1543) a +first-class citizen rather than a convention-follower. An absent blob for a +declared file object — or an entry pointing outside the bundle — is a +cross-document refusal in `anyblockvalidate`, like every other manifest +path (SPEC §2c). Cost: manifest weight (max observed 2,242 entries ≈ +200 KB in the heaviest space — noise next to the blobs themselves) and one +indirection when browsing by hand. + +**Recommendation: B as the mechanism, A as the layout.** The manifest +`files` map is the authoritative binding — the only rule a reader needs, +author-writable, layout-free, and refusable by tooling. Our exporter then +CHOOSES to lay blobs out adjacently (same directory, same stem — the file +object's id under §1.3, a readable stem again in the later human mode — +sanitized real extension), so doc and blob always sort side by side — +but adjacency is convention, carried by the map, never load-bearing. No +fallback stem-matching in the reader: one mechanism, or the two drift. The +document itself carries no path, `source` keeps meaning what its relation +says it means, and the round-trip comparator never has to special-case an +archive artifact out of the diff. + +Blob extension sanitation (cosmetic only, since the map binds and +`file_mime_type` travels in the document): `file_ext` lowercased and +restricted to `[a-z0-9]{1,10}`; failing that, the conventional extension +for `file_mime_type`; failing that, `bin`. `anyblock.json` as a computed +final suffix is impossible by construction (`[a-z0-9]{1,10}` admits no dot). + +### 1.5 Concurrency and determinism + +The constraint: `writeDoc` runs as concurrent queue tasks +(export.go:426-428), while the dictionary accumulates across every document +and the manifest accumulates paths. Three options were on the table: + +- **Single-threaded compose** (what the harness does). Simplest, but the + emit phase is where all the I/O and marshal cost lives — the biggest + observed space is 4,884 documents plus 2,242 blobs, and an all-spaces + export is 38k documents — and serializing it is a real regression + against today's exporter for zero correctness gain. +- **Two-phase (emit concurrently, compose from re-read files)**. Dies on + the zip writer: entries cannot be re-read before `Close` + (writer.go:130-148), so composition facts must be captured in-process + anyway — the re-read variant only works for directory exports and would + fork the code path. +- **Plan/emit/finish with a mutex-guarded accumulator** — chosen. The plan + phase (§1.1) removes the one ordering-sensitive computation — filenames — + from the concurrent section entirely: under §1.3 each name is a pure + function of its own id, and the plan fixes the manifest tables before + the first task starts (`namer.Get`'s nondeterminism is retired with the + naming scheme itself, §1.3). What remains shared during emit is commutative map/set insertion + (installed keys, dictionary entries, option vocabularies, used property + keys, the index lift) — guarded by one mutex, held for microseconds per + document against marshal work measured in milliseconds (the §9a census + alone is 4.2 ms → 6.7 ms on a 1,630-block document, SPEC.md:5200), so + contention is noise. Determinism of the OUTPUT never leans on write + order: `finish` sorts everything it writes, and the package's canonical + marshallers already sort keys and refuse unstable forms (SPEC §2c/§2f; + I1, SPEC §11). + +Writer-level concurrency is already safe: `zipWriter.WriteFile` serializes +on its own mutex (writer.go:130-131), and `dirWriter.WriteFile` +(writer.go:66) needs no lock because the plan guarantees distinct paths. +The zip's per-entry `Modified` timestamps come from document state +(`lastModifiedDate`, export.go:1272), not from the clock, so archive bytes +stay stable; the one clock leak is the archive's own root/temp name +(`Anytype.20060102.150405.99`, writer.go:30), which names the artifact, not +its contents. + +**Bounded width.** The export queue is ALREADY width-bounded — `NewQueue(…, +4, …)` (export.go:152-156; process/queue.go:42-44, 94) runs at most 4 tasks +at once, and the N queued tasks are thin closures, not loaded objects. So +in-flight marshal work is capped today; the unbounded term is cache +retention, not task count (§1.6). The width itself should follow the repo's +existing prior art for exactly this problem: the reindex limiter caps +cross-space passes at **2 on mobile, 4 on desktop** +(`maxConcurrentSpaceReindexFor`, core/indexer/reindexlimiter.go:15-20, +wired at indexer.go:123), with a rationale comment describing precisely our +situation — "each pass cold-builds every … object into the space's object +cache and relies on the cache TTL to release it, so … the resident set +peaks at hundreds of MB" (reindexlimiter.go:8-14). Recommendation: keep 4 +on desktop (matches today's export behaviour), drop to 2 on +mobile, same platform switch. A width cap trades wall-clock for peak RAM; +the trade is cheap here because emit is storage-read-bound (the same +argument reindexlimiter.go:12-13 makes), so halving width on mobile costs +much less than 2× wall-clock while halving the in-flight term. + +*Implemented as two runners.* Which pool runs emit is injectable +(`anyblock.EmitRunner`). Driven from the RPC the tasks go to the export +queue itself — that is what keeps the progress numbers a client already +watches counting for this format — and the queue is created with +`anyblock.EmitWidth()` workers for it, so the width above still bounds the +resident set. Driven from the Go API (tests, cmd tooling) the package's own +fixed-width pool runs them. Both honour cancellation, which the first +implementation ignored entirely: the internal pool stops FEEDING on +`ctx.Done()`, every task checks `ctx.Err()` before it loads anything (the +queue-backed runner already holds all N tasks, so only the per-task check +can stop that one), and an abandoned run returns without writing +`index.json` or `properties.json` — a bundle whose index states documents +the emit never wrote is worse than no bundle. Tasks already in flight are +allowed to finish: each holds a loaded object and possibly a half-written +file. + +**Close after write — active, immediate, TTL-independent.** The cache has +no refcount, and its PASSIVE path is TTL-only — `GC()` closes entries where +`isActive() && lastUsage.Before(now-ttl)` (any-sync +app/ocache/ocache.go:343-355, entry.go:42-46; `WithTTL(60s)` + +`WithGCPeriod(time.Minute)`, objectcache/cache.go:80-85) — so an exporter +that does NOTHING retains **throughput × 60-120 s** of loaded CRDT trees: +4 workers at a few ms per document sustain hundreds of documents per +second, thousands of resident trees for a big space, each far larger than +its exported JSON. But the exporter need not do nothing, and the active +path does not wait for the TTL at all (verified): `ocache.TryRemove` +delegates to `e.value.TryClose(c.ttl)` (ocache.go:250-271), and +`smartBlock.TryClose` **ignores the TTL argument entirely** — its whole +body is `TryLock()`-or-fail, `IsLocked()`-or-fail, else `closeLocked()` +(core/block/editor/smartblock/smartblock.go:1153-1162), where `IsLocked` +counts sessions with an ACTIVE event sender, i.e. clients that currently +have the object open in the UI (smartblock.go:633-641). **An object closes +iff nobody else has it open, immediately, regardless of TTL or last-usage +time.** The repo's bulk-walk precedent already works this way: the +fulltext indexer opens the object, extracts, then calls +`TryRemoveFromCache(ctx, objectId)` and logs rather than fails on error +(core/indexer/fulltext.go:330-352, the call at :348; +core/block/service.go:222-234 → objectcache/cache.go:161). The emit task +ends the same way: write, observe, `TryRemoveFromCache`, log-only on +failure. No TTL or GC-period tuning appears anywhere in this design — both +are irrelevant once the exporter closes actively; TTL remains only as the +backstop that collects the failure cases below. + +The failure modes, all benign and all bounded: + +- **`TryLock` fails** — someone holds the lock at that instant. Transient; + skip it, no retry loop, the passive TTL collects it later. +- **`IsLocked` true** — the user genuinely has the object open. Not + evicting it is the CORRECT behaviour, not a limitation; the term it + contributes to peak RAM is user-bounded (§1.6), never space-bounded. +- **Two editors refuse to close unconditionally** — `SpaceView.TryClose` + (core/block/editor/spaceview.go:141) and `accountObject.TryClose` + (core/block/editor/accountobject/accountobject.go:340) both + `return false, nil`. Both are singletons, and measured across all 28,542 + exported corpus documents there are **zero** `space_view` and zero + `chat` documents in an export — noted for completeness, irrelevant to + export memory. +- **`chatobject.storeObject.TryClose`** refuses while a subscription is + active (core/block/editor/chatobject/chatobject.go:635-649) — again + user-bounded, and again a kind exports do not carry. + +**The GO-7333 dependency, named.** The recommended path reaches a known, +filed-but-unfixed any-sync bug: `ocache.TryRemove` on an entry still in +`entryStateLoading` can nil-deref at ocache.go:271 — which is exactly the +`e.value.TryClose` call this design leans on — or hang permanently +(`setClosing` flips loading→closing but `e.load` is never closed, so later +waiters block forever), and races on `e.value`. It is a race window — the +indexer has been calling this on every indexed object without it obviously +firing — but a high-volume exporter would be the third and heaviest +caller. The exporter's own call happens after its own `cache.Do` returns, +so the entry it evicts is loaded, not loading; the race arms only when +another caller (UI open, sync) re-loads the same object concurrently. +Weighed in Q11; the design assumes close-after-write and names the bug as +its dependency. + +### 1.6 The memory model + +Hard constraint: exporting a huge space must not hold all object content in +memory — peak RAM bounded by a window, not by space size. The pipeline's +terms, honestly labelled, with measured sizes (corpus: 77 spaces, 28,542 +documents; the worst single space 4,884 documents): + +**O(all objects) — retained for the whole export, and accepted:** + +- **Collected details.** `Doc` holds only `*domain.Details` + (export.go:200-204); content is never resident in the collection — + blocks/state load per-document inside the emit task via `cache.Do` + (export.go:1222) and go out of scope after the write. Measured: **16.9 MB + of `properties` JSON across all 28,542 documents, 593 B average**; as + in-memory proto Structs with Go map overhead, budget several times that — + call it a few tens of MB for a 38k-object account. This term is inherent + to computing the dependency closure and the plan, and it is the price + this design knowingly pays. (`transformToDetailsMap`, export.go:227-235, + re-wraps the same `Details` pointers, not copies — and the native emit + path does not need `SetKnownDocs` at all, since references print as full + ids with resolvers wired from the store.) +- **The plan table** (§1.1): one path string + kind per document, ~100 B + each — ~3 MB at 28k documents. +- **The composer aggregates** — and never the documents. What emit retains + per document is: used property keys (a set; a space USES a median 57 + keys, SPEC §2f), installed bundled keys, divergent dictionary entries, + option vocabularies, type→path manifest entries (22 types/space median, + SPEC §2c), and the index lift. The proof of size is the files these + aggregates become: measured per space over the sweep, `properties.json` + is median 13.4 KB, p90 26.8 KB, **max 120.8 KB**, total 1.42 MB across + all 77 spaces; `index.json` median 2.6 KB, max 8.3 KB. The aggregate is + three orders of magnitude below the content that streams through it. + +**O(in-flight window) — the content term, and the cap is the whole lever:** + +- **Loaded CRDT trees in the object cache.** A loaded smartblock holds the + full change history — far larger than its exported JSON. Because §1.5's + close-after-write is immediate and TTL-independent (an object closes iff + nobody else has it open, smartblock.go:1153-1162), the resident content + set is **≈ the emit width, exactly** — at most N export-loaded objects at + any instant, where N is the concurrency cap. Peak content RAM is + therefore **not** O(throughput × TTL) and **not** proportional to space + size; the cap is a real design parameter with a clean meaning — "at most + N objects resident for the export" — not a throughput guess. That is + what justifies the default: 4 on desktop / 2 on mobile per the + reindex-limiter precedent (§1.5), i.e. at most four trees plus their + marshal buffers in flight. + +**O(UI-open objects) — user-bounded, not space-bounded:** + +- Objects the close call correctly refuses: locked-right-now (transient), + UI-open (`IsLocked`, smartblock.go:633-641), the always-refusing + singletons and subscribed chat stores (§1.5 failure modes — and the + corpus shows zero of those kinds in any export). This term scales with + what the user is looking at, never with what is being exported. + +**O(width) spikes — bounded but non-trivial, stated:** + +- **Marshalled document buffers.** Each emit worker holds ONE loaded state + plus its marshalled bytes (kept briefly for the used-key scan, §1.1) + before write. Measured: blocks total 153.1 MB JSON at 5.4 KB average, but + the **largest single document is 9.30 MB** and 23 documents exceed + 256 KB. Worst case width × max ≈ 4 × 9.3 MB ≈ 37 MB of JSON buffers — + a spike, not a leak, and nothing in the pipeline ever holds more than one + document's content per worker. +- **Blobs are streamed, never buffered**: `saveFile` pipes + `file.Reader → wr.WriteFile → io.Copy` (export.go:1306-1311, + writer.go:66-90), and the native emit keeps that shape. + +Summary — the peak-RAM model this pipeline is designed to: + +``` +peak = O(in-flight emit window) width x (one tree + one marshal buffer); + the concurrency cap controls it (4/2) + + O(UI-open objects) user-bounded, never space-bounded + + O(all details) 16.9 MB JSON / 28,542 docs measured; + a few tens of MB resident, accepted + + O(bundle aggregates) <= 120.8 KB max observed per space +``` + +Without the active close, the passive TTL window (throughput x 60-120 s) +dominates everything and the constraint is not met — which is why +close-after-write is design, not optimization. + +### 1.7 What the composer owes the format + +- **I1 at bundle scope**: `finish` re-reads `index.json` and + `properties.json` through `UnmarshalIndex`/`UnmarshalPropertyDictionary` + before writing, as the harness does (main.go:983-1012) — a bundle this + exporter writes that the package refuses is this exporter's bug, found + at export time. +- **Omissions are lifts, never drops**: the space-settings document, the + widget object, and matching bundled-relation documents are omitted only + through the package predicates, whose lift-before-omit ordering and + reconstruction checks (`WidgetsSnapshot` verified via `snapshotdiff`, + main.go:786-800) come along unchanged. +- **Deterministic bytes end to end**: same space state ⇒ same file set, + same names, same bytes per file. This is a testable property and should + be a test: export twice, compare trees. + +--- + +## 2. Alternatives considered and rejected + +**Layout: keep the legacy directory names.** Rejected: `relations`/ +`relationsOptions` reintroduce the word the format's vocabulary banned +(PRINCIPLES rule 3), the camelCase compounds contradict the format's own +naming rule (SPEC §1 Naming), and since the importer provably never reads directory +names (import/pb/converter.go:338-341 is the only path rule), compatibility +buys nothing. + +**Layout: genuinely flat — every document in one directory.** The purer +endpoint of §1.3's id rule: `objects/.anyblock.json` for ALL kinds +makes id→path a total pure function with no kind probe at all, and with +id filenames the kind directories' human value is thin anyway (opaque +names in legible folders). Declined, not killed: the kind split was +approved (Q1), it still separates blobs from documents and keeps kind +tallies visible when debugging, the 7-directory probe it costs is bounded +and free in practice (§1.2, the zip central directory), and — because no +reader dispatches on paths — collapsing to flat later is a convention +change, not a format change. Per-object folders (one directory per +document) stay rejected outright: they double every path and answer +nothing. + +**Naming: pure slugs with dedup counters.** Never viable, on measurement +(28,542 corpus documents): + +| hazard | count | +|---|---| +| empty name | 654 (2.3%) | +| contains a path-hostile char (`/ \ : * ? " < > \|`) | 2,144 (7.5%) — `/` 1,514 · `:` 972 · `\|` 288 · `"` 161 · `?` 109 | +| leading/trailing space or dot | 444 | +| non-ASCII | 2,192 (7.7%); NFC ≠ NFD for 250 | +| longer than 100 UTF-8 bytes | 495 (max 2,536 bytes — over the 255-byte component limit ten times over) | +| Windows reserved device names (CON, PRN, AUX, NUL, COM1-9, LPT1-9) | 0 | + +Deduplicating by slug alone, every one of the 77 spaces has collisions — +4,964 documents (17.4%) collide with a sibling; pages only is still 1,569 +of 9,688 (16.2%) in 42 of 77 spaces; file objects are worst at 25.6% (one +space holds 373 files that all slug to `github-com-cover`). The "rare" +collision suffix would be the fourth-commonest thing in the archive, and +any first-writer-wins counter is nondeterministic under the concurrent +queue — `namer.Get` (export.go:1435) is this alternative's existing +implementation, and its `rand.Int63n` suffix is the exhibit. Raw names +with escaping instead of slugging fail the same table with extra steps +(`ApiSlug`'s `/`, `#`, `@` leakage, SPEC.md:6222-6235, is the same lesson +one layer down). + +**Naming: hybrid `--.anyblock.json`.** This document's own +prior proposal, fully specified (deterministic slug transform, 8-char id +suffix, case-folded planning, suffix-lengthening tie-break) — and +**overturned by the human on the reference-resolution argument** (§1.3): a +reference carries an id and nothing else, so under hybrid names the only +path from a reference to its file is scanning documents (or a +name-carrying index that must then be kept in step — the same indirection +§9a's deleted `refs` legend was killed for, one level up). The slug half +bought listing legibility and nothing else; that value moves whole into +the later human-readable mode, where it belongs, bundled with type-subdir +grouping so no mode is half-legible. The measurements above survive the +overturn: they still prove slugs need an id beside them wherever slugs +appear — now in that mode's design instead of the default's. + +**Blobs: keep the `source` detail hack.** Rejected — it is the thing being +replaced: it destroys a real user value in a real editable relation +(relations.json:930), the destruction round-trips through import +(import/pb/converter.go:404-414), and it makes a document's content a +function of the archive that contains it. + +**Blobs: a path member on the file_object envelope** (e.g. `"file": +"files/x.png"`). Rejected: it moves the Source hack into the format instead +of deleting it — the document would describe its container, export ≠ +Marshal for one kind, the §11 comparator needs an exemption, and a document +copied between bundles silently dangles. The format's own precedent is the +other way: `icon.file` holds an object id, never a path (SPEC §2b), and the +33 corpus objects holding filesystem paths in `coverId` are treated as +damage, not data (SPEC §11(e)). + +**Blobs: adjacency convention alone (no manifest map).** Rejected as sole +mechanism: it makes a layout convention load-bearing in a format that +explicitly refuses to define layout (SPEC.md:2319), leaves authored bundles +(consumer 2) with no way to point at assets laid out their own way, and its +"exactly one sibling" invariant is unpoliceable except by tooling that +would then be reimplementing the map it refused to write. Kept as our +exporter's LAYOUT under the map (§1.4). + +**Concurrency: single-threaded compose / re-read-based compose.** Both +rejected in §1.5 — the first for cost on the measured worst cases, the +second because the zip writer cannot re-read its own entries. + +--- + +## 3. QUESTIONS + +Q1-Q3 were answered by the human on 2026-08-26 and are kept below as +settled records, SPEC §15 house style — the decision, the overturned +alternative, and the reasoning that decided it. Q4-Q11 remain open. + +**Q1. Directory names — SETTLED: approved as proposed, `objects/` flat.** +The set `objects/ types/ templates/ properties/ options/ participants/ +files/` stands (§1.2), `properties/` beside `properties.json` included. +The human added one ruling the proposal had left implicit: **`objects/` +stays FLAT by default** — no type subdirectories. Type grouping belongs +exclusively to the later human-readable mode (§1.3, Q3), and when that +mode builds it, directories are named by the resolved type NAME slug +(stored key → type document → name), never the `type` wire spelling — 617 +ordinary objects (6.4%), across 52 distinct types, spell their `type` as a +bson key, which would otherwise mint 52 opaque hex directories. The +measured case against default type-subdirs: median space has 20 ordinary +objects across 3 types, and 115 of 359 type directories (32%) would hold +≤ 2 objects. The kind-split-vs-flat tension this creates with Q2's id rule +is resolved in §1.2 (bounded 7-directory probe, stated plainly; the +genuinely-flat alternative recorded in §2). + +**Q2. Document filenames — SETTLED: `.anyblock.json`, hybrid +overturned.** The standing proposal was hybrid `--`; the human +rejected it on the reference-resolution argument, recorded in full in +§1.3: a reference carries an id and nothing else, the format addresses +everything by id, so id→file must be a pure function rather than a scan — +"right now it's transparent — we have ids everywhere"; an authoring agent's +minted ids can BE its filenames. Rename-stability of backups is a +secondary bonus, not the reason. The slug-hazard and collision +measurements move to §2, where they still kill pure slugs — now in +support of this conclusion instead of the hybrid. + +**Q3. Readable filenames as an option — SETTLED, inverted.** Not +"ids-only as an option": ids ARE the default, and **human-readable output +is the option, added later** — one mode bundling BOTH readable filenames +AND type-subdirectory grouping under `objects/`, so no mode is +half-legible (one rule per mode). The default forecloses nothing: no +reader dispatches on paths, and the manifest carries whichever paths the +writing mode chose (§1.3). The mode itself is deliberately NOT designed in +this document; §1.3 records the measurements its designer will need. + +**Q4. Blob binding — manifest `files` map (Alt B) with adjacent layout?** +Why it matters: this is the `source`-hack replacement, it touches +`index.json`'s schema (a format change: new manifest member, +`index.schema.json`, §2c text), and it decides how authored bundles ship +assets. Options: (a) manifest map + adjacent layout, no reader-side +stem-matching (§1.4 recommendation); (b) adjacency convention only, no +format change; (c) both mechanisms with map-wins precedence. +**Recommendation: (a). (b) leaves consumer 2 without free layout and makes +convention load-bearing; (c) is drift by construction.** + +**Q5. May a native bundle's file_object documents live beside blobs in +`files/`, given the pb importer skips that directory wholesale?** +Why it matters: a native bundle fed to the LEGACY pb importer +(import/pb/converter.go:338-341) would have its file documents silently +skipped — but a native bundle is not pb-importable anyway (different codec, +different extension), so the question is really whether we guarantee +anything about legacy importers seeing native bundles. Options: (a) no +guarantee — native bundles are read by native wiring +(`anyblockconvert`/`ObjectImportExperience` path), the layouts are +independent; (b) keep file documents out of `files/` (a `file_docs/` split) +purely for defensive overlap. **Recommendation: (a) — the defensive split +re-creates the legacy two-directory correlation cost to protect a path that +cannot parse the files anyway.** Verified at implementation: the pb +importer parses every `.json` file as jsonpb (converter.go:285), so a +native bundle fed to it fails on every document in every directory — the +`files/` skip changes no outcome, and no partial import can silently drop +just the file documents (SPEC §2c records this under the exporter's +convention). + +**Q6. Which RPC surface does the native exporter answer to?** +Why it matters: `model.ExportFormat` today has `Protobuf`/`JSON` (pbjson) +routed by `isAnyblockExport` (export.go:499); the native format needs an +addressable enum value, which is a protocol change in anytype-proto, and a +deprecation story for `Export_JSON` (pbjson). Options: (a) new enum value +(e.g. `Export_AnyBlockJSON`), pbjson untouched until clients migrate; +(b) repurpose `Export_JSON` in place — silently changes what existing +clients receive; (c) ship native behind `Export_JSON` + a request flag. +**Recommendation: (a); (b) breaks consumers on a version boundary they +can't see, (c) is a dialect switch inside one format id.** +Shipped as (a) (2026-08-27): `Export_AnyBlockJSON = 6` on +`model.Export.Format`, `Export_JSON` (pbjson) untouched. Three things the +wiring settled that the question did not ask: the export service passes the +doc set it already collected rather than letting the exporter re-collect +(one whole-space query per export, not two — `Exporter.ExportCollected`); +emit runs as `process.Queue` tasks through an injected runner, so progress +and `ProcessCancel` behave exactly as they do for the five legacy formats, +with the queue created at the emit width for this format (§1.5); and +`ExportSingleInMemory` answers with ONE document, no bundle files, per Q7's +position below. The predicate that used to route pb/pbjson was named +`isAnyblockExport` — a name that predates this format and now reads as its +opposite — and was replaced by an explicit format switch rather than +extended. + +**Q7. Does the native format become the DEFAULT space backup, and on what +timeline does pb export retire?** +Why it matters: decides how much compatibility weight the writer carries +(e.g. whether anyone still needs `profile` emitted) and what +`ExportSingleInMemory` (export.go:112) serves. Not answerable by +measurement — product call. **Recommendation: native becomes default only +after the native import path ships and a full-account round-trip soak +matches the sweep's 99.98%; single-object in-memory export emits exactly +one document, no bundle files, per PRINCIPLES rule 7 ("a document stands +alone").** + +**Q8. Extension: settle SPEC §15 #1 as `.anyblock.json`?** +Why it matters: §15 #1 (SPEC.md:6215) still leans bare `.json`, but the +bundle now legitimately contains blobs that are themselves `.json` files +(12 corpus file objects have `file_ext == "json"`), and `DiscoverJSONFiles` +plus the importer need one cheap, collision-free document test. +Options: (a) `.anyblock.json` (what the harness already writes, +main.go:377); (b) bare `.json` with content sniffing via `DetectFormat`. +**Recommendation: (a) — the double extension is the entire skip-rule for +non-document files, and it costs nothing; update §15 #1 to settled.** + +**Q9. Should `index.json` carry any cross-space super-index for +multi-space exports?** (re-answered after Q2's overturn — the prior +recommendation used the rejected hybrid scheme for space directories) +Why it matters: `spaces//` wrappers make each space a self-contained +bundle; a top-level listing (space names → directories) would help a human +facing 77 CID-named directories, but it is a new format surface with no +reader yet. Options: (a) nothing at top level — plain `spaces//`, +consistent with Q2: a space reference is an id, and id→bundle-root stays a +pure function; (b) a minimal top-level `index.json` naming each space +bundle; (c) readable space directory names — now part of the later +human-readable mode (Q3), not the default, by the same one-rule-per-mode +principle. **Recommendation: (a) for the default mode; readable space +directories ride the human mode when it is designed. (b) only if a real +consumer materializes — each per-space `index.json` already carries the +space's own name, so a tool can build the listing in one pass.** + +**Q10. Where does the promoted composition code live?** +Why it matters: the composer must be importable by `core/block/export/anyblock` +AND the cmd tools, and `cmd/internal/anyblockbatch` is importable by +neither production code nor anything outside `cmd/`. Options: (a) the +store-wired composer in `core/block/export/anyblock`, with the pure +byte-level pieces (used-key scan, path planning helpers) exported from +`pkg/lib/anyblockjson`; (b) a new `pkg/lib/anyblockjson/compose` +subpackage holding the whole bundle-level composition, wired by both. +**Recommendation: (b) — SPEC §13 already gives composition a named home +("bundle tooling"), the harness's spaceComposer moves there nearly intact, +and the cmd tools shed their private copy; `core/block/export/anyblock` +then only wires store, cache, and writer to it.** + +**Q11. Close-after-write ships against unfixed GO-7333 — fix first, or +gate?** +Why it matters: the memory model (§1.6) stands on the emit task calling +`TryRemoveFromCache` after every write (§1.5), and that path reaches the +filed-but-unfixed any-sync bug GO-7333 — `ocache.TryRemove` on a +still-loading entry can nil-deref at ocache.go:271 (the very +`e.value.TryClose` call), hang later waiters forever, or race on +`e.value`. The exporter would be the heaviest caller of this path ever. +Options: (a) fix GO-7333 in any-sync first and make the exporter depend on +the bumped version; (b) ship close-after-write anyway, accepting the same +race the fulltext indexer already runs on every indexed object +(fulltext.go:348) — the exporter's call lands after its own `cache.Do` +returns, so the entry is loaded, and the window arms only on a concurrent +re-load by another caller; (c) TTL-only until fixed — rejected by the +memory model itself (§1.6: the passive window is throughput-proportional +and unbounded by space size). **Recommendation: (a) — the fix is small and +already scoped in the GO-7333 filing, and an export that can hang an ocache +entry forever is a worse failure than the memory peak it prevents; (b) is +acceptable as an interim only if the any-sync bump cannot land in the same +release, since the indexer has soaked the identical race in production at +scale.** + +--- + +## 4. Migration and compatibility notes + +**Existing exports.** Nothing changes for them: legacy pb/pbjson archives +keep importing through the pb importer, whose only path rule +(import/pb/converter.go:338-341) native bundles never relied on. The legacy +writer, `namer`, and md/pb/dot/graphjson converters are untouched — the +extraction moves collection OUT of `export.go`; the legacy writing path +keeps calling it through the same interface. + +**Native bundles** are read by the native wiring only +(`cmd/anyblockconvert` → `ObjectImportExperience` path today; the +production native importer is separate future work). A native bundle is not +a valid pb import and does not pretend to be. + +**Markdown later.** The md exporter keeps its own naming (`makeMarkdownName`, +export.go:1362) and writer for now; the collect interface below is +format-agnostic (`Closure` replaces `isProtobuf`), so md can migrate onto +the same seam later without this design changing — that migration is +explicitly not designed here. + +**Extracted collection interface** (outline only — signatures, no bodies): + +```go +// core/block/export/collect +package collect + +type Closure int + +const ( + // ClosureContent — the md-style closure: nested objects and linked + // files only (export.go processNotProtobuf, :593). + ClosureContent Closure = iota + // ClosureDerived — the collect-everything-derived closure the native + // format wants: types, relations, options, templates, dataview + // dependencies, recommended relations (export.go processProtobuf, :610). + ClosureDerived +) + +type Request struct { + SpaceId string + Ids []string // empty = whole space (export.go getExistedObjects, :1138) + Closure Closure // replaces the bare isProtobuf bool (export.go:503-504) + IncludeNested bool + IncludeFiles bool + IncludeArchived bool + IncludeBacklinks bool + IncludeSpace bool + StateFilters *state.Filters +} + +type Doc struct { + Details *domain.Details + IsLink bool +} + +type Collector interface { + Collect(ctx context.Context, req Request) (map[string]*Doc, error) +} +``` + +```go +// pkg/lib/anyblockjson/compose (per Q10 recommendation) +package compose + +// Plan is the deterministic path table (a pure per-id function under §1.3) +// plus the manifest tables, built single-threaded from the collected +// details before any emit (design §1.1; omission is decided at emit). +type Plan struct { /* id → {path, kind}; manifest tables */ } + +func BuildPlan(docs map[string]DocMeta, opts PlanOptions) (*Plan, error) + +// Composer accumulates bundle facts during concurrent emit and writes the +// two bundle files at finish. Observe* methods are safe for concurrent use. +type Composer struct { /* unexported; mutex-guarded */ } + +func NewComposer(opts anyblockjson.Options, plan *Plan) *Composer +func (c *Composer) ObserveDocument(id string, data []byte, sw SnapshotMeta) error +func (c *Composer) ObserveOmitted(sw SnapshotMeta) error +func (c *Composer) Finish() (index, properties []byte, err error) // re-read-verified (I1) + +// UsedPropertyKeysFromBytes — the byte-level promotion of +// cmd/internal/anyblockbatch.UsedPropertyKeys (scan.go:908), shared with +// the cmd tools (design §1.1). +func UsedPropertyKeysFromBytes(doc []byte) (map[string]bool, error) +``` + +```go +// core/block/export/anyblock — the wiring: store + cache + writer around compose +package anyblock + +type Exporter struct { /* picker, objectStore, fileService, resolvers */ } + +func (e *Exporter) Export(ctx context.Context, req collect.Request, wr writer.Writer) (succeed int, err error) +// internally: collect → compose.BuildPlan → queue tasks +// {Marshal + wr.WriteFile + blob stream + composer.Observe*} → composer.Finish +// → wr.WriteFile(index.json, properties.json) (the postProcess seam, export.go:1529) +``` + +**SPEC follow-ups this design creates** (to be filed with the SPEC when +implementation starts): the manifest `files` member (Q4) — schema + §2c +text + `anyblockvalidate` cross-checks; §15 #1 settled to `.anyblock.json` +(Q8); a §2c note recording THIS exporter's directory and filename +convention in the "one exporter's convention" slot. diff --git a/pkg/lib/anyblockjson/HANDOFF_API.md b/pkg/lib/anyblockjson/HANDOFF_API.md new file mode 100644 index 0000000000..9090e64364 --- /dev/null +++ b/pkg/lib/anyblockjson/HANDOFF_API.md @@ -0,0 +1,184 @@ +# AnyBlock JSON switched to raw display names — what the API layer needs to know + +Written for the API v2 workstream, which merged `go-7383-anyblockjson` at +`1ccf34e7c` — one commit before the switch. Nineteen commits landed after that +point. **`core/api` was deliberately not touched**: `git diff --stat +1ccf34e7c..HEAD -- core/api/` is empty. Nothing here has broken the API. But the +format now spells properties differently from the API, and that divergence is a +decision the API layer owns rather than one this change made for it. + +--- + +## 1. What changed + +The format used to spell a property key by deriving an api slug — the stored key +run through `strcase.ToSnake`, or the space's `apiObjectKey` when it had one: + +```json +"properties": { "name": "Sprint 12", "created_date": "…", "due_date": "…" } +``` + +It now spells every property and type key by the entity's **display name**, NFC +normalized and otherwise verbatim — bundled entities included, from the +`relations.json` / `types.json` names: + +```json +"properties": { "Name": "Sprint 12", "Creation date": "…", "Due date": "…" } +``` + +`api_object_key` is no longer read anywhere on the format's resolution path. It +is still written by `objectcreator` and still read by `core/api` — that half is +untouched. + +## 2. Why + +Four things drove it, in the order they were established. + +**A measured eval, not a preference.** The concern was that small models fumble +space-bearing JSON keys. Tested A/B on `google/gemma-4-e4b` (192 generations, +208 property observations per arm), graded by the real codec. Result: raw names +were handled *at least as well* as snake_case — 152/160 vs 152/160 on the +classes where both conventions have an unambiguous target, p = 1.0, and 192/192 +parsed as JSON with zero dropped spaces, zero case drift, zero invisible-character +keys. The failures ran the other way: asked to *derive* a key, models improvised +(`완료` → `completed`, `Дата выполнения` → `due_date`) and improvised +*differently across documents* — cross-document key stability 85.7% for derived +against 96.6% for raw. + +**Derivation was a step in every writer's path.** Normalization is computable +from the name, but every model has to do it, every time, and get it right. Raw +naming deletes the step. + +**It deleted a whole failure class.** Under normalization a name could fail to +produce a key at all — `#`, `☕`, `C++` normalize to empty or to `c` — so the +format carried an empty-normalization fallback, a leading-`_` escape for +digit-initial and keyword names (`50% done` → `_50_done`), and a stored-key +fallback. None of those can arise now; all three are deleted rather than +reimplemented. + +**Transliteration was actively wrong.** `unidecode` rendered Japanese 作業内容 as +the Chinese reading `zuo_ye_nei_rong`, Korean 완료 as `wanryo`, and Arabic مهمة +as `mhm@` — with an `@` the api key grammar does not even admit. The format's own +rule preserves the script: `Задача` stays `задача`, `作業内容` stays `作業内容`. + +## 3. Resolution is forgiving, and that matters for the API's choice + +The format folds on read — casefold plus separator-strip — so **the old +spellings still resolve**: + +``` +"Name" "name" "NAME" → name +"Creation date" "creation_date" "created_date" → createdDate +"Due date" "due_date" "DUE DATE" → dueDate +``` + +An API that keeps emitting `due_date` into an AnyBlock document is understood. +The reverse is not automatic: a consumer that expects `due_date` and receives +`"Due date"` needs the same fold, which `bundle.FoldApiKey` provides. + +## 4. What did change under the API's feet + +`core/api` is untouched and its **api keys are unchanged** — `bundle.ApiSlug` +derives from the internal *key*, never from the name, so `audio_genre` and +`space` remain what they were. `ApiSlug`, `ApiSlugFromName`, `SanitizeApiSlug`, +`MintApiSlug`, `MintApiSlugFromName` and `FoldApiKey` are all still there. + +**But thirteen bundled display NAMES moved.** If anything in the API matches on +a name rather than a key, check these: + +| stored key | was | now | +|---|---|---| +| `relationKey` | Relation key | **Property key** | +| `relationOptionColor` | Relation option color | **Property option color** | +| `relationReadonlyValue` | Relation value is readonly | **Property value is readonly** | +| `relationFormatObjectTypes` | Relation's target object types | **Property's target object types** | +| `featuredRelations` | Featured Relations | **Featured properties** | +| `headerRelationsLayout` | Header relations layout | **Header properties layout** | +| `recommendedRelations` | Recommended relations | **Recommended properties** | +| `recommendedFeaturedRelations` | Recommended featured relations | **Recommended featured properties** | +| `recommendedHiddenRelations` | Recommended hidden relations | **Recommended hidden properties** | +| `recommendedFileRelations` | Recommended file relations | **Recommended file properties** | +| type `relationOption` | Relation option | **Property option** | +| `audioGenre` | Genre | **Audio genre** | +| type `space` | Space | **Space settings** | + +All ten `relation*` relations are `hidden: true`, so no user-visible label moved +for them. `audioGenre` and the `space` type are the two user-visible changes — +`space` is hidden too, and `spaceView` deliberately keeps the name "Space", +because that is the object a user thinks of as a space. + +The first eleven exist because the retired alias table (`alias.go`) used to respell +`relationKey` as `property_key` on the wire. Deleting that table without renaming +the underlying names would have put "Relation" back into the format on ~6,900 +documents. The rename does the same job with no table behind it. + +## 5. The decision the API layer owns + +The format is settled. The API surface is not, and these are genuinely separate +— the format is a document at rest, the API is a request/response contract with +different consumers and a different compatibility story. + +**Option A — keep `api_key`.** No client breaks, and a stable identifier that +survives renames is exactly what a long-lived integration wants. Cost: two +vocabularies in one product, and a caller who reads a bundle and then calls the +API has to translate between them. + +**Option B — switch to raw names.** One vocabulary everywhere; an agent reading a +user's "set Due Date to Friday" maps it straight through. Cost: a breaking change +for existing clients, and renames move the address. + +**Option C — configurable, stated explicitly per request.** A header or parameter +declaring which vocabulary the caller speaks, so both are first-class and the +choice is the caller's. Cost: two code paths to keep honest, and a default to +choose for callers who say nothing. + +**A shape worth considering** — it follows from the audiences rather than from +the format: + +- **Tool wrappers and agent-facing surfaces: raw names.** The whole argument for + the format applies unchanged. An agent has the user's words and the document's + words; making them the same removes a translation step that the eval showed + models perform unreliably. This is the case where raw names are clearly right. +- **Programmatic integrations (CRM-style, long-lived scripts): `api_key`.** A + stable identifier that survives a rename is the point. A CRM sync addressing + `?tags[in]=urgent` should not break because someone retitled a property in the + UI. + +That is Option C with the default decided by surface rather than by caller — the +tool layer speaks names, the raw REST surface speaks keys, and each is the right +default for who is holding it. + +**One consequence to weigh if the API keeps `api_key`:** 22 of 514 option api +keys in a 77-space account are *not* derivable from the current name — they were +minted before a rename and kept the old spelling (`"Awareness"` → `discovery`, +`"Chat Management"` → `chat_managemetn`, typo included). So `api_key` is not a +pure function of the name, and any code assuming it can re-derive one is wrong on +about 4% of real options. That cuts both ways: it is the strongest argument that +`api_key` carries real information a name does not, and the strongest argument +that it is a second identity to keep in sync. + +## 6. Things that are useful whichever way you go + +- **`bundle.FoldApiKey`** — casefold + strip `_`/`-`. Exact match should always + win before the fold is consulted; two keys folding together is an ambiguity to + surface loudly, never to resolve by guess. +- **`MintApiSlug` / `MintApiSlugFromName`** (`pkg/lib/bundle/apislug.go`) — the + grammar-safe minting helpers. Added because the app was storing api keys that + no key route could accept: measured, 27 of 1,530 stored keys violated + `^[A-Za-z0-9_]+$`, including `lists_[in_work]`, `manual_export_&_import` and + `[?]_medium` — the last being `➡️ Medium` with the emoji unidecoded to a + literal `[?]`. All six app minting sites now route through these. +- **The API's key derivation is unchanged** and still splits on case and digit + boundaries (`Web3` → `web_3`, `P2P` → `p_2_p`). The format deliberately does + *not* — it has no derivation step at all: a name IS the key, NFC-normalized + and otherwise verbatim. If the API ever adopts name-derived keys, that + difference matters. +- **The compact filter grammar has no quoted-key form**, so a spelling no + identifier folds onto — `C++`, `50% done` — has no compact representation. + Recorded in SPEC §6.2.1. Relevant if the API exposes compact filters over + name-spelled properties. + +## 7. Where to read more + +- `pkg/lib/anyblockjson/SPEC.md` §3 — the authority on spelling and resolution. +- `pkg/lib/bundle/apislug.go` — the API's own slug machinery, unchanged. diff --git a/pkg/lib/anyblockjson/HANDOFF_AUTHORING_SCENARIOS.md b/pkg/lib/anyblockjson/HANDOFF_AUTHORING_SCENARIOS.md new file mode 100644 index 0000000000..a450eee6c9 --- /dev/null +++ b/pkg/lib/anyblockjson/HANDOFF_AUTHORING_SCENARIOS.md @@ -0,0 +1,391 @@ +# Handoff — authoring-mistake scenario catalogue + +**Status: specified, not started.** Nothing has been built yet. This document is +self-contained: it assumes no context from the session that produced it. + +Repo: `/Users/roman/anytype/anytype-heart_go-7383-anyblockjson` (a git worktree — +work only there, never `cd` to the main checkout). + +--- + +## 0. The one rule that matters most + +**Never run a state-changing git command.** No `checkout`, `restore`, `stash`, +`clean`, `reset`, `commit`, `add`. Read-only git (`git show`, `git log`, +`git diff`) is fine. + +This is not boilerplate. During the session that wrote this document, two +separate agents ran `git checkout` to "clean up after themselves" and destroyed +a large body of concurrent uncommitted work, twice. The worktree carries ~120 +modified files that are **intentional and uncommitted**. To remove a scratch +file, delete it by exact name (`rm path/to/file`). + +Also: add **no new module dependency**. CI runs `license_finder` against +`anyproto/open`'s `decisions.yml`, so a new dep blocks the build until someone +files a decision entry. Everything below is implementable with the standard +library. + +--- + +## 1. The goal + +AnyBlock JSON (`pkg/lib/anyblockjson`) is a JSON document format about to freeze +at v1. Its **primary author is an LLM agent** — the spec says so, and there is a +dedicated "authoring subset" (§2g, `schema/authoring/*.schema.json`, +`ValidateAuthoring`) built for exactly that consumer. + +The goal, in the project owner's words: *an agent cannot break things, and even +when it writes something invalid it must get the correct error to fix it from.* + +That decomposes into two testable properties: + +1. **Safety** — no input causes a panic, hang, OOM, or a *silent* bad import. +2. **Actionability** — every rejection produces an error an agent can recover + from unaided. This one is barely tested today and is what this work targets. + +Actionability decomposes further, and the spec already commits to most of it: + +- **One fault, one issue** (§12 promises this). Twelve issues for one typo makes + an agent flail. +- **Correctly path-addressed** — the JSON Pointer names the offending node. +- **The repair is named** — the spec says "with the repair named" repeatedly. +- **The repair actually works** — apply what the message says and the document + becomes valid. Nobody tests this today. It is the bar the owner chose. + +--- + +## 2. Evidence already gathered + +I probed `ValidateAuthoring` with plausible LLM mistakes against +`pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/morning-run.json`. +Reproduce by writing a throwaway `_test.go` in the package that mutates that +fixture and calls `ValidateAuthoring`. Findings, verbatim: + +**Error quality is inconsistent by layer.** The curated semantic rules are +excellent: + +``` +[/properties/createdDate] "createdDate" is a timestamp the app stamps — the app +derives it, so an author does not write it. Every spelling of that key is +refused here, not just this one +``` + +The raw schema refusals are not: + +``` +[/blocks/0/colour] property "colour" is not allowed +``` + +Same document, same author, completely different quality of help. The second is +not machine-applicable: it does not say to delete the member, and does not +suggest `color`, which is one edit away. + +**No "did you mean" anywhere.** `bulleted_list` (instead of +`bulleted_list_item`) produces a 21-value list to choose from. `colour` produces +nothing. Edit-distance suggestion against the closed vocabularies is likely the +single highest-value cheap improvement in this whole surface. + +**Document-level validation structurally cannot catch the most likely agent +mistake.** Both of these are **accepted silently**: + +- `"type": "Habitt"` (a typo'd type name) +- `"last_done"` instead of `"Last done"` (a snake_case slug where the display + name belongs) — and it imports as detail key `last_done`, i.e. a **new phantom + property**, not the one the author meant. + +This is not a bug in `Validate`. A single document carries no vocabulary, and §3 +says a term the resolution chain does not know passes through verbatim. +**Bundle-level checking does catch it** — `anyblockbatch.CheckPropertyFormats` +reports both the spelling and what it resolved to. + +**Consequence for the design: the test unit must be the bundle, not the +document.** The highest-value agent errors are only catchable there. + +**Silent normalizations exist by design** — e.g. a scalar where a multi-select +list belongs (`"Frequency": "Daily"`) is accepted and normalized. Correct +behavior, but the author gets no signal that what they wrote was not canonical. +Worth capturing as `must_warn` scenarios (probably currently `known_gap`). + +--- + +## 3. Plan + +| Stage | What | +|---|---| +| 1 | Build the harness: scenario file format, parser, JSON Patch engine, runner, 3 seed scenarios | +| 2 | Fan out scenario authoring across 10 fault families (parallel agents) | +| 3 | Adversarial verification of every scenario by a different agent | +| 4 | Completeness critic: which implemented refusals have no scenario | + +Stage 1 must finish and be proven before stage 2 — the scenario files are the +tests, so the format has to be real first. + +**The output is not a green suite. It is a punch list**: every scenario marked +`known_gap` is an authoring mistake that currently gets no error or an unusable +one, with a reproduction attached. That list is the deliverable, to be fixed +before the format freezes. + +--- + +## 4. Stage 1 — the harness + +### Where + +**`cmd/internal/anyblockbatch`.** That package describes itself as "the import +wiring", already imports `pkg/lib/anyblockjson`, and owns the bundle-wide +checks. Putting the harness there lets one runner validate at both the document +and bundle level without the format package importing upward (SPEC §13 forbids +the format package depending on import/export wiring). + +Scenario files: `cmd/internal/anyblockbatch/testdata/scenarios//.md` + +### The scenario file format + +Implement the parser to this exactly — stage-2 agents will write hundreds of +these against this spec. + +````markdown +--- +id: prop-slug-instead-of-name +category: property-spelling +source: SPEC §3 — a key's spelling is the entity's display name +surface: document +target: objects/morning-run.json +verdict: must_error +expect_path: /properties/last_done +expect_message: + - Last done +status: expected +--- + +Prose: what the author did, why it is a plausible mistake, what the correct +form is. One or two paragraphs. + +## Mutate + +```json +[{"op": "move", "from": "/properties/Last done", "path": "/properties/last_done"}] +``` + +## Repair + +```json +[{"op": "move", "from": "/properties/last_done", "path": "/properties/Last done"}] +``` +```` + +Field semantics: + +| Field | Meaning | +|---|---| +| `id` | kebab-case, unique across all scenarios, MUST equal the filename stem | +| `category` | free string, used for grouping in the summary | +| `source` | SPEC section or code symbol this is derived from. Required, non-empty | +| `surface` | `document` or `bundle` | +| `target` | path relative to the bundle root of the file to mutate | +| `verdict` | `must_error` \| `must_warn` \| `must_accept` | +| `expect_path` | JSON Pointer where the issue must appear. Required for `must_error`/`must_warn`, forbidden for `must_accept` | +| `expect_message` | list of substrings the message must contain (may be absent) | +| `status` | `expected` (behavior implemented) \| `known_gap` (documents a gap) | + +Sections: `## Mutate` (required) and `## Repair` (required unless verdict is +`must_accept`), each a fenced ```json block holding a JSON Patch array. + +Write a **minimal front-matter parser** — only scalars and lists of scalars are +needed; do not add a YAML dependency. Fail loudly with the filename on any +unknown key, missing required key, or malformed patch. A broken scenario must be +a loud failure, never a silent skip. + +### JSON Patch subset + +Implement `add`, `remove`, `replace`, `move` over `map[string]any` / `[]any`, +with RFC 6901 pointer parsing including `~0`/`~1` unescaping and `-` for array +append. **Unit-test the patch engine itself** — it is load-bearing for every +scenario, and a silently wrong patch makes a scenario pass for the wrong reason. + +### The base bundle + +`pkg/lib/anyblockjson/testdata/authoring/habit_tracker/`: + +``` +index.json +properties.json +types/habit.json +objects/start.json +objects/morning-run.json +objects/weekly-review.json +``` + +Load it fresh per scenario. **Never mutate it on disk.** For bundle-surface +scenarios, materialize the mutated bundle into `t.TempDir()` and run the checks +over real files. + +Note it is already covered by `TestAuthoringExample_HabitTracker` in +`pkg/lib/anyblockjson/authoring_test.go`, which asserts the pristine bundle is +valid, warning-free and internally coherent. Read that test first — it shows how +the bundle hangs together. + +### What the runner asserts + +1. Load the pristine bundle; apply `Mutate` to `target`. +2. Validate for the surface: + - **document** → `anyblockjson.ValidateAuthoring` for object and type files; + `ValidateAuthoringIndex` for `index.json`; `ValidateAuthoringPropertyDictionary` + for `properties.json`. Collect warnings separately via `ValidateWarn`. + - **bundle** → materialize to a temp dir and run the same check set + `cmd/anyblockvalidate/main.go` runs. Read that file; the set is + `ScanFormats`, `DictionaryFormats`, `MergeDictionaryFormats`, + `CheckPropertyFormats`, `CheckViewProperties`, `CheckIndexTargets`, + `CheckManifestFiles`, `CheckBundleIds`, `CheckSharedSelects`, + `UnboundFileDocuments`. Signatures are in + `cmd/internal/anyblockbatch/scan.go`. +3. Assert the verdict: + - `must_error` — a `*anyblockjson.ValidationError` containing an issue at + `expect_path` whose `Message` contains **every** `expect_message` substring. + - `must_warn` — no error, and a warning at `expect_path` matching likewise. + - `must_accept` — no error and no warnings. +4. **Repair convergence** (skip only for `must_accept`): apply `Repair` to the + *mutated* document; the result must validate with **no error and no + warnings**. This is the strict bar — it proves the stated repair actually + recovers the document. +5. **Known-gap inversion.** `status: known_gap` means the behavior is not + implemented. Run the same checks; if they now **pass**, FAIL with a message + telling the author to change `status` to `expected` — so closing a gap forces + the catalogue to update. If they still fail, record and `t.Log`; do **not** + fail the suite. + +### Gotcha that will bite you + +`ValidateAuthoring` runs the full validation first, then the subset schema. When +a document is valid AnyBlock JSON but outside the authoring subset, +`validateAuthoringSubset` emits a **preamble issue at the document root** *plus* +the real refusal. So "exactly one issue" is wrong as an assertion — the runner +must tolerate extra issues and look for the one at `expect_path`. + +Observed shape: + +``` +issues: 2 + - [] valid AnyBlock JSON, but outside the authoring subset — … + - [/properties/createdDate] "createdDate" is a timestamp the app stamps — … +``` + +### Deliverables + +1. `cmd/internal/anyblockbatch/scenario.go` — parser + patch engine. +2. `cmd/internal/anyblockbatch/scenario_test.go` — unit tests for both, + including malformed input. +3. `cmd/internal/anyblockbatch/authoring_scenarios_test.go` — + `TestAuthoringScenarios`, walking the scenario directory. +4. **Three seed scenarios** proving all three verdicts and both surfaces: + - `must_error`, document: writing stored key `createdDate` into `properties` + is refused at `/properties/createdDate`, message contains + `the app derives it`. (Verified — this is the preamble case above.) + - `known_gap`, document: `"type": "Habitt"` in `objects/morning-run.json` is + currently accepted silently; it *should* error naming the unknown type. + Write as `verdict: must_error`, `status: known_gap`. (Verified accepted.) + - one `bundle`-surface scenario genuinely exercising the bundle checks — e.g. + a widget target in `index.json` naming an id no file declares, or a + property spelling used in an object but declared nowhere. + +### Summary output + +At the end print, to test output: per category, counts of `ok` / `known_gap` / +`broken`; then the **full list of known gaps** with id and one-line prose. That +list is the punch list this exercise exists to produce — make it easy to read +and easy to paste into an issue. + +--- + +## 5. Stage 2 — the fault families + +One agent per family. Each reads its spec sections **and the code implementing +the refusals**, writes scenarios, and self-verifies by running the harness. +Every scenario must cite its `source`. + +| Family | Derived from | +|---|---| +| `property-spelling` | §3 — slug vs display name, stored key, typo, case, NFC/NFD, edge whitespace, a name that is another entity's stored key | +| `denied-keys` | §2b and §3 deny rules; `omittedrelation.go`, `systemtrim.go` — icon/cover flat spellings, `internal_key`, transient/derived keys | +| `block-structure` | §4, §5, §6.1, §7, §7a — unknown type, indent bounds and jumps, missing required members, table row/column/cell rules, transparent containers | +| `enum-values` | every closed vocabulary: `blockvocab.go`, `viewvocab.go`, `json.go` — block type, align, layout, view type, filter condition, sort direction, date preset, relation format, icon/cover format, card style, embed processor, option color | +| `dataview` | §6.2, `dataview.go`, `filters.go` — filter naming an unknown property, bad condition/format pairing, missing day-count operand, `group_by`, duplicate view ids, columns referencing undeclared properties | +| `type-document` | §2a, §2e — `type_settings` shape, `property_definitions`, `template_for` misuse, `kind` mismatches, root `type_properties` | +| `bundle-coherence` **(bundle surface)** | §2c and the `anyblockbatch` checks — undeclared property spelling, dangling widget/link target, missing manifest file, reserved id, duplicate id, unbound file document, shared select conflict | +| `inline-markup` | §8.1–§8.4 — malformed mention, unbalanced marks, bad escape, bad link destination, unknown tag, astral-plane text | +| `property-values` | §3 — scalar vs list, wrong JSON type per format, malformed RFC 3339 date, unknown option, null, number precision | +| `subset-boundary` | §2g, §4a — export-only members (`store`, `root`, `fields`, `source`, `object_orders`) in an authoring document | + +Guidance for family agents: + +- Prefer **plausible** faults over exotic ones. The target is what an LLM + actually emits: a slug where a name belongs, a near-miss enum, a scalar where + a list belongs, a member at the wrong nesting level, a stored key copied from + API docs. Random byte corruption is a different job (see §7). +- A scenario whose fault is currently **accepted** is not a failure to report — + write it as `known_gap`. Those are the most valuable entries in the catalogue. +- Every `Repair` must be a genuine inverse: applying it must return the document + to validity, not merely delete the offending member if deletion loses the + author's intent. + +--- + +## 6. Stages 3 and 4 + +**Stage 3 — adversarial verification.** A scenario can be wrong in ways its +author will not see: the mutation does not produce the fault the prose claims; +the repair is not a true inverse; `expect_path` is off by a level; the assertion +passes for the wrong reason (e.g. matching the preamble issue instead of the +real one). Have a *different* agent re-run each family's scenarios and classify +them. Treat a scenario that passes for the wrong reason as broken. + +**Stage 4 — completeness critic.** Enumerate every refusal the code actually +implements — `addIssue` call sites in `validate.go`, the closed `enum`s in +`schema/*.json`, and the `anyblockbatch` check functions — and report which have +**no scenario**. The uncovered remainder is itself a finding, and it is how we +know the catalogue is honest rather than merely large. + +--- + +## 7. Explicitly out of scope for this phase + +Deferred deliberately by the project owner, to a later stage: + +- **Round-trip / conversion testing** — mutating the protobuf corpus, the §11 + byte fixpoint, and `snapshotdiff.Compare` snapshot-equivalence. That is the + *export* direction and belongs to phase 2. +- **Coverage-guided byte fuzzing** for crash/hang/OOM safety. Valuable, but a + different oracle; random bytes die at the JSON parse layer and teach nothing + about error quality. A separate research document, + `pkg/lib/anyblockjson/HANDOFF_FUZZING_RESEARCH.md`, covers this in depth if it is wanted. + +--- + +## 8. Verify before reporting + +``` +go build ./... +go vet ./cmd/... +go test ./cmd/internal/anyblockbatch/... +``` + +Report the verbatim output including the summary table. Do not claim green +without running it. + +If a seed scenario does not behave as §4 describes, **say so plainly and show +what actually happened** — the probe behind this document may have been wrong, +and a correction is more useful than a workaround. + +--- + +## 9. Two findings likely to dominate the punch list + +Worth knowing up front, because they will recur across families: + +1. **"Did you mean" is absent everywhere.** Edit-distance suggestion against the + closed vocabularies is a small, contained change with an outsized effect on + whether an agent recovers unaided. +2. **The two error layers are inconsistent.** Curated §12 wording is genuinely + good; raw schema refusals are bare (`property "colour" is not allowed`). The + catalogue will show exactly which faults fall through to the bare layer — + that is the list of places to add curated wording before the freeze. diff --git a/pkg/lib/anyblockjson/HANDOFF_FUZZING_RESEARCH.md b/pkg/lib/anyblockjson/HANDOFF_FUZZING_RESEARCH.md new file mode 100644 index 0000000000..8c7f58b17d --- /dev/null +++ b/pkg/lib/anyblockjson/HANDOFF_FUZZING_RESEARCH.md @@ -0,0 +1,679 @@ +# Fuzzing AnyBlock JSON — options, verified + +Scope: `pkg/lib/anyblockjson` at v1 freeze. Read: SPEC.md §11 (round-trip), §12 +(validation), §13 (API), `snapshotdiff/`, `roundtrip_test.go`, +`markdownblocks_test.go` (the one existing fuzz target), `cmd/anyblockroundtrip`, +`.github/workflows/test.yml`. + +--- + +## 0. Verified library facts (checked 2026-08-29) + +| Thing | Verified status | Verdict | +|---|---|---| +| Go native fuzzing (`go test -fuzz`) | Go 1.25.7 declared in go.mod, toolchain 1.26.5 locally. `f.Fuzz` args limited to `string, []byte, int*, uint*, float*, bool` (go.dev/doc/security/fuzz). Seeds from `f.Add` + `testdata/fuzz//`; generated corpus in `$GOCACHE/fuzz`. Automatic minimization; failing input written to `testdata/fuzz/`. | **Use it. Zero deps.** | +| Go fuzz per-input hang detection | **Verified in local GOROOT source** `src/internal/fuzz/worker.go:492`: `time.AfterFunc(10*time.Second, func(){ panic("deadlocked!") })` per input. The `workerTimeoutDuration = 1s` constant is *worker shutdown*, not per-input — the go.dev doc's "1 second per execution" is wrong/misleading. | 10 s hard watchdog. Too coarse — add your own budget. | +| Go fuzz OOM detection | None. `GOMEMLIMIT` only makes GC work harder. | Must hand-write a memory oracle. | +| `pgregory.net/rapid` | v1.3.0, published **2026-03-30**; repo `flyingmutant/rapid` last code push **2026-04-30**; 870★; **MPL-2.0**. Has `rapid.MakeFuzz` bridging any rapid test into `testing.F`. Reflection `Make[T]`/`MakeCustom` with `MakeConfig{Types, Kinds, Fields}`. | **Maintained. Best PBT lib for Go.** License needs a `license_finder` decision (see §5). | +| `rapid` on interface fields | **Verified in `make.go`**: no `reflect.Interface` case; falls to default and **panics** `"unsupported type kind for Make: %v"`. `MakeConfig.Types[ifaceType]` is checked *before* the kind switch, so an interface key is a working escape hatch. | Loud failure, not silent. Good. | +| `AdaLogics/go-fuzz-headers` | Last **code push 2024-08-06** (≈2 y stale); no tagged release, pseudo-version only; 111★; Apache-2.0. | **Dormant.** | +| `go-fuzz-headers` `AdaptArbitrary` | **Does not exist.** Full exported index: `NewConsumer`, `GenerateStruct`, `GenerateWithCustom`, `AddFuncs`, `GetString/Int/Bytes/…`, `FuzzMap`, `TarBytes`, `CreateFiles`. The brief's premise is wrong. | Correct the assumption. | +| `go-fuzz-headers` on interface fields | **Verified in `consumer.go`**: `fuzzStruct`'s kind switch has **no `reflect.Interface` case**; an interface field falls through and is left **nil, with no error**. | **Disqualifying** for this format — see §1. | +| `google/gofuzz` | **ARCHIVED**; last push 2022-11-07. | **Rule out.** | +| `leanovate/gopter` | Last push 2026-04-20; 637★; **MIT**. | Maintained fallback if MPL blocks rapid. Weaker than rapid. | +| `thepudds/fzgen` | Last **code push 2024-07-23**; 116★; Apache-2.0; author's own README still says "work in progress… approaching beta quality". | **Dormant + self-declared pre-beta.** Skip. | +| `santhosh-tekuri/jsonschema/v6` | **Already a direct dependency** (v6.0.2), used by `index.go` and `authoring.go`. Draft 2020-12. | Schema oracle is free — no new dep. | +| OSS-Fuzz | FAQ: **"My project is not open source. Can I use OSS-Fuzz?" → no.** Acceptance: "significant user base and/or critical to global IT infrastructure", case-by-case, weighted on "exposure to remote attacks" and dependent-project count. | anytype-heart ships under **Any Source Available License 1.0** — source-available, *not* OSI open source. **Do not pursue.** | +| ClusterFuzzLite | `google/clusterfuzzlite`, last push 2026-02-12, not archived, Apache-2.0, 535★. GitHub Actions / GitLab / Cloud Build / Prow; Go supported. | The realistic "OSS-Fuzz-grade" option — but see §5 for why a cron `go test -fuzz` beats it here. | +| `AdamKorcz/go-118-fuzz-build` | Last push 2025-12-22, active, Apache-2.0, 31★. (The shim that makes `testing.F` targets run under libFuzzer, needed for OSS-Fuzz/CFL.) | Only relevant if you pick CFL. | +| `hypothesis-jsonschema` | Last push 2025-12-05, active, MPL-2.0, 280★. Generates instances from a JSON Schema (Python/Hypothesis). | The only credible off-the-shelf schema-driven generator. Cross-language. See Option 4. | + +**Could not verify:** whether `go-fuzz-headers`' `AddFuncs`/`Funcs` map actually +dispatches on an *interface* `reflect.Type` key (the source excerpt I read did not +show the lookup site). Moot — see §1. + +--- + +## 1. The gating problem: where do snapshots come from + +This is the crux and deserves the most space. Everything in the Marshal direction +(oracles I1, byte-fixpoint, snapshot-equivalence) needs a supply of +`*model.SmartBlockSnapshotBase` values. + +### Why the reflection generators fail here + +Two fields carry all the information in the format, and **both are gogo-protobuf +oneofs, i.e. unexported Go interfaces**: + +- `model.Block.Content` — `isBlock_Content`, **18 variants** (`BlockContentOfText`, + `…OfFile`, `…OfDataview`, `…OfTable`, `…OfLink`, `…OfLatex`, `…OfWidget`, …). +- `types.Value.Kind` — `isValue_Kind`, 6 variants (null/number/string/bool/struct/list), + and `SmartBlockSnapshotBase.Details` is a `*types.Struct` of them. + +Consequences, verified against each library's source: + +- **`go-fuzz-headers.GenerateStruct` leaves both nil, silently.** You would get + a fuzzer that generates blocks with no content and details with no values, + explore roughly nothing, and never be told. This is the worst failure mode + available — a green fuzzer that isn't testing anything. +- **`rapid.MakeCustom` panics** `unsupported type kind for Make` — you find out + immediately, and `MakeConfig.Types[]` (obtainable via + `reflect.TypeOf(model.Block{}).FieldByName("Content").Type`, since the + interface itself is unexported) is a documented override point. + +So: **whichever library you pick, you hand-write the oneof dispatch anyway.** +~18 block-content constructors + 6 value kinds + the dataview/table sub-shapes. +Realistically 300–500 lines of generator. The library buys you the *plumbing +around* that code, not the code itself. Judge the strategies accordingly. + +### The four strategies, compared + +**(a) Mutate the real protobuf corpus — recommended, and it needs zero generator code.** + +`cmd/anyblockroundtrip` already writes every exported object as a +`pb.SnapshotWithType` protobuf binary (`main.go:537` reads `.pb` files with +`proto.Unmarshal`, `:699` writes `original.pb`). So the corpus already exists as +a by-product of the tool the team already runs. + +The fuzz target is then, in full: + +```go +func FuzzSnapshotRoundTrip(f *testing.F) { + // seeds: every .pb from a sweep run, plus goldens re-serialized + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 1<<16 { return } + var sw pb.SnapshotWithType + if proto.Unmarshal(data, &sw) != nil { return } // the structure filter + base := sw.Snapshot.GetData() + if base == nil { return } + ... Marshal → Validate (I1) → Unmarshal → Marshal → byte-compare + }) +} +``` + +Why this is the best defect-per-effort here: + +- Protobuf binary is **unusually mutation-friendly**: field tags are varints, most + single-byte flips land on a valid wire type, and length-prefixed submessages + survive splices. A high fraction of mutants decode. Contrast random bytes into + a JSON parser, where almost nothing survives. +- `proto.Unmarshal` is a *free, exact* structural filter. You get well-formed + snapshots with hostile *contents* — which is exactly what you want, because + the snapshot's contents are documented as untrusted (§11: "The snapshot's + block graph is untrusted"). +- It reaches shapes a hand generator would never think of: invalid UTF-8 in + strings (gogo's generated `Unmarshal` does not validate UTF-8 — there is + already an `invalidutf8_test.go`), out-of-range enums, cyclic `ChildrenIds`, + duplicate block ids, oversized keys. +- **`BlockContentDataviewFilter.NestedFilters []*BlockContentDataviewFilter` is + recursive in the proto** (models.pb.go:5018). Self-splicing a filter submessage + doubles nesting depth per mutation — the deep-filter-tree defect is reachable + *from this direction too*, geometrically fast. +- Coverage guidance does the corpus curation for you: the engine keeps mutants + that reach new branches in `export.go`/`validate.go`. + +Hazards, and the mitigations: + +- That same proto recursion means `proto.Unmarshal` itself can recurse deeply on + hostile input. Go grows stacks to 1 GB then takes an **unrecoverable** fatal + error. Cap input length (`if len(data) > 1<<16 { return }`, the pattern + `FuzzMarkdownImports` already uses) — at ~3 bytes per nesting level that caps + depth around 20 k frames, which is fine. +- Mutants are not distributionally realistic. That is a feature for exact oracles + and a **liability for `snapshotdiff.Compare`** — see §2, Option 1. + +**(b) Hand-written `rapid` generators — the right *second* step, not the first.** + +You write the 18-way content generator and the value-kind generator, compose +snapshots from them, and drive with `rapid.Check` / `rapid.MakeFuzz`. What this +buys over (a): + +- **Shrinking of a *snapshot*, not of bytes.** When a mutated-proto target fails, + Go minimizes the *byte string*; the minimized bytes still decode to a snapshot + you have to read by hand. rapid shrinks the *structure* — "a document with one + paragraph whose text is `""`". For a format whose failures are structural, + this is a real ergonomic difference. +- **Controlled distribution**, which is what makes the `snapshotdiff` oracle + usable: you know what went in, so a finding is either a real loss or a + normalization you need to admit. +- Reaches valid-but-rare combinations the corpus simply does not contain (a + template with a `template_for`, a type document with all four recommended roles + empty, an `iconOption` of exactly 1 with no other icon channel). + +Cost: the 300–500 lines above, plus MPL-2.0 license paperwork (§5). + +**(c) Derive snapshots from fuzzer bytes with a hand-written byte cursor.** +i.e. your own 60-line `type source struct{ b []byte }` with `next() byte`, +`nextString()`, feeding the same hand-written generator as (b). Middle ground: +coverage-guided like (a), structured like (b), no new dependency, no MPL question. +Worth knowing about, but it duplicates what (a) gets for free from +`proto.Unmarshal`. **Only build this if coverage shows (a) can't reach something.** + +**(d) Reflection-based struct filling (`go-fuzz-headers`, `rapid.Make`, `gofuzz`).** +Rejected above. `gofuzz` is archived; `go-fuzz-headers` silently nils the oneofs; +`rapid.Make` panics without a hand-written override — at which point you are in (b). + +### Verdict + +> **Corpus mutation beats generation here, decisively, and by more than usual — +> because the corpus is already produced by an existing tool, and because the +> transport (protobuf binary) happens to be one of the most mutation-friendly +> formats there is.** Start at (a). Add (b) in month two, specifically to unlock +> the `snapshotdiff` oracle and structural shrinking. Never build (d). + +--- + +## 2. The options + +Ordered by what I would do first. + +--- + +### Option 1 — Native `go test -fuzz`, corpus-seeded, with explicit invariant *and resource* oracles + +**What it is.** Three (eventually five) `testing.F` targets in +`pkg/lib/anyblockjson`, no new dependencies. Two of them take document bytes, one +takes protobuf bytes (§1a). The novel part is not the fuzzer, it is that each +target asserts *typed invariants and a resource budget*, not "did it panic". + +The resource oracle is the piece that matters most and that nobody ships for you: + +```go +func budgeted(t *testing.T, in []byte, f func()) { + var s [1]metrics.Sample + s[0].Name = "/gc/heap/allocs:bytes" // cheap, no stop-the-world + metrics.Read(s[:]); before := s[0].Value.Uint64() + start := time.Now() + f() + metrics.Read(s[:]); alloc := s[0].Value.Uint64() - before + // amplification, not absolute: bytes allocated per input byte + if alloc > 32<<20 && alloc > 2000*uint64(len(in)) { + t.Fatalf("alloc amplification: %d bytes in, %d allocated", len(in), alloc) + } + if d := time.Since(start); d > 500*time.Millisecond { + t.Fatalf("slow input: %d bytes in, %s", len(in), d) + } +} +``` + +Use **allocated bytes as the primary budget and wall time as a loose secondary** — +allocation counts are near-deterministic and machine-independent; wall time on a +GitHub `macos-15` runner is not, and a tight time bound is the single biggest +flake source available. (Go's own 10 s watchdog stays as a backstop.) + +**Directions and oracles.** + +| Target | Direction | Oracles | +|---|---|---| +| `FuzzDocumentBytes` | bytes → Validate/Unmarshal | **I2** (`Validate(d)==nil` ⟺ `Unmarshal(d,bare)==nil`), crash, **resource**, and on success **I1** + **byte fixpoint** | +| `FuzzSnapshotRoundTrip` | snapshot → Marshal → … | **I1**, **byte fixpoint** (§11.3), crash, **resource** | +| `FuzzFragmentAgreement` | both | **cross-surface agreement** (oracle 3), I2 | +| later: `FuzzIndex`, `FuzzPropertyDictionary` | bytes | crash, resource, schema conformance | + +I2 caveat, and it is real: **`Validate` takes no `Options`; `Unmarshal` does.** +Run I2 with `Options{GenerateId: seqIds("f")}` and nothing else wired. With +resolvers wired, §3's resolver-dependent rejections (ambiguous spelling with no +`ScopedKeyVocabulary`) make disagreement *legal* and the oracle becomes noise. + +Schema conformance is a fourth oracle and costs one line, because +`santhosh-tekuri/jsonschema/v6` is already a dependency: after every successful +`Marshal`, run the output through the compiled `object.schema.json`. Note this is +*weaker* than I1 (Validate = schema **plus** ~15 semantic rules §12), so it earns +its place mainly on the `authoring/` subset, where the subset schema is a +separate artifact that can drift from `ValidateAuthoring`. + +**Benchmark defects it would plausibly have caught.** + +- 2500×2500 table, 15 KB → 635 MB / 34 s — **YES, high confidence.** Repeating a + `{"id":"c1"}` entry inside a `columns` array is a bread-and-butter splice + mutation from a table seed, and the alloc-amplification bound fires long before + 635 MB. The 10 s watchdog would also catch the 34 s version. +- Deep filter trees, 382 KB → 9.1 s / 3.7 GB — **YES.** Note 9.1 s is *under* Go's + 10 s watchdog, so crash-only fuzzing would have **missed** it; the alloc bound + catches it at a fraction of the depth. Reachable from both directions (the + proto `NestedFilters` recursion). +- `checkNumbers` pointer rebuild, 703 KB → 12.2 s / 4.9 GB — **YES.** Quadratic in + node count; a byte fuzzer grows documents readily. Caught by the alloc bound + early, by the watchdog late. +- 1 000 000-char property key / raw NUL, CR, ESC — **YES, via I1 from the snapshot + direction, and this is the nicest hit in the list.** The schema bounds property + names at `maxLength: 128` (mirrored at `validate.go:1625`, `maxPropertyKeyLen`). + Mutate a detail key in the proto corpus to 200 chars → `Marshal` emits it → + `Validate` rejects it → **I1 violated**, reported with the exact key. No + expected output needed. +- Duplicate JSON keys, last-wins — **NO. Honestly, none of these options finds it.** + `encoding/json` last-wins is deterministic and both `Validate` and `Unmarshal` + do it identically, so I2 is silent; the document round-trips byte-stably. It + needs a purpose-built duplicate-key scan over the token stream. Worth writing as + a plain unit check; it is not a fuzzing problem. +- NFC/NFD equivalent keys — **MEDIUM, and only with the right seed.** Go's mutator + is byte-level; it will not synthesize a valid NFD sequence by chance. Put a + seed containing both forms of `é` in `testdata/fuzz/`, add the postcondition "no + two keys in a `Marshal` output are NFC-equal but byte-distinct", and duplication + mutations find it. General lesson: **a byte fuzzer finds what the seed corpus + makes reachable.** +- Fragment surface dropping unknown members — **YES**, but by Option 3's oracle; + see there. +- No decompression-ratio bound in the archive chain — **NO, and not in this + package.** That code is `core/block/import/common/source/zip.go`. It wants its + own ~30-line `FuzzZipSource` target with an output-size budget, which would + find it in minutes. Separate ticket, high value, low effort. + +**Integration effort.** ~250 lines total for the three targets plus the +`budgeted` helper. The hard parts, in order: + +1. **Determinism.** Fuzz targets must be deterministic or the engine reports + phantom crashers. `Options.GenerateId` must be the counter (`seqIds`, already + in `roundtrip_test.go:54`). Guard against map-iteration leakage by running each + input twice and comparing outputs for the first week — `json.go` is described + as an ordered canonical writer, so this should hold, but verify rather than + assume. +2. **Seed conversion.** `testdata/fuzz//` files must be in Go's corpus text + format, not raw JSON. Avoid the converter entirely: `//go:embed testdata/*.json` + and `f.Add(...)` in the target, so the goldens seed themselves and stay in sync. +3. **Picking the amplification constants.** Measure the existing corpus first, set + the bound ~10× above the observed p99, tighten later. + +**CI story.** Two distinct things, and conflating them is the usual mistake: + +- **Regression (every PR, free).** Files under `testdata/fuzz//` run as + ordinary unit tests under plain `go test`. The existing `test.yml` job picks + them up with no change at all. Every crasher the engine finds becomes a + permanent regression test the moment you commit it. +- **Discovery (nightly).** A new scheduled workflow: `go test -run=XXX + -fuzz=FuzzDocumentBytes -fuzztime=20m ./pkg/lib/anyblockjson/`, one step per + target, `actions/cache` on `$GOCACHE/fuzz` keyed per target so the generated + corpus carries over between nights. On failure the engine writes the minimized + input to `testdata/fuzz/` — upload it as an artifact and open an issue; a human + commits it. `nightly.yml` already exists as the cron pattern to copy. + Note `test.yml` runs on `macos-15` runners; fuzzing is CPU-bound and Linux + runners are cheaper and faster — use `ubuntu-latest` for the fuzz job. +- **Flake risk: low**, if you use allocation budgets rather than wall-clock, and + keep `snapshotdiff` off the fuzz path (see below). + +**Ongoing cost.** Low. No dependency to track. Decay mode is the usual one: +targets rot when the API changes, and nobody notices the nightly job going red. +Mitigate by making the nightly failure open a Linear issue rather than an email. + +--- + +### Option 2 — `pgregory.net/rapid` generators for the snapshot direction + +**What it is.** v1.3.0 (2026-03-30), MPL-2.0, actively maintained, 870★, no +dependencies outside stdlib. Hand-written generators over the block/property model +(§1b), driven either by `rapid.Check` in a normal test or by `rapid.MakeFuzz` +inside a `testing.F` so the same property runs under coverage guidance. + +**Directions and oracles.** Snapshot direction only, but it is the *only* option +that can safely drive the strongest oracle: + +- **I1**, **byte fixpoint** — same as Option 1, redundantly. +- **Snapshot equivalence (`snapshotdiff.Compare`)** — this is the one that needs a + controlled distribution, and it is the reason this option exists. + +**Why `snapshotdiff` must not run on mutated input.** `Compare` is calibrated +against real accounts and consults the format's own exported predicates +(`DroppedMissingObjectRef` at `refs.go:182`, `DroppedDeletedIconRef` at +`refs.go:119`, `DroppedTypeProvenanceKey` at `typesettings.go:151`, +`OmittedBundledRelation`/`RelationInstallArtifactKey`/`InstallStampedDefault` in +`omittedrelation.go`). The package's own comments record that an *unadmitted* +normalization once produced **1 344 false failures in a single sweep**. Feed it +adversarial mutants and you will manufacture that situation on purpose, every +night. So: + +> **Exact oracles (I1, I2, byte fixpoint, resource, crash) on mutated input. +> Judgement-laden oracles (`snapshotdiff.Compare`) on controlled input only.** + +That line is the single most important design decision in this whole report. + +**Benchmark defects.** Adds little over Option 1 on the listed eight — the +resource defects are byte-side, and rapid biases toward *small* values, so it +would need an explicit "occasionally 5 000 columns" generator to find the table +blow-up. Its real value is the class of defect **not** on the list: silent field +loss that is byte-stable in both directions and therefore invisible to the +fixpoint oracle. That is precisely what the steer is pointing at, and only +`snapshotdiff` sees it. + +**Integration effort.** The big one: 300–500 lines of generator (18 block-content +variants, 6 value kinds, dataview views/filters/sorts, table column/row/cell, +envelope kinds incl. `object_type` and `property`). Hard parts: + +1. The oneof override. `MakeConfig.Types` keyed on + `reflect.TypeOf(model.Block{}).FieldByName("Content").Type` — the interface is + unexported, so reflection is the only way to name it. Verified as a supported + path (`Types` is consulted before the kind switch), but I have not run it. +2. Keeping the generator's *legality* aligned with the spec as the spec moves — + this is the decay mode, and it is worse than Option 1's. +3. Deciding which `Compare` findings are new normalizations vs. real loss. This is + human work per finding, permanently. + +**CI story.** Runs as a plain unit test (`rapid.Check`) on every PR in a few +seconds, and as a `-fuzz` target nightly via `rapid.MakeFuzz`. rapid does its own +shrinking and prints a reproducible seed. Flake risk: **moderate** — every new +legal normalization added to the format shows up as a red build until admitted. +That is arguably correct behaviour, but budget for it. + +**Ongoing cost.** Highest of the options. One owner, and the generator must be +updated in the same commit as any format change — the same "same-commit +discipline" the package already applies to the `Dropped*` predicates. + +**License note.** MPL-2.0 test-only dependency. `test.yml` runs `license_finder` +against `anyproto/open`'s `decisions.yml` — **this will need a decision entry +before the build goes green.** If that is a fight you don't want, `gopter` (MIT, +maintained, last push 2026-04-20) is the fallback, or drive the same hand-written +generator from a byte cursor (§1c) and add no dependency at all. rapid wins on +merit — generics, better shrinking, the `MakeFuzz` bridge — but the margin over a +hand-rolled byte cursor is smaller here than usual, precisely because you are +hand-writing the generators either way. + +--- + +### Option 3 — Cross-surface differential / metamorphic targets + +**What it is.** Not a tool — a family of ~40-line assertions where the *expected +output is another code path in the same package*. No dependency, no generator, no +oracle to design. + +The relations available: + +1. **I2**: `Validate(d) == nil` ⟺ `Unmarshal(d, bare) == nil`. Spec §12 states it + flatly: *"Validate and Unmarshal agree, in both directions."* +2. **Fragment vs. whole document**: the same block run through + `UnmarshalBlocks(run, opts)` vs. embedded in a `{"version":2,"blocks":[…]}` + envelope and run through `Unmarshal`; the same filter tree through + `UnmarshalFilters` vs. inside a dataview block; the same subtree through + `MarshalBlockSubtree` vs. cut out of a full `Marshal`. +3. **Round-trip through the fragment surface**: `MarshalBlockSubtree ∘ + UnmarshalBlocks` fixpoint. +4. **Authoring subset**: `ValidateAuthoring(d) == nil` ⟹ `Validate(d) == nil` + (the subset is documented as *strict*, so one-way implication is the relation). +5. **`filterstring` vs. the JSON filter tree** — there is already a + `filterstring_agreement_test.go`, so the pattern is established in the package. + +**Benchmark defects.** Catches the fragment-drop defect — +`UnmarshalFilters`/`UnmarshalSorts` silently dropping unknown members while the +whole-document path refused them — **directly, and it is the only option that +does.** That defect has already occurred once, which makes this the only oracle in +the report with a *demonstrated* hit rate on this codebase. It catches nothing +else on the list. + +**Integration effort.** Lowest by a wide margin. ~40 lines per relation. The hard +part is stating each relation precisely enough that legal differences don't +register — e.g. the fragment surface mints ids where the document path may not, so +compare after id-normalization. + +**CI story.** These are ordinary table tests *and* fuzz targets — write them as +`testing.F` so they get both. Zero flake risk once the relations are stated +correctly. No corpus management. + +**Ongoing cost.** Near zero. Each new fragment entry point added to +`fragment.go`/`filters.go` should come with its agreement relation; that is a +review-checklist item, not a maintenance burden. + +**Verdict on the framing question "does metamorphic/differential deserve its own +option?" — yes, emphatically.** It is the highest defect-per-line item here, it +needs no generator, and the format's own spec hands you the relations. It is not +a substitute for Option 1 (it finds no resource defects) but it should ship in +the same week. + +--- + +### Option 4 — Schema-driven generation from the published JSON Schemas + +**What it is.** Use `schema/object.schema.json`, `index.schema.json`, +`properties.schema.json` and `schema/authoring/*` as a *grammar*: generate +conforming instances, then mutate them in targeted ways (drop a required member, +violate an `enum`, exceed a `maxLength`, swap a discriminator `type`). + +Tooling reality: + +- **Nothing off-the-shelf does this in Go.** I found no maintained Go + schema→instance generator. `santhosh-tekuri/jsonschema/v6` validates; it does + not generate. +- The credible off-the-shelf generator is **`hypothesis-jsonschema`** (Python, + MPL-2.0, active as of 2025-12), driven against a small Go CLI harness that reads + a document on stdin and prints the verdicts of `Validate`/`Unmarshal`/`Marshal`. + That is the only genuinely *language-agnostic* option in this report. +- A hand-written Go schema walker is ~200 lines for this schema and gives you + in-process speed and no Python in CI. + +**A property of *this* schema that matters.** §12 states the block definition is +**deliberately non-recursive** (no `children`; table cells use a separate +`cellBlock` definition to cut the block↔cell cycle), and the *only* recursive +definition left is the dataview filter tree. Non-recursive schemas are exactly the +ones generators terminate on cleanly. So schema-driven generation is more +tractable here than for a typical document format — and the one recursion it has +is the very place the deep-nesting defect lives. + +**Directions and oracles.** + +- Conforming instances → **I1's contrapositive is not available** (schema-valid ≠ + `Validate`-valid, since `Validate` = schema + ~15 semantic rules), so you cannot + assert "generated ⟹ accepted". What you *can* assert: schema-invalid ⟹ + `Validate` rejects (the schema is a subset of the rules), and every accepted + document must round-trip. +- Deliberately non-conforming instances → **the error-quality oracle**: exactly one + fault should produce exactly one issue, at the right JSON pointer. §12 makes a + point of this ("`oneOf` reported 10 issues for one wrong member and never named + the alternatives; `if`/`then` reports one and does"). A generator that injects + *one* schema violation at a known pointer and asserts the reported pointer + matches is a genuinely good, and genuinely unusual, test — and directly serves + the LLM-producer consumer the spec calls out. +- **Coverage seeding.** The best pragmatic use: generate a few thousand conforming + instances once, keep the ones that add coverage, commit those as + `testdata/fuzz/` seeds for Option 1. Schema-driven generation as a *corpus + bootstrapper*, not as the running engine. + +**Benchmark defects.** Weak on the list. It reaches the table and filter shapes +only if you explicitly tell the generator to emit large arrays and deep nesting — +the schema has 21 `maxItems`/`maxLength` occurrences and (per the brief) no bound +on grid size, so an unguided generator emits *small* instances. It finds none of +the resource defects on its own; it would not find the fragment disagreement (it +generates whole documents); it would not find the duplicate-key or NFC issues. + +**Integration effort.** Medium-high, and cross-language if you take +`hypothesis-jsonschema`: a Go CLI harness (~100 lines), a Python driver, a Python +toolchain in CI. The Go walker avoids that at the cost of writing and maintaining +a generator for draft 2020-12 features you actually use (`if`/`then`, `$ref`, +`propertyNames`, `const` discriminators). + +**CI story.** Awkward. Property-based Python in a Go repo's CI is a maintenance +liability, and the schemas move as the format moves. Best run as a one-off +corpus-bootstrapping exercise, and as a permanent *small* Go-side test for the +error-quality oracle. + +**Ongoing cost.** Medium, and it decays badly: the generator must track schema +edits, and nobody will remember it exists. + +**Verdict: do the cheap 20 % of it.** The error-quality single-violation test in +Go, yes. Full schema-driven generation as the main engine, no. + +--- + +### Option 5 — Continuous fuzzing infrastructure (OSS-Fuzz / ClusterFuzzLite) + +**OSS-Fuzz: not realistic. Two independent blockers, both verified.** + +1. The FAQ answers *"My project is not open source. Can I use OSS-Fuzz?"* with a + flat no. `anytype-heart` ships under **Any Source Available License 1.0** — + source-available, not OSI open source. This alone likely ends it. +2. Acceptance requires *"a significant user base and/or be critical to the global + IT infrastructure"*, weighted on remote-attack exposure and dependent-project + count. A document codec inside one desktop app is a hard sell even setting + licensing aside. + +Integration cost, had it applied: a `projects/anytype-heart/` directory upstream +(Dockerfile, `build.sh`, `project.yaml`), a Google-account committer contact, and +`go-118-fuzz-build` to compile `testing.F` targets under libFuzzer. Non-trivial, +and it puts your build in someone else's repo. + +**ClusterFuzzLite: technically viable, probably not worth it.** Active +(2026-02-12), Apache-2.0, GitHub Actions supported, Go supported. It gives you +PR-scoped fuzzing, crash deduplication, coverage reports and corpus persistence. +But it requires the same OSS-Fuzz-style Docker build, and this repo's build is +heavy (protoc, tantivy, CGO — see `test.yml`). You would spend a week on the +container to gain deduplication and a coverage dashboard. + +**What to do instead: a cron `go test -fuzz` job.** ~30 lines of YAML, no +container, corpus persisted with `actions/cache`, minimization and regression +capture already built into the Go toolchain. That is 95 % of the value for 5 % of +the effort, and it is the right answer until the nightly job is actually finding +things and someone wants a dashboard. + +--- + +## 3. The three framing questions, answered + +**Go-specific or JSON-generic?** +**Go-specific, decisively.** Three reasons particular to this case: (i) the two +strongest oracles are *snapshot*-level (I1 and snapshot-equivalence) and a +`*model.SmartBlockSnapshotBase` cannot cross a process boundary without you +writing a serializer for it — you would be building a CLI harness to lose +fidelity; (ii) `snapshotdiff.Compare` is a Go API with `Options` and resolver +capabilities as parameters, and reimplementing its ~1 300 lines of admitted +normalizations out-of-process is absurd; (iii) the resource oracle needs +in-process allocation counters, which no external mutator can give you. A JSON- +generic mutator would be confined to the hostile-bytes half — the half that is +already well served by `go test -fuzz` with a good seed corpus. The single +exception where language-agnostic tooling earns its keep is schema-driven +*instance generation* (Option 4), because no Go tool does it — and even there the +output is just seed files for the Go fuzzer. + +**Attack via the JSON Schema, or by other terms?** +Three sub-strategies, and they pay off very differently here: + +- *Coverage-guided byte mutation* (Option 1) — **the main engine.** Wins because + the seeds are real and plentiful and the code paths are deep. Finds every + resource defect on the benchmark list. +- *Grammar/structure-aware mutation* — **the surprise winner, in an unusual form.** + The right grammar for the Marshal direction is not the JSON Schema, it is + **protobuf wire format**, mutated by the generic byte fuzzer with + `proto.Unmarshal` as the structural filter. You get structure-aware mutation + without writing a grammar. +- *Schema-driven generation* (Option 4) — **best as a corpus bootstrapper and as + the driver of the error-quality oracle**, not as the running engine. Its + fundamental limit is that `Validate` is schema **plus** semantics, so + schema-conformance is neither necessary nor sufficient for acceptance, and the + strong oracles do not attach to it. + +**What genuinely automates this, versus what must be hand-built?** + +*Automated, off the shelf:* the mutation engine, corpus management, crash +minimization, regression capture, the 10 s hang watchdog — all in the Go +toolchain, today, for free. Structural generation of snapshots — free, via +`proto.Unmarshal` over the existing `.pb` corpus. Schema validation — free, +`santhosh-tekuri/jsonschema/v6` is already a dependency. + +*Must be hand-built, and nobody sells it:* every oracle. The alloc/time +amplification budget (~20 lines, and it is the highest-value 20 lines in this +report). The I1/I2/fixpoint assertions (~15 lines each, trivial once stated). The +cross-surface agreement relations (~40 lines each). The `snapshotdiff` gating +policy. The seed corpus curation. And, if you go to Option 2, the 300–500-line +oneof-aware snapshot generator — which **no library will write for you**, because +gogo oneofs defeat every reflection-based filler in the Go ecosystem. + +The honest summary: **the tooling is ~10 % of this job and the oracles are ~90 %, +and this format is unusual in that its oracles are already written down.** + +--- + +## 4. Recommendation + +**Do first (week 1):** Option 1 (native fuzzing, corpus-seeded, with the +allocation-amplification oracle) + Option 3 (cross-surface agreement). Together +they are ~300 lines, add no dependency, need no license decision, plug into the +existing `test.yml` for free as regression tests, and would have caught **four of +the eight** benchmark defects — including all three resource-exhaustion ones and +the fragment disagreement. + +**Add later (month 2, if week 1 pays off):** Option 2 (`rapid` + hand-written +snapshot generators), specifically and only to unlock `snapshotdiff.Compare` on a +controlled distribution and to get structural shrinking. Gate it behind the +license decision for MPL-2.0; if that stalls, drive the same generators from a +hand-rolled byte cursor and skip the dependency. + +**Do cheaply and separately:** a `FuzzZipSource` target in +`core/block/import/common/source/` with an output-size budget. Thirty lines, and it +finds the decompression-ratio defect — which nothing in `anyblockjson` can reach. + +**Skip:** OSS-Fuzz (license + acceptance criteria, verified). `go-fuzz-headers` +(dormant, and silently nils the oneofs — the worst possible failure mode). +`google/gofuzz` (archived). `fzgen` (dormant, self-declared pre-beta). Full +schema-driven generation as an engine (keep only the single-violation +error-quality test). ClusterFuzzLite, for now. + +**Accept as not-fuzzable:** duplicate JSON keys last-wins. Write it as a unit +check over the token stream and move on. + +--- + +## 5. Concrete first-week sketch + +**Target 1 — `FuzzDocumentBytes`** *(hostile input; oracles: I2, I1, fixpoint, +resource, crash)* + +``` +data → Validate(data) → vErr + → Unmarshal(data, Options{GenerateId: seq}) → uErr +assert (vErr == nil) == (uErr == nil) -- I2 +if uErr == nil: + out := Marshal(sbType, snap, bare) + assert Validate(out) == nil -- I1 + snap2 := Unmarshal(out, …); out2 := Marshal(…) + assert out == out2 -- §11.2 fixpoint +all of it wrapped in budgeted() -- resource +``` + +Seeds: `//go:embed testdata/*.json` (`rich.json`, `rich_compact_ids.json`, +`rich_compact_omit.json`, `rich_omit_ids.json`, `containers.json`, +`testdata/authoring/*`) — plus a script-extracted set of the JSON literals already +embedded in the package's `*_test.go` files (there are hundreds, covering every +block type, mark case and envelope variant), plus one NFC/NFD seed and one +astral-plane seed. If a sweep has been run, add `-dump-json` output. + +**Target 2 — `FuzzSnapshotRoundTrip`** *(round-trip; oracles: I1, byte fixpoint, +resource, crash)* + +``` +if len(data) > 1<<16 { return } +proto.Unmarshal(data, &pb.SnapshotWithType) -- structural filter; skip on err +Marshal → assert Validate(json1) == nil -- I1 +Unmarshal(json1) → Marshal → assert json1 == json2 -- §11.3 +budgeted() around the whole thing -- resource +``` + +Seeds: the `.pb` files `cmd/anyblockroundtrip` already writes (`original.pb` per +artifact directory, and every `.pb` under a `-keep-exports` run). If no sweep +output is at hand, `proto.Marshal` the snapshots built by `richSnapshot()` and the +`snapshotdiff/` fixtures — a dozen seeds is enough to start; the engine grows the +corpus. + +Deliberately **not** in this target: `snapshotdiff.Compare`. It goes in a +companion non-fuzz test, `TestCorpusSnapshotEquivalence`, that walks the committed +`testdata/fuzz/FuzzSnapshotRoundTrip/` corpus and any real `.pb` corpus available, +asserting §11.1 (`Import(Export(S)) ≡ N(S)`) on *unmutated* inputs only. Same +comparator, controlled distribution, no manufactured false failures. + +**Target 3 — `FuzzFragmentAgreement`** *(cross-surface; oracles: agreement, I2)* + +``` +data → treat as a single block object + A: UnmarshalBlock(data, "", opts) + B: Unmarshal(`{"version":2,"blocks":[` + data + `]}`, opts) +assert (A errored) == (B errored) +assert blocks equal after id normalization +``` +plus the same shape for `UnmarshalFilters`/`UnmarshalSorts` against a dataview +block, and the `MarshalBlockSubtree ∘ UnmarshalBlocks` fixpoint. + +Seeds: single-block and single-filter literals lifted from `fragment_test.go`, +`filters_test.go`, `datefilter_test.go`. + +**Also week 1, non-fuzz:** measure the alloc-per-input-byte distribution over the +existing corpus so the amplification constants are evidence-based rather than +guessed; and add the double-run determinism check to all three targets, to be +removed once it has been green for a week. + +**CI wiring:** nothing for regressions (the existing job picks up +`testdata/fuzz/`). One new scheduled workflow on `ubuntu-latest`, three steps of +`-fuzztime=20m`, `actions/cache` on `$GOCACHE/fuzz`, artifact upload on failure. diff --git a/pkg/lib/anyblockjson/OVERVIEW.md b/pkg/lib/anyblockjson/OVERVIEW.md new file mode 100644 index 0000000000..091005c709 --- /dev/null +++ b/pkg/lib/anyblockjson/OVERVIEW.md @@ -0,0 +1,279 @@ +# AnyBlock JSON — what it is and why it looks like this + +A readable, strictly-validatable JSON representation of an Anytype object. +It replaces `.pb.json` (jsonpb of `SnapshotWithType`) as the export/import +format, and it is the document shape API v2 serves and accepts. + +`SPEC.md` is normative and long. This is the short version: the decisions +that shaped it, why each one was made, and what they look like. + +The audience assumption behind almost every decision: **the reader and +writer is often a language model.** That is not a nice-to-have framing — +it is what settled most of the arguments below, usually against the choice +a human-only format would have made. + +--- + +## The document + +```json +{ + "$schema": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "version": 2, + "id": "bafyreieqh63jv…", + "type": "Page", + "icon": { "format": "emoji", "emoji": "🔥" }, + "properties": { + "Name": "Project Phoenix", + "Status": ["In progress"] + }, + "option_ids": { "Status": { "In progress": "bafyrei…opt1" } }, + "blocks": [ + { "id": "b1", "type": "heading_2", "text": "Goals" }, + { "id": "b2", "type": "paragraph", + "text": "Ship the **new export** by Q3 with Roman" }, + { "id": "b3", "type": "bulleted_list_item", "text": "Flat JSON schema" }, + { "indent": 1, "id": "b4", "type": "bulleted_list_item", "text": "Validate in CI" }, + { "id": "b5", "type": "checkbox", "checked": true, "text": "Draft spec" }, + { "id": "b6", "type": "code", "language": "go", + "text": "func main() {\n\tfmt.Println(\"hi\")\n}" } + ] +} +``` + +Four things are load-bearing and worth noticing before the rationale: +blocks are a **flat array**, formatting lives as **markdown inside +`text`**, `properties` is a plain **key → value** map whose keys are +display names, and the handful of +things that are a CHOICE rather than a value — the icon, the cover — are +typed objects with a `format` member, so the alternatives appear in the +error message rather than only in the spec. + +--- + +## The decisions + +### 1. Blocks are flat, with an integer `indent` — not a nested tree + +Pre-order array; `indent` omitted when 0; no `children` key anywhere. + +**Why.** A nested tree needs a recursive schema (`$defs/block` referring to +itself), and a recursive schema **cannot be used with constrained decoding +or provider strict-mode**. Constrained decoding is precisely the mechanism +that rescues small models — in our prior-art review it took a 7B model from +0% to 75% on valid emission. Trading it away to keep `children` was not +close. + +Two supporting reasons. A truncated flat array is still a **valid prefix** +of the document, so a cut-off generation degrades to "fewer blocks" rather +than "unparseable". And transformer failure on structured output tracks +*depth*, not length — a real corpus datum here is that typical documents +nest ~6 deep, with outliers to 26, which is beyond the reliable depth for +sub-7B models and beyond some providers' nesting caps. + +The cost is honest: an off-by-one `indent` silently mis-parents a block +where a misplaced `children` bracket would have been a parse error. That is +paid for with strict monotonicity validation on import (an indent jump +greater than +1 is an error, path-addressed), plus a documented lenient +clamp mode that follows CommonMark's rule. + +### 2. Inline formatting is markdown inside `text`, not mark ranges + +`Ship the **new export**` — not `{"text": "...", "marks": [{"from": 9, "to": 21, "type": "bold"}]}`. + +**Why.** The protocol stores marks as UTF-16 offset ranges. Models cannot +produce or maintain offset bookkeeping — this is a well-documented failure +(Google Docs' "write backwards" workaround; Liveblocks' "great at +rewriting, terrible at patching"). Markdown puts the formatting *where the +formatting is*, so editing a sentence cannot desynchronize it from its +marks. + +A whitelist of tags covers what markdown lacks: ``, ``, ``. Emoji marks are materialized +into the text — lossy by design, and the only deliberate loss in the +format. + +### 3. Names, not ids, wherever a human wrote the name + +`select` and `multi_select` values are option **names** (`"In progress"`), +in property values, filter values and custom orders alike. Properties and +types are addressed by their **display names, raw** — `"Creation date"`, +`"type": "Page"` — bundled and custom alike. The derived +api-slug spelling (`created_date`) is no longer written; documents that +carry it keep resolving, because a derived slug always lands in its own +key's fold class. + +**Why.** An id is unguessable, so a model must fetch before it can write; +a name is already in the user's request. Import creates missing options by +name, matching the existing import semantics. The trade — two options with +the same name collapse on import — was accepted explicitly, and it is +recorded as a known anomaly rather than hidden. + +Extending the same rule to property and type keys was decided by +measurement, not symmetry: an A/B eval found that copying a name +byte-exactly is a solved behavior even at 4B scale, while *deriving* a +slug from a name is where models improvise — and improvise differently in +the key slot and the filter value that references it, the divergence that +silently unbinds a view from its property. SPEC §3 is the rule, including the per-document collision +ladder and the `property_internal_keys` / `type_internal_keys` legends +that keep an exported document invertible with no space to ask +(`option_ids`, in the example above, is the same idea for select options: +the id rides beside the name). + +### 4. Presence is meaningful + +Property values are written **verbatim**, including `false`, `0`, `""`, +`[]` and `null`. The omit-empty-and-default canonicalization applies only +to block attributes and envelope fields. + +**Why.** This one was decided by data. The first production sweep flagged +14,032 "issues" that were all the same thing: default scalars +(`is_hidden:false`, `revision:0`) that canonicalization had dropped. The +ruling was that **a user setting a property to empty is a fact**, and the +format has no business deleting it. Blocks are different — an absent +attribute there genuinely means "default". + +### 5. Vocabulary chosen for outsiders, not for the codebase + +`relation` → **property** everywhere. `smartBlockType` → `kind`. +`header*` → `heading*`. `bulleted` → `bulleted_list_item` (the name common +block-editor APIs share). Formats are +`select`/`multi_select`/`text`/`files`/`objects` — the REST API's names, +not the internal `status`/`tag`/`longtext`; the stored shorttext/longtext +split has one name between them, `text`. And everything the format defines +is spelled `snake_case`, digits included — property and type keys are +exempt, because they name things a user named and spell the display name +raw (decision 3). + +**Why.** The instruction was "rename everything, minimize new terms". The +format is read by people and models with no exposure to Anytype's +internals, and the largest single source of confusion was a vocabulary +that only made sense if you knew the history. + +How far the rename reaches is worth stating precisely, because an earlier +draft of this document overclaimed it. The format's own vocabulary — +member names, kinds, block types, and every bundled display name (eleven +bundled names were renamed for it: "Relation key" → "Property key", +"Featured Relations" → "Featured properties", the "Relation option" type → +"Property option", …) — no longer says "relation" anywhere. The word still +reaches a document from the two sources no vocabulary rename can touch. +The app's STORED keys keep their spellings (`relationKey`, +`featuredRelations`, the `relation` type key, …), and a document records a +stored key verbatim exactly where fidelity demands an identity rather than +a name: the envelope `internal_key`, and the values of the +`property_internal_keys` / `type_internal_keys` legends — measured on the +pre-rename corpus, each such key appears there on roughly 150 of 28,831 +documents. And user data is user data: a property someone named +"Relation", an object called "Company relation template" — their words, +carried verbatim, no rename's business. + +### 6. The compaction that survives is the legend-less one + +Full object ids are ~59-character CIDs — a single mention can cost more +tokens than the sentence containing it. There used to be two compactions: +one that shortened object references behind a `refs` legend, and one that +relabels document-local block/row/column/view ids to short suffixes. The +first is deleted; only the second is left. + +- **`CompactBlockLabels`** relabels doc-local block/row/column/view ids to + their last 5 characters. **Legend-less and lossy.** `CompactIds` is now an + alias for it. +- `OmitIds` drops ids entirely, for generation. +- **Object references are written in full, on every shape.** + +**Why the "lossless" half died and the "lossy" half stayed.** An indirection +table has three obligations — it must be carried, kept in sync, and read +back — and the object legend failed all three. API v2 removed the same +legend from its read shape after measuring a net token *loss* per document +and finding that it trapped write-back: an agent editing an object-valued +property through a label has to keep the legend in step, and one that +regenerates the document without it silently re-points every reference. The +freeze review measured a 200-item collection growing 32.7% under compaction. + +A block label has none of those obligations. It is a placeholder inside its +own document, never an address outside it, and a write endpoint resolves one +against the live object by unique suffix. Nothing to desynchronise. + +### 7. The round-trip contract is a fixed point, not byte-equality + +`Import(Export(S)) ≡ N(S)`, and `Export∘Import` is idempotent and +byte-stable — where `N` is a documented normalization (structural blocks +dropped, option ids resolved to names, marks canonicalized, deprecated +fields cleared, and so on). + +**Why not byte-equality with arbitrary input.** Because the format +deliberately drops things: structural blocks are regenerated by the editor +at first open (they are layout-dependent — a note has no title), and +normalization is the point rather than an accident. Promising byte-equality +would have meant carrying every legacy shape forward forever. + +### 8. Validation is discriminator-first, with path-addressed errors + +The schema branches on `type` before validating a block, rather than +presenting a flat `oneOf`. + +**Why.** A flat `oneOf` produces "does not match any of 23 schemas", which +is useless to a model *and* to a human. Discriminator-first produces "at +`/blocks/7/columns`: a `table` requires `columns`". Errors are the repair +instruction, so they are addressed to the exact path that is wrong. + +--- + +## What it has been tested against + +The format is swept over a real production account — every object exported +to AnyBlock JSON, re-imported, and re-exported, comparing both the state and +the bytes. That sweep is `cmd/anyblockroundtrip`; its run history and every +figure belong in `ANOMALIES.md`. + +What the state comparison covers is narrower than it sounds. `snapshotdiff` +compares detail values (up to the documented normalizations) and the plain +text of text blocks as a multiset — not marks, not block order, not table +shape, not dataview content, not file or bookmark metadata; and +byte-stability is self-consistency of the pipeline, so a systematic drop can +be byte-stable and invisible. Its own package doc says it: findings are +triage input, not proof. + +Within those limits it earns its keep. The pre-flat sweep (run 3, 35,369 +objects) round-tripped 99.86% byte-identically; the flat-encoding sweep that +followed (run 4, 35,372 objects) left 21 failures, all in categories already +known. The most recent round-trip sweep, over a 36,808-object account, is +where the last round of findings came from — and no pass rate for it is +recorded anywhere. The fixes made since have unit tests, not a +re-measurement. Separately, the native bundle exporter is verified against +the corpus by the same harness in `-native` mode — 28,542 documents +checked for layout, kind classification, determinism (every space exported +twice, trees byte-compared) and per-document fidelity against a +same-process pb export; `EXPORTER_DESIGN.md` records that run. + +Every anomaly found along the way is written up in `ANOMALIES.md` rather +than smoothed over — including two genuine silent-data-loss bugs the sweeps +caught that no unit test had. That is the reason to check the round-trip +contract against real data instead of merely believing it, and the same +reason not to read a pass rate as a proof of it. + +--- + +## Deliberate non-goals + +- **Not a wire format.** It is an export/interchange and agent-editing + format; the CRDT protocol is unchanged. +- **Not byte-equal to arbitrary input** — see decision 7. +- **No backward-compatibility burden yet.** The format has never shipped, + which is why the decisions above could still be reversed on evidence. +- **Emoji marks are lossy**, materialized into the text. The only + deliberate loss. + +--- + +## Where to look next + +| | | +|---|---| +| `PRINCIPLES.md` | the ten rules the format answers to, and the order they yield in | +| `PRINCIPLES_SHORT.md` | the same ten rules on one screen | +| `SPEC.md` | normative, complete, §14 has a full worked example | +| `ANOMALIES.md` | every real-data oddity found, with evidence | +| `EXPORTER_DESIGN.md` | the native bundle exporter: pipeline, layout, corpus verification | +| `cmd/anyblockroundtrip` | the production sweep harness | +| `schema/*.json` | the hand-authored JSON Schema (2020-12) | diff --git a/pkg/lib/anyblockjson/PRINCIPLES.md b/pkg/lib/anyblockjson/PRINCIPLES.md new file mode 100644 index 0000000000..138c87ee9b --- /dev/null +++ b/pkg/lib/anyblockjson/PRINCIPLES.md @@ -0,0 +1,386 @@ +# AnyBlock JSON — design principles + +Status: living document · applies to format version 1 (SPEC draft) · +Package: `pkg/lib/anyblockjson` + +`SPEC.md` says what the format *is*. This document says what it is *for* +and the rules every part of it answers to — the rules a change to the +format, to this package, or to the API v2 document surface must either +serve or knowingly bend. `PRINCIPLES_SHORT.md` is the one-screen version +of this document; `OVERVIEW.md` walks through the individual decisions; +`ANOMALIES.md` records what real data did to them. + +--- + +## What AnyBlock JSON is + +One JSON document per Anytype object: an envelope (`version`, `id`, `type`, +`kind` when not derivable), a `properties` map of key → value, and a flat +`blocks` array whose nesting is an integer `indent`. Rich text is a Markdown +subset inside `text`. Types are documents too (`kind: "object_type"` with +`type_properties`); a bundle of documents adds an `index.json`. + +It replaces `.pb.json` as the export/import format, it is the document shape +API v2 serves and accepts, and it is what an agent reads and writes when it +edits an object. The same bytes serve every door. + +A document as an agent might write it — no block ids (minted on import), +select options by name with the id legend riding beside them, formatting +inline: + +```json +{ + "version": 2, + "type": "Task", + "icon": { "format": "emoji", "emoji": "🚢" }, + "properties": { + "Name": "Ship the export", + "Status": ["In progress"], + "Due date": "2026-09-30T00:00:00Z" + }, + "option_ids": { "Status": { "In progress": "bafyrei…opt1" } }, + "blocks": [ + { "type": "heading_2", "text": "Goals" }, + { "type": "paragraph", + "text": "Lossless **and** readable, with Roman." }, + { "type": "checkbox", "checked": true, "text": "Draft the spec" }, + { "indent": 1, "type": "bulleted_list_item", "text": "Validate in CI" } + ] +} +``` + +--- + +## The rules + +### 1. Lossless for meaning + +**What the user expressed survives a round trip. What the system bookkeeps +may be normalized. Every accepted loss is written down.** + +The contract is a fixed point, not byte-equality: `Import(Export(S)) ≡ +N(S)`, and `Export ∘ Import` is idempotent and byte-stable (SPEC §11). `N` +is the written-down normalization — structural blocks the editor +regenerates, restrictions it rebuilds, ids, offsets, cached formats, UI +state. None of that is meaning. Text, marks, properties, views, column +widths and option vocabularies are, and they round-trip. + +Two consequences, both decided against the format's first instinct: +**presence is meaningful** — a property set to `false`, `0`, `""`, `[]` or +`null` is a fact the user created and is written verbatim; the omit-default +canon applies to block attributes and envelope fields only (§3) — and +**escape hatches over drops** — data with no first-class shape rides along +in `fields`, `root` and `store` (§2, §4a). The accepted losses are few and +listed, never smoothed over: emoji marks materialize into text (§8.1), +same-named options of one property collapse (§3), block-label compaction is +lossy and opt-in (§9a). The contract is checked against real data rather +than merely asserted: `cmd/anyblockroundtrip` exports, re-imports and +re-exports a production account, and every case it turns up is in +`ANOMALIES.md`. Evidence, not proof — its comparator sees detail values and +text-block text, not marks, block order, tables or dataviews — so the +figures stay with the run history in `ANOMALIES.md`, where what they measure +is written down. + +### 2. Readable by a stranger + +**A person who has never seen Anytype internals can read a document, +understand it, and hand-edit it.** + +Structure is visible as a flat list with an `indent`; formatting is the +Markdown the reader already knows; dates are RFC 3339; layouts, enum values +and block types are names, never numbers (§3, §5); keys come in a fixed, +meaningful order with `text` last (§4). Machinery that exists only for the +editor is hidden: a table is `columns` and `rows` of `cells`, not a subtree +of wrapper blocks with composite ids (§6.1); title, description and icon are +properties, not blocks (§7). + +The test: if understanding a field needs the codebase, the field is wrong. +A stranger has the editor for reading a page; the document is for the +moments when they don't — a diff, a backup, a git repository, a review of +what an agent just wrote. + +### 3. Borrow words, don't coin them + +**Every name answers to something the reader already knows. Internal names +never appear.** + +Precedence when naming anything: the term Anytype's public API already +uses; then HTML, CommonMark, SQL and the vocabulary common block-editor +APIs share; a new word only when none of those has one. The format owns exactly six Anytype concepts — object, property, +type, option, set/collection, space (§1) — and everything it defines is +`snake_case`, digits included, stated as a rule so a name added later needs +no decision (§1 *Naming*). So `relation` → `property`, `smartBlockType` → +`kind`, `header_1` → `heading_1`, `status`/`tag`/`longtext` → +`select`/`multi_select`/`text`. Property and type KEYS are exempt — they +name things a user named, and spell the display name verbatim (§3). +No name this format or the bundle mints says "relation" any more — eleven +bundled display names were renamed to keep that true once raw naming made +names the wire vocabulary ("Relation key" → "Property key", "Featured +Relations" → "Featured properties", the "Relation option" type → +"Property option"). The word still reaches a document where a STORED key +is recorded verbatim for fidelity — the envelope `internal_key`, the +values of the `property_internal_keys` / `type_internal_keys` legends — +and where a user put it in a name: addresses and user data are not +vocabulary, and neither is this rule's to rename. + +Borrow only where the meaning matches. `dataview` stayed `dataview` rather +than becoming `database`, because a dataview references objects it does not +own (§6.2): a familiar word that lies costs more than a new one. + +### 4. Nothing to guess + +**A valid document needs only what is in the author's head and in one +example. No offsets, no ids to fetch first, no bookkeeping — and that +includes small models.** + +The writer is often a language model, and the bar is set at the small end. +The question for any shape is *would a 3–7B model under a grammar emit this +correctly?* If not, the shape is wrong, not the model. That question settled +most of the format's arguments, usually against what a human-only format +would have chosen: + +- Inline formatting is Markdown inside `text`, not offset ranges (§8): + models cannot keep offset bookkeeping; Markdown puts the formatting where + the formatting is. +- Blocks are a flat pre-order array with an integer `indent`, not a nested + tree (§4): a recursive schema cannot be used under constrained decoding — + the mechanism that took a 7B model from 0% to 75% valid emission in the + prior-art review — and a truncated flat array is still a valid prefix. +- Ids are optional and minted on import (§9); numbering, structural blocks + and restrictions are derived, never written (§5, §7). +- The schema is closed and exhaustive (`additionalProperties: false`, + enumerated values, discriminator-first), so there is nothing to invent + (§12); at the API door every endpoint serves its schema and one worked + example (an API v2 convention). +- Errors are repair instructions — path-addressed, naming allowed values, + one fault → one issue — so generate → validate → feed back converges + instead of drifting (§12). + +What a model cannot be shown an example of, it will hallucinate. *Names, +not ids* (rule 6) is this rule applied to references. + +### 5. Token-efficient, not terse + +**Spend no token that carries no meaning; never save one by making the +reader decode.** + +The free savings are taken: defaults and empties are omitted so the common +case costs nothing (§4); doc-local block ids relabel to their last five +characters — a 59-character CID costs more tokens than the sentence around +it (§9a); the API sends compact JSON and minimal rows. The saving NOT taken +is the one that looked biggest: object references were compacted behind a +legend until two independent measurements found it a net token *loss* — a +label used once costs more than it saves, and a 200-item collection grew +32.7%. *Names, not +ids* is a token rule too: the expensive unit is a round trip, not a byte, +and a name the author already has saves a fetch. + +The limit is decoding. The format stays per-block JSON objects with +readable keys because every terser encoding tried cost more than it saved — +a tabular encoding produced 44.6% valid one-shot generations against JSON's +75%, and a 2026 study of agent-facing conventions found aggressive +compression raised total session cost 67% while cutting input tokens 17%. +Token efficiency means semantic density: every token carries meaning the +reader uses, nothing is repeated that a legend can carry once, and nothing +needs a lookup or a reasoning step to mean something. + +### 6. Names, not ids + +**Wherever a human would write a name, the format carries the name.** + +Select options are names — in values, filter values and custom orders alike +(§3, §6.2). Properties and types are addressed by their display names, +layouts and block types by name. Only objects keep ids, because nothing +else about an object is unique — and even those may carry an informative +`#name` suffix (`bafyrei…#roman`) that import trims without ever resolving +it, so a reader sees what a reference points at while the id stays the +whole address (§9). + +An id is unguessable: a model must fetch before it can write, or it invents +one — the hallucination surface in its purest form. A name is already in the +request. Import creates missing options by name, as the CSV importer and +the public API already do. The cost is accepted and listed: +same-named options collapse; renaming an option breaks the link on +reimport (§3). + +### 7. A document stands alone + +**One exported object is understandable and re-importable without the space +it came from.** + +The compaction that survives is the one that needs no inverse: a block label +is a placeholder inside its own document, never an address outside it, so +there is no table to carry, keep in sync or read back — which is exactly the +three obligations the deleted object legend failed. What the envelope carries +instead is identity, not compaction: `property_internal_keys` and +`type_internal_keys` for the stored key behind each custom spelling, +`option_ids` for the option each select name means (§3, §9a) — a spelling +that reads back as a *different* property or type +in a reader that cannot ask the space is the defect a sweep saw as twelve +objects whose dataview came back pointing at another property. The mechanism behind it was established after +that sweep, and its guard is demonstrated by unit test, not by a +re-measurement: those twelve have not been swept again since it landed. A +type document carries its property definitions with their option +vocabularies, colors and target types, in one file (§2a); a bundle carries +`index.json` and is versioned as one artifact (§2c). + +The intention goes further than the format currently does. The structural +facts a reader needs about an object — which format a custom property has, +the vocabulary of a select, the display name behind a key — should travel +with the document, so that a single-object export is a complete artifact +for a reader with no space and no resolver. Today formats resolve through +caller-wired resolvers (§3 *Format resolution*) and a names sidecar is +listed as a future extension (§1 *Non-goals*). That gap is tracked, not +accepted. + +### 8. Strict in, canonical out + +**Export has one spelling of every document. Import validates strictly, +addresses every error to a path, and never guesses.** + +Canonical output — fixed key order, omitted defaults, minimal escaping (§4) +— is what makes diffs meaningful and `Export ∘ Import` a fixed point. +Import accepts liberal forms only where listed and canonicalizes them +(`heading_4`, `equation`, `_x_`, HTML entities, date-only strings); an +unknown block type, an unknown tag attribute, a `children` key, an +over-deep indent are errors, not silent drops (§4, §8, §12). Lenient modes +exist, are opt-in, and report every clamp with its path. + +Never guess: the `anytype://object` deep link is matched by exact form; a +tag-shaped sequence the version does not define is literal text plus a +warning; an object id is an object id, never a label some legend might +rebind (§8.1, §9a, §10). `Validate` and `Unmarshal` accept and reject the same +documents, and `Marshal` never emits what `Validate` rejects (§11, §12) — +the promises that make "this document imports" a statement rather than a +hope. And a check has to earn its place: it catches something silent *and* +traces to a mechanism, or it does not ship — every marginal warning makes +the ones that matter cheaper to ignore (§12). + +### 9. One shape, every door + +**Export, import, bundles, API v2, templates and prompt examples all speak +the same document. There are no dialects.** + +A file export, an `index.json` bundle, a `GET /v2/…/objects/{id}` body, a +`POST` that creates an object, the worked example in an API schema — same +envelope, same blocks, same vocabulary, same validator. `OmitIds` and +`CompactBlockLabels` are serializations of one format chosen per consumer, +not formats; the canonical full-id form is the +round-trip form (§9, §9a). Anything learned from reading an export applies +to the API, and the other way round. + +This is why the format, not the API, owns block ids (a flat array with +optional stable ids is exactly what id-addressed edit operations need), the +filter tree (the compact filter string parses to it; §6.2.1), and the +validation errors the API returns. Markdown remains the lossy human format: +served read-only with warnings, never an editing channel. The package is +pipeline-agnostic — it depends on the model and the bundle, never on the +import or export pipelines (§13) — because the format is upstream of both. + +### 10. Evolution is explicit + +**One version integer. A reader refuses what it does not know, migrates +what it does, and every change is a bump.** + +`version` is the sole authority on format identity (§10). A reader rejects +a newer document with a dedicated error naming both versions, accepts an +older one by migrating it forward, and every format change bumps — a closed +schema has nothing additive to offer an older reader anyway. Inside `text`, +where no version marker can live, canonical output escapes every tag-shaped +`<` so the whole tag space stays reserved for later versions, and the +Markdown delimiter set is closed: a future mark is a tag, never new +punctuation (§8.2). + +This is the opposite of HTML's *degrade gracefully*, on purpose: an +interchange format with one reader per version and no partial semantics is +better off refusing than silently half-reading. It would be the wrong trade +for a wire format — one reason AnyBlock is not one. + +--- + +## When rules collide + +Decide in this order: + +1. the user's meaning survives (rule 1); +2. a model — small ones included — can write it (rules 4, 6); +3. a stranger can read it (rules 2, 3); +4. it costs few tokens (rule 5); +5. it is convenient to implement, or faithful to an internal name. + +The order is not a wish; it is what the format's recorded decisions already +did: *presence is meaningful* (1 over 4 — 14,032 omitted defaults put +back); *flat blocks* (2 over 3 — a nested tree reads a little better to a +person, but a model cannot be constrained to it; humans have the editor, +models only have the bytes); *option names* (2 over the strictest reading +of 1 — the collapse is listed, because the alternative is an id no one can +write); *`dataview`, not `database`* (understandable over familiar); +*refuse newer versions* (a one-rule contract over client convenience). + +Two things never yield: `Marshal` never emits what `Validate` rejects, and +no loss is silent. + +## Non-goals + +- **Not a sync or wire format.** The CRDT protocol is unchanged; AnyBlock is + interchange, backup, and the agent editing surface — which is what makes + rule 10's refusal affordable. +- **Not a replacement for the Markdown export.** Markdown stays the lossy + human format, served read-only with warnings. +- **Not byte-equal to arbitrary input.** The canonical form is the fixed + point (rules 1, 8). +- **Not forward-compatible** (rule 10). +- **Not a per-object schema language.** Per-type validation artifacts — a + JSON Schema, a prompt-ready property table — are derived one-way from type + documents and never imported (§2a). +- **Not an archive layout.** One object per document; how files are laid out + and binaries shipped belongs to the writer (§1). +- **Not yet:** the isomorphic HTML sibling (the Pandoc precedent) and the + compact filter string *inside documents* (its parser ships for the API; + the document field is reserved, §6.2.1). + +## How the rules are kept + +- A hand-authored JSON Schema 2020-12 — closed objects, exhaustive enums, + non-recursive block definition, discriminator-first dispatch — embedded + in the package and published at a stable URL (§12). +- Package tests: golden files, property-based round-trip tests over + generated states, and a corpus invariant that `Validate` and `Unmarshal` + agree (§11, §12). `cmd/anyblockroundtrip` sweeps a production account; + `ANOMALIES.md` is its ledger. +- Review rules for changes: a new name follows rule 3's precedence; a new + field either round-trips (rule 1) or is output-only and marked + `x-output-only` (§4a); a new validation check catches something silent + and traces to a mechanism (rule 8); a new serialization option is a view + of the one format, never a dialect (rule 9). + +## How this document changes + +- A change to the format, to this package, or to API v2's document surface + names the rule it serves — or the rule it bends and the cost it accepts, + recorded in the commit that makes it and, when real data drove it, in + `ANOMALIES.md`. +- Rules move on evidence — a sweep, a benchmark on the target model tier, a + falsified assumption — never on taste. Several already have: nested → + flat (constrained-decoding evidence); key allowlist → deny rule (59 real + keys falsified it); omit-default properties → presence is meaningful + (14,032 flags); camelCase → snake_case (a model wrote + `bulleted_list_item` unprompted against a camelCase draft). +- When this document and `SPEC.md` disagree, one of them has a bug, and the + change that fixes it says which. +- It is short on purpose. Details belong in `SPEC.md`, decisions in + `OVERVIEW.md`, data in `ANOMALIES.md`. + +## Prior art — what was taken, and what was left + +| Source | Taken | Deliberately left | +|---|---|---| +| [HTML Design Principles](https://www.w3.org/TR/html-design-principles/) (W3C) | the shape of this document; *do not reinvent the wheel*, *pave the cowpaths* (rule 3); *priority of constituencies* (collisions); *well-defined behavior* (rule 8) | *degrade gracefully* — reversed in rule 10, which a non-wire format can afford | +| [CommonMark](https://spec.commonmark.org/) | readability as the overriding goal; the inline subset and escaping; spec-by-examples → golden files | the delimiter-run algorithm — replaced by an exact-inverse stack parser, because byte-stability needs an inverse (§8.3) | +| [Djot rationale](https://github.com/jgm/djot#rationale) | linear parsing; no expressive blind spots; one spelling per construct | — | +| Block-editor APIs (common vocabulary) | block and property names (`bulleted_list_item`, `heading_1`, *property*); options by name | `{id, name}` option objects (rule 6); `database` (rule 3) | +| Atlassian Document Format | an envelope with a single `version` integer; `type`-discriminated nodes | additive-within-a-version; nested `content` trees (rule 4) | +| [Portable Text](https://www.portabletext.org/) | JSON blocks as the unit; a legend referenced by key (`markDefs` → `property_internal_keys`, `type_internal_keys`, `option_ids`) | marks as arrays on spans — Markdown in `text` instead (rule 4); a legend for object references, measured a net loss (rule 5) | +| [JSON Canvas](https://jsoncanvas.org/), [OKF](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) | a short spec with its purpose stated first; goals and non-goals up front; longevity, readability, interoperability as the brief | — | +| Anytype public REST API (`core/api`) | format names (`select`, `multi_select`, `text`, `objects`, `files`); snake_case member names | id/key duality; value fields named after formats; the derived slug vocabulary (keys spell display names, §3) | +| Agent-API evidence 2024–2026 ([Ustynov 2026](https://arxiv.org/abs/2604.07502)) | id-addressed edits; constrained decoding as the small-model floor; examples over prose; SQL-shaped filters; the validation loop as product surface; compact but not exotic | tabular/TOON-style output by default; raw JSON Patch; whole-document rewrite as the default edit | diff --git a/pkg/lib/anyblockjson/PRINCIPLES_SHORT.md b/pkg/lib/anyblockjson/PRINCIPLES_SHORT.md new file mode 100644 index 0000000000..ffe253e126 --- /dev/null +++ b/pkg/lib/anyblockjson/PRINCIPLES_SHORT.md @@ -0,0 +1,56 @@ +# AnyBlock JSON — the rules, on one screen + +The one-screen version of `PRINCIPLES.md`, which carries the rationale and +the evidence. `SPEC.md` is the format itself. + +AnyBlock JSON is the one document shape for an Anytype object: what export +writes, what import reads, what a bundle ships, what API v2 serves and +accepts. It is written as often by a language model as by a person, and it +must be readable by someone who has never seen Anytype's internals. + +## Ten rules + +1. **Lossless for meaning.** What the user expressed survives a round trip; + what the system bookkeeps may be normalized; every accepted loss is + written down. +2. **Readable by a stranger.** A person who has never seen Anytype internals + can read a document, understand it, and hand-edit it. +3. **Borrow words, don't coin them.** HTML, CommonMark, SQL, our own public + API, the vocabulary block editors share; six Anytype terms; no name the + format or the bundle mints says `relation` — only recorded stored keys + and user-given names still do. +4. **Nothing to guess.** A valid document needs only what is in the author's + head and in one example — no offsets, no ids to fetch first, no + bookkeeping. Small models included. +5. **Token-efficient, not terse.** Spend no token that carries no meaning; + never save one by making the reader decode. +6. **Names, not ids.** Wherever a human would write a name, the format + carries the name. +7. **A document stands alone.** One exported object is understandable and + re-importable without the space it came from. +8. **Strict in, canonical out.** One spelling on export; strict, + path-addressed validation on import; never guess — fail loudly. +9. **One shape, every door.** Export, import, bundles, API v2, templates, + prompts: the same document, no dialects. +10. **Evolution is explicit.** One version integer; a reader refuses what it + does not know; every change is a bump. + +## When rules collide + +The user's meaning survives › a model (a small one too) can write it › a +stranger can read it › it costs few tokens › it is convenient to implement +or faithful to an internal name. + +Two things never yield: `Marshal` never emits what `Validate` rejects, and +no loss is silent. + +## Not + +A sync or wire format · a replacement for the Markdown export · byte-equal +to arbitrary input · forward-compatible. + +## Using the rules + +A change to the format, to `pkg/lib/anyblockjson`, or to API v2's document +surface names the rule it serves — or the rule it bends and the cost it +accepts. Rules move on evidence, never on taste. diff --git a/pkg/lib/anyblockjson/SPEC.md b/pkg/lib/anyblockjson/SPEC.md new file mode 100644 index 0000000000..7b6467b739 --- /dev/null +++ b/pkg/lib/anyblockjson/SPEC.md @@ -0,0 +1,5245 @@ +# AnyBlock JSON — format specification + +Status: **draft** · Format version: **2** · Package: `pkg/lib/anyblockjson` + +A human- and agent-readable JSON serialization of Anytype objects (the "anyblock" +model), designed for export, import, and generation by external tools and LLM +agents. It replaces the raw `jsonpb` dump (`.pb.json`) as the recommended JSON +interchange format. + +Design lineage: the envelope and block tree follow the Atlassian Document +Format (nested `type`-discriminated tree, a single `version` integer — +though evolution here is never additive, §10); inline formatting uses a +Markdown subset inside text +strings; the vocabulary follows Anytype's public REST API (`core/api`) and +common block-editor conventions wherever an established term exists — the format should be +readable, and mostly writable, by someone who has never seen Anytype +internals. + +## 1. Goals + +1. **Readable** — a person can read and hand-edit a document; structure is + visible as nesting, formatting as familiar Markdown; every name answers to + something the reader knows from HTML, Markdown, SQL, or common REST APIs. +2. **Generatable** — an LLM or script can produce a valid document from one + example, without offsets, id cross-references, or Anytype-specific + instructions. +3. **Strictly validatable** — a published JSON Schema (draft 2020-12) covers + the structural format; the Go package validates on import (schema + + semantic checks, including the inline grammar) and returns structured, + path-addressed errors. +4. **Lossless round-trip** — `Import(Export(S))` reproduces the state `S` up + to the normalization defined in §11; export is canonical and + `Export ∘ Import` is idempotent (§11). + +### The four consumers + +The four goals above are not four independent wishes. Each is claimed hardest +by one of four consumers, and the consumer that claims a goal hardest is the +one that sets its bar. + +| | Consumer | May assume | Owes | +|---|---|---|---| +| **1** | **Export and import** — backup, migration, round trip: `Marshal`/`Unmarshal` (§13), an archive or a bundle on disk (§2c) | a live space at both ends, and that the bytes it reads are bytes it wrote | goal 4 (lossless up to §11, canonical output) and goal 1 | +| **2** | **Authored documents** — an agent, a script or a person writing objects, types and whole bundles for a space that may not exist yet | nothing but the document, the schema, and the bundled key table every reader ships | goal 2, and the offline half of goal 3 | +| **3** | **The API over the format** — API v2 (`core/api/v2`): explicit operations against a live space | a space, a store, and the resolvers of §13 | the store-backed half of goal 3 — resolve, refuse or create, and say which | +| **4** | **Tool wrappers over that API** — the task-shaped tool set models drive instead of the raw surface, delivered as CLI verbs and as an on-device manifest (the API v2 design record's tool layer; the record itself is retired from the tree) | everything layer 3 guarantees | nothing this format defines: a context budget | + +**Layer 1 sets the readability bar, and every other layer inherits it.** Its +reader is the one that can ask nothing — a file open in an editor, a bundle on +a disk, a space that no longer exists. So identity that cannot be spelled +readably is not demoted to an id: the label stays in place and the map goes in +the envelope — `property_internal_keys`, `type_internal_keys` and `option_ids` (§3). *Naming* above records that move being made once already: the +key ↔ key mapping went into the DOCUMENT precisely because `Validate` takes no +resolver. One grammar, so the weakest reader sets the spelling. How far this +goes is bounded and the bound is written down — formats still resolve through +caller-wired resolvers (§3) and a name sidecar is a v1 non-goal; `PRINCIPLES.md` +rule 6 (*Names, not ids*) marks that gap tracked, not accepted. + +**Which is why a select value is spelled by its name** (§3) — and not merely +because names read better. *A bundle carries no option objects.* A linked +object travels in the bundle and the importer relinks it; an option id from +another space would dangle. The name is the only address that survives the +trip, a fact about what moves between layers 1 and 2 rather than a preference +about what reads nicely. The scope is exactly select and multi_select values: +object references stay ids by decision (*Non-goals* below). The id is not lost +— it rides in `option_ids` beside the name, honoured only where the target space +still serves it as a live option of that relation (§3). A format whose +readers can always resolve a code can afford to demote the human label beside +it to a non-authoritative hint; ours frequently cannot, so the name stays +load-bearing and the id is filed where a stale one does no damage. + +**Layer 2 is why ids are optional.** Its author has none — no id to quote, no +prior state to preserve, nothing to fetch before writing. So block ids are +optional on input and `OmitIds` writes that shape back out (§9); the envelope +`id` is not part of that trade and stays. What cross-document identity a bundle +needs, it mints for itself: `index.json` and every widget target are the +bundle's own slugs, relinked on install like any other reference (§2c). An +id-less document is a first-class input and an alternative serialization, never +a dialect (`PRINCIPLES_SHORT.md` rule 9). + +**The line between layers 2 and 3 is a function signature.** `Validate(data +[]byte) error` takes bytes and nothing else — no space, no store, no resolver +(§13). Everything on the near side is checkable by an author with no account: +id collisions across a document (§4), two property spellings binding onto one +stored key, a `group_by` no view can honour (§6.2), a legend entry that can +never be consulted (§9a, §12), a malformed inline grammar (§8.1). Everything of +the form *does this already exist here* — is that type present, is that name +already an option, which of the two options named `High` did you mean, does that +id still resolve — is structurally on the far side, and no argument `Validate` +could take would move it. That is not a gap to be filled later; it is the +boundary, and it is what lets a bundle be checked before the space it creates +exists. The cross-document half belongs to the bundle tooling — +`anyblockvalidate` builds its id set from the bundle's own files to reject a +widget target no document defines (§2c), and still asks no space anything. + +**So the format has no "create this option" flag** — the request that arrives +once per reader. Three reasons; the third settles it. + +- **The format describes a state, and a state has no verb slot.** A document + says what an object *is*; `create` is something a caller *does*. +- **Its value is fixed per consumer, never per document.** Layer 2 *requires* + implicit creation — an authored bundle declares types, properties and options + no space has seen, and the bundle **is** their creation (§2 `type`, §2a, §3). + Layer 1 authors nothing: everything its documents name existed in the space + they came from, so a missing option at import is a restoration rather than a + new design decision. A field that is always true for one caller and always + false for another is describing the caller's intent, not the object. +- **Only layer 3 holds the store the question is about** — and it answers it + there, in the direction this argument predicts. The format's own import + default creates what is missing (§3); API v2 gates that behind an explicit + request parameter defaulting to off, answering a name it cannot resolve with + a did-you-mean error instead, and the tool wrapper is stricter still (the + small-model review's finding, recorded in the retired API v2 design + record). Same document, opposite + defaults, chosen by the caller. + +**Failure is loud at every layer** (`PRINCIPLES.md` rule 8, *Strict in, canonical out*): version refusal +with no partial read (§10), one unsupported file and a whole bundle declines to +install (§2c), path-addressed import errors (§12), a warning for an `option_ids` entry +nothing can consult (§9a), and layer 3 naming the candidates rather than +choosing one. The two places the line bends are both documented as such and +both compensated: name resolution answers the FIRST when two options of one +property share a name, which is the whole reason the document carries the id +beside the name (§3); and a widget target that resolves to nothing, silently, which is why the +tooling refuses it before install rather than leaving it to be discovered (§2c). + +**Layer 4 subtracts — but it has already added.** A tool set hides what a model +should not spend context on, ids first of all: the wrapper resolves enumerated +handles server-side so the model never emits a CID (the API v2 design +record's tool layer). What it +may not do is invent a dialect. It would be false, though, to say the format +owes it nothing. Block ids relabel to short +suffixes because a 59-character CID costs more tokens than the sentence around +it, and they can do so safely only because that relabeling carries no legend to +trap a write-back (§9a); `OmitIds` exists for templates and +prompt examples (§9); `blocks` is a flat array partly because guided decoders +cannot express a recursive schema. Layer 4's needs shape this format — +they are met as vocabulary and as serialization options, never as a second +format. + +**One grammar, four serializations.** The layers pay in different currencies, +so the same document is written differently at each: + +| | block ids | object refs | +|---|---|---| +| 1 · export, backup | full — the bytes re-import to the same document, up to what the editor regenerates (§7, §7a, §11) | full — always (§9a) | +| 2 · authored | none (§9, `OmitIds`) | the bundle's own slugs (§2c) | +| 3 · API v2 default read | compact by default; `?ids=full` opts out | full — always | +| 4 · read-only and tool shapes | short labels (outline, prompt examples) | hidden behind enumerated handles | + +Rows 3 and 4 are the API's choice, not the format's (API spec C4). What the +format contributes is that the one compaction it offers carries no legend +(§9a) — a read can be relabeled without a backup having to carry an +indirection table a later edit could desynchronise — and that all four rows +are one document, one validator, one version. + +### Naming + +**Every identifier the format defines is `snake_case`**: block types, field +names, enum values, and inline tag attributes. The spelling is the internal +name run through the same conversion Anytype's public REST API applies to its +own keys (`strcase.ToSnake`, `core/api/util/key.go`), **digits included** — +`heading_1`, `toggle_heading_1`, `bulleted_list_item`, `table_of_contents`. +Stating the rule rather than a list means a name added later needs no decision. + +This follows the vocabularies §1 claims lineage from — common block-editor +naming (`bulleted_list_item`, `heading_1`) and Anytype's public API +(`background_color`, `added_at`) — and, more to the point, it is the spelling a +generating model produces unprompted: the format's own pre-freeze review +records an LLM writing `"type": "bulleted_list_item"` against a camelCase +draft, which the reader then had to reject. + +**Two kinds of string are exempt, both because they name something outside the +format.** They are not inconsistencies to be tidied away later: + +- **Property and type keys** (§3) name relations and types, which live in a + space rather than in this format. Their canonical spelling is the entity's + **display name**, NFC-normalized and otherwise verbatim — `"Due date"`, + `"Plural name"`, `"Publish Date"`, `"Тоггл"` — so the exemption is the + ordinary case, visibly: spaces, capitals and any script, exactly as the + user named the thing. Where no name can spell the key at all (an empty or + unwritable name, a collision the §3 ladder cannot suffix), the stored key + is written verbatim, whatever its shape, because an exact stored key is + always its own address (§3). + The key ↔ key mapping goes into the DOCUMENT — the + `property_internal_keys` / `type_internal_keys` legends — because `Validate` + takes no resolver, and a reader with no space at all can invert them. +- **Platform identifiers** — the `dataview` block id (§7) and the `objectId` + parameter of the `anytype://object` deep link (§8.1) — name things that + exist in a live space. They are quoted, not translated. + +### The `_` namespace + +**A value beginning with `_` addresses the platform, and nothing a document or +a bundle mints may begin with one.** The platform's own addresses already live +there — `_otpage`, `_brdue_date`, `_missing_object`, `_participant_…`, +`_date_2024-01-01` — and the format borrows the same namespace for the +built-in screens and listings an `index.json` may name: `_favorite`, +`_recent`, `_recent_open`, `_set`, `_collection`, `_all_objects`, `_chat`, +`_bin`, `_widgets`, `_graph` (§2c). + +The point is that the two sets are then disjoint **by construction**, not by +inspection. While the reserved listings were bare words, a bundle shipping an +object with id `set` captured every widget that meant *the Sets listing*: the +pb importer resolves a widget target through the bundle's own id map first +(`common.UpdateLinksToObjects`) and only then asks +`widget.IsPredefinedWidgetTargetId`, so the object won, silently, with no +finding from any check. The reserved homepages had the same collision with the +precedence reversed — `builtinobjects.setWorkspaceSettings` matches the +reserved names *before* resolving an id, so a bundle object with id `graph` +could never be the homepage. One rule closes both directions. + +This is a **prefix** rule, which is why it can be permanent: "no minted id +STARTS with `_`" is a promise the platform can keep, where reserving a word +means banning a new id every time a listing is added, retroactively. The name +after the prefix is still this format's own and is still snake_case, which is +why `_all_objects` and `_recent_open` are spelled that way rather than +quoting the live space's `allObjects` / `recentOpen`: the format's spelling +is its own everywhere, and the wire's camelCase comes back at the boundary +with everything else (`WireWidgetTarget`). + +The prefix is translated away at the wire boundary, where the importer's own +spellings are bare (`favorite`, `graph`), by `WireWidgetTarget` / +`WireHomepage`. Writing `_set` into a link block would be strictly worse than +the shadowing it replaces — an unrecognised target becomes +`addr.MissingObject` and the widget is then stripped without an error. + +**Which is why a bundle-local id may not be one of those bare spellings +either** — `favorite`, `recent`, `recentOpen`, `set`, `collection`, +`allObjects`, `chat`, `bin`, `widgets`, `graph`. The prefix rule alone would +only move the collision one step downstream: the link block that leaves this +format saying `_set` reaches the importer saying `set`, and +`handleLinkBlock` still resolves through the bundle's ids first. The prefix +is what makes the *format* unambiguous — a reader never has to guess which +of the two kinds a target meant, and a typo inside the namespace can be +refused by name. The wire-word ban is what makes the *wire* unambiguous. +Both, or neither is worth having. + +(The JSON Schema's own `$defs` names — `blockCore`, `tableCell`, … — are +neither: they are schema-internal labels a document never contains, and they +keep JSON Schema's conventional camelCase.) + +So `{"type": "callout", "icon": {"format": "emoji", "emoji": "💡"}}` and +`{"properties": {"icon_emoji": "☕"}}` are both correct in the same document, +and they are not the same thing spelled twice: the first is a field this +format defines, the second is a key belonging to the data — and +it can only be a SPACE-MINTED relation that happens to be stored under that +name, because the bundled `iconEmoji` is refused there (§2b). 54 production +objects hold exactly that pair. `{"properties": {"wikiPerson": …}}` is where +the two part company. + +### Terminology + +The format uses six Anytype concepts; everything else is borrowed vocabulary: + +- **object** — a page-like unit (page, task, note, …); one JSON document per + object. +- **property** — a typed key-value on an object (stored + internally as a *relation* — the internal name never appears in the + format). +- **type** — the object's user-level type (`page`, `task`, `bookmark`…), + identified by a key. +- **option** — a named choice of a `select`/`multi_select` property. +- **set vs collection** — a *set* is a live query over a type; a + *collection* is a manually curated list of objects. Both are presented + through dataview blocks/objects. +- **space** — the container all object ids resolve within (never appears in + documents; ids are space-local). + +### Non-goals (v1) + +- Replacing the Markdown export (stays as the lossy human format). +- Multi-object archives: this spec defines a **single object per document**; + archive layout (one file per object, file binaries alongside) is owned by + the export writer, unchanged. +- Resolving object references to names (values stay ids, except + select/multi_select options — §3; a name sidecar may be a future + extension). +- An HTML-style sibling format (planned separately, isomorphic to this one — + the Pandoc precedent). + +## 2. Document envelope + +```json +{ + "$schema": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "version": 2, + "id": "bafyreieqh63jv…", + "type": "Page", + "icon": { "format": "emoji", "emoji": "🔥" }, + "properties": { … }, + "blocks": [ … ] +} +``` + +Fields, in **canonical order** (§4): + +| Field | Type | Req | Notes | +|---|---|---|---| +| `$schema` | string | no | Schema URL; written by export, ignored by import except for version detection (§10). | +| `version` | int | **yes** | Format version. This spec defines `1`. Every format change bumps it — there is no additive-within-a-version rule (§10). | +| `kind` | string | no | System-level object kind, snake_case (`page`, `profile_page`, `template`, `archive`, `widget`, `chat`, …) — from `model.SmartBlockType`. `chat` is `ChatDerivedObject`: a standalone chat object whose identity is `internal_key`, like a type's; its messages live in the CRDT store, not in snapshots, so it always imports empty. (`chat_object` is the deprecated predecessor; `discussion` is a hidden type.) **Omitted whenever derivable**: absent means `page`. It is the SOLE authority on whether a document is a template — `template_for` is admitted on it, the second type slot exists on it, and no type spelling implies it (§3). A template therefore always spells its kind. An unrecognized value is a validation error listing the allowed values. | +| `id` | string | no | Object id. Written by export; import treats it as informational (a new id is minted on import) except for resolving intra-export links. Written in full, like every object reference — object references are never compacted (§9a). | +| `type` | string | no | The object's type **slug** (`page`, `task`, `object_type`…) — the key vocabulary of §3, not the stored `ot-`-prefixed key. Maps to `object_types[0]` in the snapshot. Absent when the snapshot has no object types (legacy/system objects). Import inverts the term through the §3 chain in the type namespace — the document's own `type_internal_keys` legend first, then the vocabulary in force (bundled table offline, the space's stored slugs inside a node) — and hands the resulting stored key to the wiring, which resolves it — matching an existing type or creating one (the Markdown importer's behavior). A term the chain does not know passes through verbatim — an exact stored key is always its own address (§3). No spelling is reserved: `template` is an ordinary type term that a legend or a vocabulary may bind wherever it likes, because `kind` — a field no chain touches — carries the template semantics it used to carry. The one exception is a byte comparison, not a resolution: a document with **no `kind`** whose `type` is literally `template` is the legacy spelling of a template and is refused, naming the repair (§10). | +| `template_for` | string | no | Only for templates: the target type slug (`object_types[1]`), same vocabulary and legend as `type`. Admitted on `kind: "template"` and nothing else — present without it, or without a `type` beside it to be `object_types[0]`, is a validation error. Note what this is NOT keyed off: the template's own type. A template whose `object_types` do not begin with the template key is a shape the model permits. The target does not depend on what `object_types[0]` holds. | +| `internal_key` | string | no | Identity key of *system* objects (types, properties). This is the STORED identity key (a `uniqueKey`'s internal part), written verbatim: unlike every key slot in §3 it is **not** translated, so for an object whose stored key is a minted BSON it does not match the slug the public API serves as that object's `key`. The name says what the value is — an id the app MINTS (a bson for a custom definition, the camelCase bundled key for a bundled one), never something an author derives — where the word `key` used to name this stored id AND a property definition's spelling one level down, one word for two concepts (§15 #14). Because it is verbatim, its charset is whatever the store already holds: a relation option's key is built from the option's *name*, so `completion_status_Not Started`, `…_C/C++` and `…_тогглы` are all real stored keys. The rule is therefore a deny rule — non-empty, no control characters, at most 255 characters — not an allowlist. An allowlist was tried and falsified: it failed 59 objects of a 36 808-object account, every one a relation option. Never emitted for ordinary documents. | +| `property_settings` | object | on `kind: "property"` | Only for property documents, where it is **required**: the definition of the property this document IS — one `propertyDefinition` (§2d, §2e). Carries `format` (required, a §3 format NAME — never a raw enum number; stands for the stored `relationFormat` key, which `properties` refuses), `include_time` and `object_types`, each present exactly when its stored key is, value included. Illegal on every other kind. | +| `icon` | object | no | The object's icon — ONE object whose `format` selects the variant (§2b). Stands for the stored `iconEmoji` / `iconImage` / `iconName` / `iconOption` keys, which `properties` refuses. | +| `cover` | object | no | The object's cover — same shape, three variants (§2b). Stands for the stored `coverId` / `coverType` / `coverScale` / `coverX` / `coverY` keys, which `properties` refuses. | +| `properties` | object | no | The object's properties, §3. | +| `type_settings` | object | no | Only for type documents (`kind: "object_type"`, `"bundled_object_type"`): everything that defines the TYPE, in one gated subtree — `layout`, `api_key`, `plural_name`, `default_template`, `default_view`, and `property_definitions` (§2a). Present on any other kind → validation error. The root spelling `type_properties` is refused with the repair named. | +| `property_internal_keys` | object | no | Legend: the stored property key each spelling in this document names (§3). Written for every spelling the **bundled table does not bind to the key being written** — a slug the table cannot invert (a space's own key) *and* the **identity entry**, which is the ordinary case: a custom key written verbatim names itself, because nothing else in the document says the term is a stored key rather than somebody's slug. A reader consults it **before** its own vocabulary and takes the value as **authoritative**: it is not liveness-checked, deliberately (§3). Absent only from a document whose every spelling is bundled. | +| `type_internal_keys` | object | no | Legend: the stored type key each type slug in this document names — `property_internal_keys`' twin on the TYPE namespace, written and consulted under the same rule (§3). A separate map, deliberately: a space may slug a relation and a type onto one term, so one map could not carry both meanings of a shared spelling. | +| `option_ids` | object | no | Legend: the id of the option each select/multi_select **name** in this document stands for — nested, `{property spelling: {option name: option id}}` (§3, §9a). Written **unconditionally** wherever export spells an option by name; dropped by `OmitIds` (§9). Read as a **hint**, not an address: an id is honoured only where the target space still serves it as a live option of that relation, and otherwise the name resolves exactly as it did before the legend existed. | +| `blocks` | array | no | The document's blocks as a **flat pre-order array**; nesting via `indent` (§4). | +| `items` | array | no | For collection objects: member object ids, in order (from the internal collection store key `objects`). Present on a non-collection document → validation error — enforced by the import *wiring* (collection-ness resolves against the space's types, not offline); the package's `Validate` checks structure only (implementation decision). | +| `store` | object | no | Escape hatch: remaining internal store content as a free-form JSON object, with the `objects` key lifted into `items`. Output-only (§4a). (Named `store` — its internal name — to avoid colliding with the collection concept.) | +| `root` | object | no | Escape hatch for non-default root-block attributes (`fields`, `background_color`); absent in the common case. Output-only (§4a). | + +The root block of the snapshot (whose id equals the object id) is +**implicit**: its subtree becomes the `blocks` array (its direct children are +the indent-0 blocks). + +**The title and description are properties (§3), not blocks; the icon and the +cover are envelope fields of their own (§2b).** There is no title block in a +document (§7), and no icon block. + +Snapshot fields **excluded** from the format: + +- `fileInfo` — only present on old-format (deprecated) file objects; export + drops it, import leaves it empty. +- `relationLinks` — deprecated protocol-wide, scheduled for removal; not + represented. Property formats are handled via resolvers (§3). +- `removedCollectionKeys` — dropped (meaningful only for change replay, not + for fresh imports). +- `fileKeys`, `extraRelations` — deprecated in proto. + +### 2a. Type documents (`kind: "object_type"`) + +A type is not just a schema: it also owns the views over its objects +(columns, filters, sorts — presented through a dataview). Both live on one +object in the underlying model, so the format keeps them in **one +document** — a type never splits across files, and no JSON Schema file is +involved. + +```json +{ + "version": 2, + "kind": "object_type", + "internal_key": "task", + "icon": { "format": "icon", "name": "hammer", "color": "orange" }, + "properties": { "Name": "Task", "Description": "…" }, + "type_settings": { + "layout": "todo", + "api_key": "task", + "plural_name": "Tasks", + "default_template": "bafyrei…", + "default_view": "table", + "property_definitions": [ + { "property": "Due date", "name": "Due date", "format": "date", "section": "featured" }, + { "property": "Assignee", "name": "Assignee", "format": "objects", "section": "featured" }, + { "property": "Status", "name": "Status", "format": "select", + "options": ["Backlog", {"name": "In progress", "color": "blue"}, + {"name": "Done", "color": "lime"}] } + ] + }, + "blocks": [ { "type": "dataview", … } ] +} +``` + +**Everything that defines the type lives in `type_settings`** — one gated +subtree. Nesting is not tidiness: §2d already put one root `allOf` +conditional on the schema, five more root fields would be five more, the +eval found models putting `type_properties` on non-type documents precisely +BECAUSE the root had no conditionals, and many constrained decoders do not +implement `if`/`then` at all. One group is one conditional, and a per-kind +generated schema includes or omits it in one move. The five settings members +lift from `properties` (their flat spellings — `recommended_layout`, +`api_object_key`, `plural_name`, `default_template_id`, `default_view_type` +— are refused there ON TYPE DOCUMENTS with the repair named; the refusal is +kind-scoped where §2b's and §2d's are unconditional, because `apiObjectKey` +is real data on 9,725 relation documents, where it stays an ordinary +property): + +| member | stored key | shape | +|---|---|---| +| `layout` | `recommendedLayout` | the recommended layout of objects OF this type, as a layout name; a stored number outside the vocabulary passes through raw, and an unknown NAME is refused (it would import as a string onto a number detail, silently read as `basic`). | +| `api_key` | `apiObjectKey` | the type's public API key. **`api_key`, not `slug`**: of 1,326 corpus type documents with one, it differs from the document's own spelling in 247 (the `property` type's api key is `relation`, the word the public API kept when the format renamed the concept) — calling it a slug would imply it is the term used elsewhere in the document, which for those 247 it is not. | +| `plural_name` | `pluralName` | the plural display name. | +| `default_template` | `defaultTemplateId` | the object id of the template new objects start from — a scalar: the stored value is a list in every corpus document, with at most one entry (55 of 142; 87 empty), and a second entry is dropped with a warning. | +| `default_view` | `defaultViewType` | the default view type, as a §6.2 view-type name; same raw-number/unknown-name policy as `layout`. | + +The five follow the **§4 omit-empty canon** — a `pluralName` of `""` (145 +corpus docs) or a `defaultTemplateId` of `[]` (87) says nothing a reader +could act on — unlike §2d's members, which are a property's definition and +mirror presence exactly; the comparator reads the same rule through +`DroppedEmptyTypeSetting` (§11). + +The type's REMAINING details (`name`, `description`, `is_hidden`, +`order_id`, …) stay in `properties` under their stored keys (§3); its icon +is the envelope field every object has (§2b) — a type's icon is where the +`icon` variant is overwhelmingly used, since all 1,530 objects in the corpus +carrying an `iconName` are types. The four recommended-property id lists +(`recommended_featured_properties`, `recommended_properties`, +`recommended_file_properties`, `recommended_hidden_properties`) are +**replaced** by `type_settings.property_definitions` — resolved entries, +never raw property ids. The word is `property_definitions` rather +than `properties` because the document already uses that word for property +VALUES at the root, and one word carrying two meanings in one file is the +same shape as the `featured_properties` collision below — one word per +concept. + +**A type document does not carry its own install provenance.** Seven stored +keys are omitted on export and dropped on import (stale, not wrong — the +transient-key policy, scoped by kind), each admitted to the drop +individually against 1,760 corpus type documents (§15 #12; the verdicts +live on `typeProvenanceKeys`, and §11 N(S) records the normalization): +`layout` and `resolvedLayout` (ONE distinct value each — "object_type" — +derivable from the kind), `smartblockTypes` (occurs only on installed +copies of bundled types, restating the bundled table), `sourceObject` +(derivable from the type key: `_ot`), `origin` (how the INSTALL +happened — on ordinary objects origin is real provenance and stays), +`addedDate` (epoch-zero on 1,600 of 1,627), and `setOf` — which is the +type document's **own id** on 1,756 of 1,757, re-stamped by +`WithForcedDetail` from the object's id on every init, so it is a function +of the id rather than a fact about the type. + +Six candidates FAILED the admission test and stay in `properties`: +`is_hidden` (cannot be proven install-only), `order_id` (the user's own +ordering of types), `layout_width`/`layout_align` (the type object's own +page display, set by a person where non-zero), `featured_properties` — +which means what this type OBJECT features, while `section: "featured"` +means what objects OF this type feature: the two differ in 361 of 400 +corpus cases, so they are two things, not one — and **`revision`**, which +was admitted at first and then failed. `systemobjectreviser` short-circuits +on `bundleRevision <= localObject.GetInt64(revisionKey)`; an absent +revision reads 0, the guard stops firing, and the bundled definition is +copied over the local one for `name`, `pluralName`, `recommendedLayout`, +`isHidden` and `relationMaxCount`. Of 1,599 installed bundled type +documents, **40 carry a local name the reviser would overwrite** (key +`relation` is locally "Relation", bundled "Property") and 36 a local plural +name. Dropping it reverts a user's rename on restore, silently. + +`property_definitions` entry fields (canonical order): + +| Field | Type | Req | Notes | +|---|---|---|---| +| `property` | string | no* | The property's document-facing SPELLING — a key slot like any other, inverted through `property_internal_keys` (§3). Deliberately not called a key: the word used to name this spelling AND the envelope's stored id at once (§15 #14). | +| `internal_key` | string | no* | The property's STORED internal key, verbatim — never run through the §3 ladder, because a stored id is its own address and the bundled fold would rebind a slug-shaped one (`due_date` onto `dueDate`). Export writes it beside `property` for fidelity; an author never needs it, and cannot produce a correct one for a custom property (the app mints those — a bson id). *An entry must state an identity: `property`, or `internal_key`, or a `name` the spelling derives from; when both `property` and `internal_key` are present the spelling wins, and export writes an agreeing pair. A custom property whose entry states no `internal_key` gets a FRESH minted internal key from the import wiring's create path, the way the app mints one when a user creates a property — the spelling must not silently become the stored key. | +| `name` | string | no | Display name. Import uses it only when the property must be **created**; an existing property keeps its own name. Every bundled key already exists, so a name given for one is inert — `{"property": "Description", "name": "Summary"}` renders as *Description*. Validation warns. If the label is the point, mint a custom key instead of reusing a bundled one. | +| `format` | string | no | Property format (§3 names). Same import rule as `name`; a conflict with an existing property's format is an error at the wiring level (the package cannot see the space). | +| `options` | (string \| object)[] | no | A select/multi_select property's **vocabulary, in display order**. Each entry is a bare option name, or `{"name": …, "color": …}` when the option's color is part of the design — the color belongs to the option rather than to a parallel array, so inserting or reordering an option cannot shift it. `color` is one of `grey`, `yellow`, `orange`, `red`, `pink`, `purple`, `blue`, `ice`, `teal`, `lime` (`util/constant`); anything else is a validation error rather than a silently ignored value. The bare string is **canonical** whenever the option declares no color, the object form otherwise — the same rule cells follow in §6.1. Leaving a color out does not mean *no* color: the wiring assigns one, cycling the palette in declaration order and skipping whatever the vocabulary claims explicitly, so a vocabulary that names no colors still gets distinct ones. (The app assigns one at random on every other creation path; cycling keeps a converted bundle identical run to run.) Options are otherwise discovered only from values that happen to be used, so a vocabulary entry no record carries would never exist — its kanban column simply absent — and a discovered option carries no `orderId`, which makes every select sort alphabetically (options order by `[orderId, name]`, `pkg/lib/database.BuildOrderMap`). Declaring them lets the wiring create each one up front with an order id. Every option needs one: the sort concatenates `orderId + name` before comparing, so an option missing an order id is compared by *name* against the others' order ids and lands arbitrarily — ahead of the whole vocabulary when its name sorts below the id alphabet, behind it otherwise. Names discovered from usage rather than declared are ordered after the declared ones. Only meaningful on `select`/`multi_select`; duplicate names are a validation error, across both forms. | +| `object_types` | string[] | no | The **type slugs** an `objects`/`files` property may point at, in priority order — a type-key slot like the envelope `type`, so it speaks the one key vocabulary (§3), claims its spellings through the same type term ledger and owes the same `type_internal_keys` legend; import inverts each entry through the legend first, and a term the chain does not know passes through verbatim. Empty means any object — an untargeted property will happily accept a random page as a task's assignee. Listing the built-in `participant` alongside a bundle's own people type is what makes the current-user filter value usable on that property (§6.2) while still allowing the seeded people as values; the client only offers it when the relation's targets include Participant. The wiring resolves each key to an id the way it resolves properties: a type the batch defines by the id its own document carries, a bundled type by its bundled url (`_ot`). Only meaningful on `objects`/`files`. | +| `description` | string | no | The property's own description (its relation object's `description` detail). Same import rule as `name`: read when the property is created, inert on an existing one. | +| `include_time` | bool \| null | no | Whether a date property's values carry a time of day. Same import rule as `name`; meaningful on `date` only. | +| `max_count` | int | no | How many values the property holds; 0 (or absent) is unlimited, the stored default. Same import rule as `name`. | +| `readonly` | bool | no | Whether the property's value is user-writable. Same import rule as `name`. | +| `default_value` | any | no | The value a new object receives for this property. Same import rule as `name`. | +| `section` | string | no | `featured` \| `hidden` \| `file` — which list the property belongs to. Absent = a regular (sidebar) property. **The one field that belongs to the type rather than the property** (§2e): of 1,614 properties declared by 2+ types within one space, zero differ in anything else. | + +An entry is the one `propertyDefinition` shape plus `section` (§2e): the +schema expresses it as a reference to `$defs/propertyDefinition` with a +layer of narrowings (`format` to the authorable vocabulary, `object_types` +to a real array), never as a restatement. The five members after +`object_types` follow the `name` rule — read when the property must be +created, inert on an existing one — and the codec hands the WHOLE decoded +definition to the resolver's create path, so a member the schema admits is +never shed at the seam. + +Export emits entries in section order featured → regular → file → hidden, +preserving order within each list, and drops ids that no longer resolve to a +property (including the `_missing_object` sentinel of already-dangling +references); legacy lists that store bare property **keys** instead of ids +resolve through the reverse lookup, falling back to the bundle for system +properties. The canonical form writes `property`, `internal_key`, `name` and `format` on +every entry (`format` defaults to `text` when absent on input), and writes the +`property_definitions` array **even when empty** — its presence is what tells +import to rebuild the lists. Import then rebuilds all four id lists — empty +sections become explicit empty lists, matching how type objects store them — +resolving each entry's identity against the space and creating missing +properties (the same policy as select option names, §3). A document without a +`property_definitions` member leaves the lists untouched. + +Property ids are space-local, so the rewrite requires a property resolver +(`Options.ResolveProperties`, §13). Without one, export leaves the four +lists in `properties` as raw id lists, and import passes unresolved keys +through in place of ids for the wiring to reconcile — the same degradation +as option values without an option resolver (§3). A document carrying both +`property_definitions` and any of the four raw lists in `properties` is ambiguous +and fails validation. + +**Dataview.** A type's views live in a single dataview block on the type +object itself. When the snapshot contains it, export writes it as an +ordinary `dataview` block in `blocks` (§6.2) — most types customize their +views, so this is the common case and it round-trips losslessly. When the +document has no dataview block, import leaves it absent and the editor +generates the default at first open (from the recommended properties, as it +does today); import never fabricates or rewrites one. Export performs no +"is this the default?" comparison — presence in the snapshot is the only +criterion. + +**Derived schemas.** `kind: "object_type"` documents are the canonical type +definition. Per-type validation artifacts — a JSON Schema constraining +objects of that type, a TypeScript-style declaration, a prompt-ready +property table — are **generated one-way** from the type document (planned +`GenerateSchema`, §13) and are never imported or treated as authoritative. +This retires the legacy per-type JSON Schema export (`pkg/lib/schema`) with +its `x-` extension keys. + +## 2b. Icon and cover + +An object's icon and its cover are each **one envelope field holding one +object**, whose `format` member says which kind it is: + +```json +{ "icon": { "format": "emoji", "emoji": "📕" } } +{ "icon": { "format": "file", "file": "bafyreicfdcmfn…" } } +{ "icon": { "format": "icon", "name": "hammer", "color": "orange" } } +{ "icon": { "format": "color", "color": "teal" } } + +{ "cover": { "format": "image", "file": "bafyreigejp…", "y": -0.25 } } +{ "cover": { "format": "color", "color": "black" } } +{ "cover": { "format": "gradient", "gradient": "pinkOrange" } } +``` + +### `icon` — four variants + +| `format` | required | optional | stands for | +|---|---|---|---| +| `emoji` | `emoji` (non-empty string) | `color` | `iconEmoji` | +| `file` | `file` (object reference) | `color` | `iconImage[0]` | +| `icon` | `name` (a built-in icon name) | `color`, `emoji` (output-only) | `iconName` | +| `color` | `color` | — | `iconOption` alone | + +- **`color` is on every variant, because it is orthogonal to the source.** + 87 production objects attach one to something other than a named icon — 53 + workspaces and 2 profiles give an avatar image its background colour, 3 an + emoji — and 29 more carry a colour with no source at all (the letter-avatar + background; the API reports every one of those as having no icon). Its + value is one of the ten palette names §2a already mandates for select + options, mapped positionally from the stored number: `iconOption: n` is + `palette[n-1]`. +- **`color` also admits a raw integer ≥ 1**, for a stored value the palette + has no name for. This is not decoration: two generators in this repo + disagree about the range (`rand.Intn(16)+1` in the pb importer, + `rand.Intn(10)+1` in the markdown one), so 12, 13 and 15 exist in real + data. It is the same escape §3 already gives a layout number outside the + enum. `iconOption: 0` is the proto zero, **not** the first colour — 145 + production objects carry it and none of them is grey. +- **`name` is an OPEN string with a shape rule, not a closed enum.** The + ~397-name vocabulary lives in `core/api/model/icon.go`, which `pkg/lib` may + not import, and closing the enum would violate I1 the first time the app + ships a new icon. All 79 distinct values in the corpus are inside the API's + set. This is where the design is weakest for an offline generator, and it + is a deliberate trade against I1 (§15). +- **`emoji` on the `icon` branch is a carry-over, and is output-only (§4a).** + Exactly 200 production objects hold both an `iconName` and an `iconEmoji` — + every one a bundled type mid-migration (`Space` 🌎/`folder` ×18, `Type` + 🥚/`extension-puzzle` ×12) — and `format` has already answered which one + wins, so the emoji is baggage rather than ambiguity. Export writes it with + a warning; a document that supplies it is not choosing an icon. + +**Precedence, when the store holds more than one source:** `iconName` → +`iconEmoji` → `iconImage`. That is `core/api/service/icon.go`'s rule, the +only precedence implementation in heart — every other converter emits all +four channels and lets the consumer decide. See §15 for what is unverified +about it. + +### `cover` — three variants + +| `format` | required | optional | stored `coverType` | +|---|---|---|---| +| `image` | `file` (object reference) | `source` (`unsplash` \| `prebuilt`, output-only), `scale`, `x`, `y` | 1 / 5 / 4 | +| `color` | `color` (an opaque name) | — | 2 | +| `gradient` | `gradient` (an opaque name) | — | 3 | + +The `coverType` relation's own bundled description is this union written as +prose: *"1-image, 2-color, 3-gradient, 4-prebuilt bg image, 5-unsplash image. +Value stored in coverId"*. + +- **One `image` branch, not three.** A generator that has just uploaded an + image has no basis to choose between "image", "unsplash" and "prebuilt", + and choosing `unsplash` writes a permanent false provenance claim into cold + storage. `source` carries the provenance and is output-only. +- **`color` and `gradient` are opaque names.** Those two vocabularies live in + the clients and appear nowhere in this repo, so validation checks the shape + only. Observed colours: `black`, `ice`, `blue`; observed gradients: + `pinkOrange`, `red`, `sky`, `blue`, `bluePink`, `greenOrange`. A name + outside the app's set validates and shows as no cover — the one corner + where the typed shape cannot do what it exists to do (§15). +- **`cover.color` and `icon.color` share a member name but not a + vocabulary.** `black` is a cover colour and is *not* in the option palette. + Read each per variant. +- **Framing (`scale`, `x`, `y`) belongs to an image and to nothing else.** + In 36,966 objects those three are non-zero only under cover types 1 and 5, + though they are *present and zero* on colours, gradients and cleared covers + alike. + +### Object references, and the layer that is allowed to fetch + +`icon.file` and `cover.file` are **object references**: the id of an image +object in this space, a bundle-local slug, or the `_missing_object` +sentinel. Never a URL, never a filesystem path. The schema enforces it with +`^[^/]+$` — a shape rule rather than a URL-scheme rule, because the compiler +runs Go's RE2, which has no lookahead, and because a slash is what every +unwritable value in 36,966 objects has in common. + +**This format does no I/O, so it can only name what the store already holds.** +A URL is a job, not a value. A layer above — the API, the use-case installer +— may extend the schema with a `url` variant, fetch it, mint the file object +and rewrite the value into the plain `file` variant *before* anything reaches +`Unmarshal`. The whole extension is one entry appended to `icon`'s `allOf` +and one value appended to `format`'s enum; the `if` guards are mutually +exclusive by `const`, so nothing already valid is reclassified, and the +reader's own diagnostics name the new variant for free, because the union a +missing-`format` verdict lists is read out of the published schema rather +than restated in code. + +That is what the typed shape buys over another flat key. The precedent for +the flat one exists and is unpoliced: +`core/block/import/notion/api/commonobjects.go` writes a raw Notion URL +straight into `coverId` with `coverType: 1`, expecting a later pass to +download and rewrite it — and on 33 production objects that pass never ran, +leaving an absolute path into a temp directory that no longer exists. The +typed variant makes that state unrepresentable at the format boundary. + +### Where they live, and why not in `properties` + +Four reasons, all forced: + +1. **`cover` is already a stored property key**, in 30 production objects, + with `pageCover` in 66 more — both Notion imports, neither a bundled + relation. A schema node keyed on a `properties` member could also be + rebound by the `property_internal_keys` legend to point at an arbitrary relation, + which is an I1 hole and a laundering primitive. Envelope field names are + outside the key namespace and immune to the legend. +2. **`properties` carries presence-is-meaningful; the envelope omits empty + (§4).** Presence-is-meaningful is what generated the noise in the first + place. All nine relations are `hidden: true` — they have no property row + for presence to be meaningful *to*. +3. **It closes a gap §4a recorded and could not fix**: `coverId`/`coverType` + were output-only with no schema node of their own to annotate. They have + one now. +4. **It fixes a live readability bug.** 54 production objects hold both the + bundled `iconEmoji` (empty) and a space-minted relation whose *stored key* + is literally `icon_emoji` (holding, in one real space, `"☕"`). Anything + reading "the icon" out of `icon_emoji` in those documents reads a + coffee-tasting note. After the lift the document carries `"icon": {…}` at + the top and `"icon_emoji": "☕"` in `properties` — visibly two different + things. + +The precedent is not `id`/`type`; it is **the §2a property list** (`type_properties` then, `type_settings.property_definitions` now): stored +keys lifted into one labelled envelope member, with the flat spelling refused +where it used to sit. + +### The nine spellings are refused in `properties` + +`iconEmoji`, `iconImage`, `iconName`, `iconOption`, `coverId`, `coverType`, +`coverScale`, `coverX`, `coverY` — under any spelling that RESOLVES to one of +them (§3), the legend included — are refused, and the refusal names the +repair: + +``` +/properties/Emoji: "iconEmoji" is written as "icon": {"format": "emoji", + "emoji": "…"} (§2b), not as a property +``` + +The refusal is **derived** from the export side's own lift list, never +restated: a restated list is how the two surfaces drifted apart the last +time (§3, `deniedPropertyKey`). It is unconditional rather than conditional +on the typed field being present, because there is no second way to write an +icon — a format with two legal spellings for one thing, one of which a small +model has seen far more of in training data, defeats the whole point. + +Resolution on the *stored* key is what keeps the 54 dual-key objects above +working: their space-minted relation resolves to the stored key `icon_emoji`, +not `iconEmoji`, so it is an ordinary property and sails through. + +### The same shape elsewhere + +A **callout block** (§5) and a **bundle index** (§2c) carry the same `icon`, +restricted to the two variants a block or a bundle can hold (`emoji`, +`file`) — one `$ref`, narrowed by an enum, not a second definition. Shipping +the envelope field without them would leave two icon conventions inside one +document, which is the defect being removed. + +## 2c. The bundle index (`index.json`) + +Every document described so far is one object. **A bundle also needs to say +things about itself** — what the space is called, what opens when a user +enters it, what the sidebar shows — and none of that belongs to any single +object. That is `index.json`, one file at the bundle root, validated against +`index.schema.json`: + +```json +{ + "$schema": "https://schemas.anytype.io/anyblock/2/index.schema.json", + "version": 2, + "name": "Company Wiki", + "description": "Everything we know, with an owner.", + "icon": { "format": "emoji", "emoji": "📚" }, + "homepage": "page-wiki-home", + "widgets": [ + { "target": "page-wiki-home" }, + { "target": "type-wiki-page", "layout": "view", "limit": 6 }, + { "target": "_favorite", "layout": "compact_list" }, + { "target": "_all_objects", "card_style": "card", "icon_size": "medium" } + ] +} +``` + +| Field | Meaning | +|---|---| +| `name` · `description` | the space's own identity, applied on install | +| `icon` | the space's icon, in exactly the shape an object's icon has (§2b), restricted to the two variants a bundle can hold: `{"format": "emoji", "emoji": "📚"}`, or `{"format": "file", "file": ""}`. The image variant needs the image object *and* its file in the archive, so a generated bundle uses an emoji. It is one `$ref` into the object schema, not a copy — an index and an object cannot disagree about what an icon is. | +| `homepage` | what opens on entering the space: an object id, or the reserved `_widgets` (the sidebar dashboard, the default) or `_graph` | +| `widgets` | sidebar widgets, in order. **The first one is what the install opens**, so the entry point goes first | + +`version` is the same format version, with the same rules, that object +documents carry (§10): one integer, one namespace, bumped together. A reader +rejects an index declaring a version newer than its own, naming both — the +same dedicated error object documents get, never a generic constraint failure. + +**A bundle is one artifact and is versioned as one.** If the index or *any* +document in it declares a version the reader does not support, the bundle is +rejected as a whole and **nothing is installed** — a bundle creates a space, +installs types and widgets and unpacks files, so a partial install leaves a +space half-built with no way for the user to tell what is missing. (A +conversion or validation *tool* may keep going to report every offending file +at once, as `anyblockconvert` does, but it must still fail the run rather than +present its output as usable.) In a well-formed bundle every file carries the +same `version`; a bundle whose files disagree is malformed, and the reader +gates on the highest version it finds. + +A widget is flat — `{ target, layout, limit, view_id, auto_added, +card_style, icon_size, description, properties }` — though the wire carries +it as two blocks (see below). The members are the two blocks' own §5 +members, verbatim: `layout` (`link · tree · list · compact_list · view`, +defaulting to `link`), `limit`, `view_id` (which of the target's views a +`view` widget shows; omitted, the target's default view) and `auto_added` +(the client placed this widget itself and treats it as its own to manage) +are the widget block's; `card_style`, `icon_size`, `description` and +`properties` are the link block's display members, with the same +vocabularies — the schema states each as one `$ref` into the object schema +rather than a copy that drifts. `properties` keys resolve through the +bundle's property dictionary (§2f), the file that answers for stored keys, +since there is no per-document legend here. + +`target` is an object id from the bundle — a page, a type, a set, a +collection — or one of the eight reserved listings `_favorite · _recent · +_recent_open · _set · _collection · _all_objects · _chat · _bin`, which name +a built-in rather than something the bundle ships. The leading `_` is what +keeps the two kinds of target apart (§1): an object id from the bundle may +never begin with one, so a bundle cannot shadow a listing with an object of +its own, and a reader never has to guess which of the two a target meant. +The inventory is what live sidebars actually hold — measured over a 77-space +account, 33 of 218 widgets name a listing (chat 11 · bin 10 · allObjects 8 · +recent 1 · set 1) — and `widget.IsPredefinedWidgetTargetId` knows every wire +spelling, so all eight survive import. + +Two index-level members belong to the sidebar without belonging to any one +widget, and both are machine state the authoring subset refuses (§2g): +`auto_widget_targets`, the client's ledger of targets it has already +auto-added a widget for — usually naming widgets NOT in the sidebar any +more, which is the point: the ledger is what stops a restored client +re-adding what the user deleted (21 of 77 spaces carry one) — and +`auto_widget_disabled`, the per-space switch that turns auto-widgets off +entirely (2 of 77). + +A `_`-prefixed target that is not one of the eight is refused by name, with +the inventory in the message. It cannot be an object id, so the alternative +diagnostic — "no object with that id in the bundle" — would point an author +with a typo at the wrong repair. + +**A bundle carries no widget document.** The sidebar of a live space is a +hidden `kind: "widget"` object whose blocks encode exactly this array: one +widget wrapper block per widget, each with one indented link child naming +the target. Measured over 77 spaces the encoding is pure scaffolding — 218 +wrapper blocks, 218 link children, in perfectly regular pairs, plus the +header scaffolding §7 already drops — and every detail the object carries is +either lifted here (`autoWidgetTargets`, `autoWidgetDisabled`), constant +(`isHidden`, the dashboard layout), or the object's own timestamps, which a +restored sidebar re-mints the way a restored space re-mints its own. +So export lifts the object into these fields (`IndexFromWidgetObject`) and +omits the document — fail-closed, like the space document beside it: an +unpaired widget block, a target the index cannot spell (two strays in the +corpus, `bookmark` and `lists`, words no client defines), a non-empty name, +real page content, or any detail this package cannot account for keeps the +document, so a widget object carrying something unforeseen travels rather +than vanishing. The comparator consults the same predicates +(`OmittedWidgetObject`, `WidgetObjectResidualKey`), so the omission and the +round-trip check cannot drift apart (§11). + +### The manifest + +The index also says **where to find what a reader must resolve by key or id +rather than by walking**. The format defines no folder layout — `objects/`, +`types/`, `files/` are one exporter's convention (below) — and an object +names its type by *spelling* alone, so without a manifest a reader resolves +a type by scanning every document for a matching `internal_key`, and a file +document's bytes by guessing at a layout. + +```json +{ "manifest": { + "types": { "Task": "types/bafyrei….anyblock.json" }, + "properties": "properties.json", + "files": { "bafyreigp3him…": "files/bafyreigp3him….png" } } } +``` + +- **`types`** — the type's CANONICAL SPELLING → the type document's path. + The canonical spelling, not a per-document term: the display name from + the shipped table for a bundled type, the stored key verbatim for a + space-minted one — a pure function of the key, the same rule the + dictionary applies to its own entry keys (§2f), because the index has no + legend and its keys must resolve through the shipped ladder alone (an + exact stored key names itself, then the name table, then the fold). A + reader inverting a document's own spelling still goes through that + document's `type_internal_keys` legend first, as everywhere. + The manifest does NOT locate options. A manifest exists + to answer a lookup a reader would otherwise have to scan for, and no reader + has that lookup for an option: the dictionary states a property's whole + vocabulary inline — each option's name, colour, position and, since the + vocabulary learned `internal_key`, its stored key (§2f) — so everything an + option MEANS is in hand before a single document is opened. The map was + 2,641 entries across a 77-space export, pointing at documents nothing + needed to read. + + `option_ids` is unaffected and does a different job: it carries option + OBJECT ids, resolved against the IMPORTING space's live store so a value + survives a rename (§9a), never against the bundle. It never needed a path + beside it. +- **`properties`** — the property dictionary's path (§2f). A pointer rather + than an inline map, because properties resolve through each document's + own legend and the dictionary is the file that answers for the keys those + legends bind. +- **`files`** — file object id → the blob's path, one entry per + file document whose bytes travel. The authoritative binding between a + `kind: "file_object"` document and its bytes: the document itself carries no + path, because a document member is not a slot for archive bookkeeping — + the lesson of the legacy `source`-clobber, which overwrote a real, + editable `url` relation that bookmarks legitimately hold, and whose + destruction round-tripped through import. Every importer holding a file + document must find its bytes and every export tool must enumerate them — + and the map has that reader wired: `cmd/anyblockconvert` copies each + binding into the installable archive and writes the archive-side + `source` detail from it (the pb importer's own contract, resolved by + `normalizeFilePath`), and the future production native importer resolves + the same map. The clobber the format banished from its DOCUMENTS is + legitimate at the archive boundary — the archive is a transport + artifact, not a document. + Keys are object ids verbatim — no re-spelling on either side — and an + authored bundle writes `"files": {"logo": "assets/logo.png"}` against its + own minted ids and any layout it likes, which is what makes an authored + bundle a first-class citizen rather than a convention-follower. The + tooling's contract, precisely: an entry that cannot be honoured — a key + naming no document, a path escaping the bundle, a blob missing at the + path — is a cross-document REFUSAL; a file document a present map does + not bind is a WARNING (its bytes did not travel — the exporter writes + the document and omits the binding when a blob cannot be streamed, and + counts it, so a partial export is loud at both ends); a bundle with no + map at all is a metadata-only export, tolerated as a mode. + The bundle is FAT (§15 #20): the bytes travel, nothing + else — no variant keys, no encryption keys, and the thin bundle's future + marker slot stays untouched. + +Paths are relative to the index file. The reader flow, with no scanning and +no folder convention: object → `type: "task"` → the object's legend → stored +key → manifest → the type file → `property_definitions` → property not +there → manifest → the dictionary → the entry; a file document → `manifest.files` +→ the bytes. + +**This exporter's convention** (the "one exporter's convention" slot, +recorded so a reader of OUR bundles knows the layout without reverse- +engineering it; none of it is format — a reader must still walk and index, +because an authored bundle may put documents anywhere): + +- One bundle root per space; a multi-space export wraps each in + `spaces//`, and the wrapper is load-bearing — the same id + legitimately recurs across spaces (448 cross-space repeats measured, + chiefly participant identities), so flattening the wrapper collides. +- Every document is `/.anyblock.json`, the id verbatim — id→path + is a pure function of the reference itself, which is the naming + decision's whole point: a reference carries an id and nothing else, so + any name-bearing filename would force a scan. The kind directories are + `objects/` (kind `page`, flat — plus any kind without a dedicated home), + `types/`, `templates/`, `properties/` (kept `property` documents only; + the rest are omitted into the dictionary, §2f), `options/`, + `participants/`, and `files/`. +- `files/` holds both halves of a file, adjacent: the document at + `files/.anyblock.json`, the blob at `files/.` with `` + the stored `file_ext` lowercased and restricted to `[a-z0-9]{1,10}`, + else the conventional extension for `file_mime_type`, else `bin` — + a blob wearing the document extension is refused at plan time, and + the manifest map above, not the adjacency, is what binds them. + Putting file DOCUMENTS in `files/` is safe against the one reader known + to skip that directory: the legacy pb importer + (core/block/import/pb/converter.go) skips `files/` while walking a PB + archive — but it also parses every other `.json` file as jsonpb, which + refuses every AnyBlock document loudly, in every directory. A native + bundle fed to it fails whole, never partially; there is no path on + which the skip silently drops just the file documents while the rest + imports. Native bundles are read by the native wiring + (`cmd/anyblockconvert` → the experience path), which walks everything. +- `index.json` and `properties.json` sit at the bundle root; there is no + `profile` file and no root `index.` home special case — the + homepage is a field of `index.json` and the home object an ordinary + document under `objects/`. + +The manifest is optional — a bundle without one is walked the way every +bundle was before it existed — and closed: the index root already refuses +undeclared members (`additionalProperties: false`), and the manifest extends +that inside itself, because an undeclared lookup table is one no reader +opens. Whether its paths resolve is the same cross-document question every +other id in the index poses, answered by the tooling, not this package +(§13). + +### How it reaches the space + +A bundle is installed with `ObjectImportExperience`, which reaches +`builtinobjects.CreateObjectsForExperience`. That is a different path from the +one the built-in use cases take (`inject`), and it reads much less. The two +outputs the wiring produces, and who reads them: + +| output | written by | read by | +|---|---|---| +| `profile` at the archive root — `pb.Profile`, raw protobuf whatever format the snapshots are in, since `getProfile` reads it with `pb.Profile.Unmarshal` | `cmd/anyblockconvert` (`profile.go`) | `CreateObjectsForExperience` reads **`name`, `avatar` and `spaceDashboardId`** — on a NEW-space install; installing into an existing space reads none of it (the whole read is gated on `isNewSpace`) | +| a snapshot with `sbType: Widget` among the objects — one root block plus a wrapper-and-link pair per widget | `cmd/anyblockconvert` (`widgets.go`), built from `index.widgets` by `WidgetsSnapshot` — the same function the round-trip verifier holds against the widget object it omits, so the install artifact and the loss check cannot drift | the pb importer: `shouldImportSnapshot` admits a Widget snapshot precisely when the import type is `EXPERIENCE`, and `objectcreator.updateWidgetObject` merges its widgets into the space's own widget object | + +| `index.json` | reaches the space as | effect | +|---|---|---| +| `homepage`, falling back to `entrypoint` | `profile.spaceDashboardId` | the space's `homepage` detail — what opens on **every** entry, and on this path the only thing that decides what a new user sees | +| `widgets` | the Widget snapshot's root children, in order | the sidebar | +| `entrypoint` | `profile.widgets[0].targetObjectId` | the object the install opens **once** — on the `inject` path only. On a bundle's own path it lands only through the `homepage` fallback above | +| `name` | `profile.name` | the space's own name, when the install CREATES the space; nothing on an install into an existing space | +| `icon` (the `file` variant) | `profile.avatar` | the space's icon (the file object's id re-mapped to its imported id), under the same new-space gate | + +Five consequences worth stating, because none is obvious from the wire format: + +- **`profile.widgets` is inert here.** On a bundle's own (pb) path + `CreateObjectsForExperience` never calls `getWidgets` or `createWidgets`; + those belong to `inject`. (Its Markdown/AI branch DOES call + `createWidgets` — one link widget built from the manifest's dashboard + page, not from `profile.widgets`, which stays unread there too.) The + wiring still fills the field, so an archive it produces is also a valid + built-in archive, but nothing on a bundle's own path reads it. **The + sidebar comes from the Widget snapshot** — which an app export carries as a stored + object, and which this format's wiring rebuilds from `index.widgets`, + since a bundle carries no widget document (above). The snapshot's + `autoWidgetTargets` / `autoWidgetDisabled` details are inert the same way: + `updateWidgetObject` merges only the BLOCKS into the space's own widget + object, so the ledger reaches the archive truthfully but no importer reads + it back yet. The index is where the state survives. +- **`name` and the icon land only on a NEW space.** + `CreateObjectsForExperience` calls `setWorkspaceSettings(profile, spaceId, + true)` — but only inside its `isNewSpace` gate, so the profile's name and + icon become the created space's own identity and can never overwrite a + name the user already chose: an install into an existing space skips the + profile read entirely. +- **`entrypoint` is encoded as the first widget.** There is no independent + field for "open this after import" — `inject` takes + `widgets[0].targetObjectId` as its starting page, and the deprecated + `startingPage` is only read when `widgets` is empty, so it cannot coexist + with a sidebar. The wiring therefore has to make the entrypoint + `widgets[0]`, prepending a widget for it when the author listed something + else first. The entry point consequently always appears first in the + sidebar. `entrypoint` exists as a separate field anyway, because expressing + it by sorting `widgets` means reordering the sidebar silently changes what + a new user sees. + + On a bundle's own path even that does not fire: `CreateObjectsForExperience` + computes no starting page and `ObjectImportExperience` returns none, so + nothing is opened once. What a new user lands on is the space `homepage` — + which is why an omitted `homepage` falling back to `entrypoint` is what + makes the field mean anything at all here. +- **Omitting `homepage` does not mean the widgets screen.** An absent + `spaceDashboardId` makes `setWorkspaceSettings` default to `widgets`, which + is the right default for a *blank* space and the wrong one for a use case: + on desktop the widgets are already in the sidebar, so it leaves the main + pane empty. So an omitted `homepage` resolves to the `entrypoint` instead, + and only an explicit `"_widgets"` or `"_graph"` gives up a real page. + +**A widget target that does not resolve loses the widget, silently.** This is +the only reference in the format whose failure produces no diagnostic at all: +`common.handleLinkBlock` rewrites a link target it cannot resolve to +`addr.MissingObject`, and `WidgetObject.Init` then removes the broken link +*and* its now-empty wrapper. The import succeeds, the widget is not there, and +the only trace is a log line. An id no document in the bundle defines is +therefore an error in `anyblockvalidate` and `anyblockconvert` rather than +something an author discovers by installing — and so is a reserved listing +the importer does not recognise, a set that is empty today (the importer +knows all eight) and that the batch checker still guards, for the day a +listing is added to this format before the importer learns it. + +Nothing per-object substitutes for this file. In particular **`isFavorite` is +not an entry point**: it adds an object to Favorites and nothing more. It +does not open anything, create a widget, or set the homepage. + +Ids in `index.json` are the bundle's own — the same slugs every other +document uses — and the wiring relinks them like any other reference. Whether +they resolve is a cross-document question this package does not answer +(§13): an index validates on its own terms while naming an object no +document defines. + +## 2d. Property documents (`kind: "property"`, `"bundled_property"`) + +A relation object IS a property definition, and its document states what it +defines in **`property_settings`** — one `propertyDefinition` (§2e), in the +format's own vocabulary: + +```json +{ + "version": 2, + "kind": "property", + "id": "bafyrei…", + "internal_key": "budget", + "property_settings": { "format": "number", "include_time": false }, + "properties": { "Name": "Budget", "Description": "Planned spend" } +} +``` + +The three members are grouped rather than sitting at the document root, +because the +dictionary entry and a type's property-definition entry are groups holding +the same shape and two patterns for one idea is the §15 #14 disease one +level up. The group is a layer over `$defs/propertyDefinition`: the members +another surface already owns are refused with the home named (`internal_key` +is the envelope's — and `property`, the spelling, is refused with it: a +property document is addressed by its stored key and its spelling is derived, +never stated — `name` and `description` are `properties`', `options` are +property_option documents), and `max_count`/`readonly`/`default_value` keep +travelling in `properties` under their stored keys until the dictionary +lifts them — admitting a second spelling of any of those here would +reintroduce exactly the duality this section removed. + +**Two kinds are property documents**, and both carry the group: `property` +and `bundled_property`. Only the first comes out of a live store — 0 of +38,061 corpus documents are the other — but the `kind` enum offers it beside +`property` with nothing marking it non-authorable, and a small model +authoring from the schema alone picked it unprompted, walking straight back +into the bug this section exists to stop. The export gate, the import gate +and the schema's `if` therefore name the same two: a half that lifts for +fewer than the schema validates for emits a document its own Validate +rejects (§11 I1), and a half that reads back fewer drops the definition. + +Two neighbouring kinds are deliberately outside the set. `property_option`, +because an option document is a value rather than a property definition, so +`format` there is an ordinary custom property key. And **`sub_object`, +because it is deprecated** — a kind being retired must not acquire a new +obligation in a format about to freeze. Nothing observable turns on it (0 +corpus documents either way); what turns on it is not extending support to +something on its way out. + +`format` here may name **every** format a store carries, including `map` — +the shape of a hidden system relation's value, whose only carrier is the +bundled `templatePlaceholders` (72 production documents). The two AUTHORED +format slots may not: `property_definitions[].format` and a dataview's +`properties[].format` reference `authorableFormat`, which is the same +vocabulary minus `map`. A relation document has to be able to say what it +defines; a type or a view has no business declaring a property whose values +only the client writes, and none does — 0 of 19,862 type-property entries +and 0 of 28,034 dataview property entries carry it. + +Exactly **three stored details lift**, and no others: + +| stored key | `property_settings` member | shape | +|---|---|---| +| `relationFormat` | `format` | a §3 format NAME — **required**. Export refuses to write a relation whose stored format it cannot name (corrupt data only: `formatNames` is total over the model enum, test-pinned), because the fallback — writing `"text"` for a format that is not text — would import as a permanent silent format rewrite, the exact disease this lift kills. `"text"` resolves per key on the way back in, through the envelope `internal_key`, exactly as a property-definition entry's format does (§3): a bundled short-text relation keeps its stored format across a round trip. | +| `relationFormatIncludeTime` | `include_time` | `true` \| `false` \| `null`. Meaningful on `date` only; a `true` against any other format is a **warning**, carried unread. | +| `relationFormatObjectTypes` | `object_types` | the target **type keys**, in priority order — a type-key slot exactly like `property_definitions[].object_types` (§2a): the §3 type vocabulary, the same term ledger, the same `type_internal_keys` legend. Non-empty against a format other than `objects`/`files` is a **warning**. Meaningful entries: `[]` is a cleared target set, `null` a stored null. | + +**Presence mirrors presence.** Each member is present exactly when its +stored key is present, and carries its value — `false`, `[]` and `null` all travel +(80 production relations hold a null `includeTime`; 8,903 hold an empty +target list). This deliberately stops the §4 omit-empty canon at these three +members, and it is the opposite of §2b's emptiness carve-out, for a reason +worth stating: these fields are the property's *definition*, not decoration, +and §15 #14's verdict was to fix the SPELLING and leave the emptiness +collapse to its own change. Mirroring is what makes the lift a pure spelling +change — the same details go in and out, so the snapshot comparator +(snapshotdiff) needed **no new rule**, where §2a and §2b each cost one. + +**Why the envelope and not `properties`.** Inside `properties` every key is +a property spelling, so a bare `format` there means "a custom property named +format" — and that is a measured live bug, not a hypothesis: in a 198-run +small-model eval, 9 of 9 attempts wrote `properties: {"format": "number"}`, +it validated with no warning, and it imported as a phantom property, leaving +the relation with no `relationFormat` at all — longtext forever, silently. +The container was the problem, not the word. The §2b reasons apply +unchanged; the schema gates the group on `kind: "property"` and keeps it +illegal at every other root, so the same member name cannot be reclassified +by kind drift. + +**The three spellings are refused in `properties`** — under any spelling +that RESOLVES to one of the stored keys (§3), legend included, on every +kind, with the repair named: + +``` +/properties/Format: "relationFormat" is written on a property + document's envelope as "format": "" in property_settings (§2d), not as a + property +``` + +(`"Format"` is the stored key's wire spelling — its display name, §3; the +stored key `relationFormat` written verbatim trips the same refusal. The +retired slugs — `relation_format` and `property_format` — resolve +to nothing any more: a denied key's fold class answers nothing, so they are +ordinary custom keys that cannot trip it.) + +The refusal is derived from the export side's own lift list, never restated +(§2b's rule), and it is unconditional: a non-relation snapshot carrying one +of the three details drops it with a warning, because there is no §2d member +off a relation document to carry it — never observed, 0 of 27,444 +non-relation documents. And on a relation document, a `properties` member +spelling one of the three MEMBER names (`format`, `include_time`, +`object_types`) is a **warning**: it names a custom property, not the +relation's own definition — the phantom shape the 9-of-9 eval failures +wrote, which with the group's member also present would otherwise validate +in silence. A warning and not a refusal because the spelling is a legitimate +custom key (a media space really can have a `format` column) and a relation +object carrying one must stay exportable (I1). Refusing a key is not refusing to NAME it: a slot +that references the relation — the Property type's own property definitions and +dataview columns, in 64 production spaces — keeps the §3 spelling +(`"Format"`, the display name), because the deny rule protects the legend +and a bundled-bound spelling needs no legend entry. + +**Target types translate at the boundary.** The store keeps +`relationFormatObjectTypes` as type OBJECT ids +(`objectcreator.fillRelationFormatObjectTypes`); the document spells type +keys. The translation is the optional `TypeResolver` capability of +`Options.ResolveProperties` (storeresolver implements it from the same one +bounded type listing the §3 vocabulary budgets): export inverts id → key, import key → +this space's id, so a resolver-wired round trip is id-exact. A bare key +legacy imports stored directly (21 production entries) passes through +**verbatim in both directions**, its own address (§3): a key is vocabulary, +and a vocabulary miss is never evidence of nonexistence. What no longer +passes is an entry the space's own store disowns (§9): the +`_missing_object` sentinel, and an object id the wired existence capability +says names no row — 56 production properties carry one, type ids from the +account where a shipped use case was AUTHORED, and an object id differs in +every space while a type key does not. Both drop, the real id with a warning +naming it; `object_types` is a list, and a list expresses absence by being +shorter. Note the store answers for CORPSES too: an uninstalled type still +has a row and still inverts through `TypeKeyById` (its id names something), +so only an id with no row at all drops. Without any resolver the whole list +passes through verbatim and the offline round trip is byte-exact — an id +the store merely could not be asked about is still the stored value's +meaning, and a backup format that deleted it on export would be +disqualifying. + +Corpus facts the design rests on (38,061 documents, 10,617 relation +documents): every relation document carries `relationFormat`, so requiring +`format` refuses nothing real; the format distribution covers 14 of 15 enum +values (everything but `relations`=101), including `map`=102 on 72 documents +— all the bundled `templatePlaceholders` relation — which is why the §3 +vocabulary gained the name; `include_time` is `true` only on dates (543), +`object_types` non-empty only on objects/files (1,089 + 167); target entries +are 1,301 ids, 21 bare keys, 9 `_missing_object`. + +## 2e. One property, one shape (`propertyDefinition`) + +A property is described by **one shape**, `$defs/propertyDefinition`, +wherever this format describes a property. The shape has exactly **three +homes**, and no fourth: + +| home | shape | +|---|---| +| a property-dictionary entry (§2f) | one `propertyDefinition` | +| a type document's property-definition entry (§2a) | one `propertyDefinition` + `section` | +| a property document's definition fields (§2d) | one `propertyDefinition` | + +The shape's eleven members: `property`, `internal_key`, `name`, `format`, +`options`, `object_types`, `description`, `include_time`, `max_count`, +`readonly`, `default_value`. The first two are one identity split into its +two concepts — `property` the document-facing spelling, `internal_key` the +stored id the app mints — because one word carrying both meanings is the +§15 #14 disease this shape existed to end (§2a's entry table has the rules). It states no `required` of its own and stays open — each +home layers over a `$ref` to it, adds its own requirements, narrows what it +must, refuses the members another surface already owns, and closes itself +with `unevaluatedProperties: false`. A home may **narrow** a shared member +(an authored home pins `format` to `authorableFormat`; a type's +`object_types` is a real array, since only a relation's stored value can +hold a null) but never restate its shape: two statements of one member +agree today and drift tomorrow (§15 #14). + +The rule is test-pinned the way the format vocabulary is: the homes are +asserted to REFERENCE `$defs/propertyDefinition`, the way +`authorableFormat` is asserted to be `propertyFormat` minus `map` rather +than restated. On the Go side the codec threads the whole decoded +definition to the resolver's create path through one builder shared by both +doors the §2a array arrives by (the document, and the PATCH-type channel), +so a member the schema admits cannot be silently shed at the seam — a +document that validates and then quietly means less than it says is worse +than one the schema refuses. + +## 2f. The property dictionary (`properties.json`) + +A bundle names every property its objects use in **one file**, +`properties.json`, at the bundle root beside `index.json` and validated +against `properties.schema.json`. It is a sibling of the index and not a +section inside it, deliberately: an index says *where* things are, a +dictionary says *what they mean* (the manifest +belongs in the index because a manifest is what an index is). + +```json +{ + "$schema": "https://schemas.anytype.io/anyblock/2/properties.schema.json", + "version": 2, + "installed": ["Creation date", "Due date", "Tag"], + "properties": [ + { "property": "6a32d4856761631534b22f85", + "internal_key": "6a32d4856761631534b22f85", "name": "Budget", "format": "number" }, + { "property": "693c14f2aa11631534b22f01", + "internal_key": "693c14f2aa11631534b22f01", "name": "Owner", "format": "objects", + "object_types": ["Space member"] }, + { "property": "Due date", "internal_key": "dueDate", "name": "End Date", "format": "date" } + ] +} +``` + +**Why it exists, measured.** 10,617 of 38,061 corpus documents are +property documents (`kind: relation` when measured, `property` now) +— 5.8% of the bytes — and 9,675 of them are installed +copies of the 194 bundled relations, **98% field-identical to +`bundle/relations.json`**. Each spends a ~967-byte document, with its own +envelope, attribution and system properties, to restate `{key, name, +format}` a table every reader already ships. The dictionary replaces those +restatements: export omits a bundled relation document whose definition +matches the table (§11), and one key in `installed` stands for it. + +Two members: + +- **`installed`** — the BUNDLED properties present in the space, spelled by + canonical name (§3): presence, not definition. A restore reinstalls each key + from the reader's own bundled table. An installed copy that DIVERGES from + the table — a rename, a changed `is_hidden` (174 of 9,675 in the corpus: + 132 by `is_hidden` alone, 8 real renames) — keeps its relation document + AND gets a full entry here, which overrides the table member for member. A + key the reader's table cannot name is **skipped, not refused**: the + bundled table grows independently of the format version, so a backup + written by a newer app must stay readable one app version back. (The + writer has no such excuse — `MarshalPropertyDictionary` refuses a key its + own table cannot name, since it would tell the reader to install nothing; + the repair is a full entry, where the format travels along.) +**Precedence, when a property is described more than once.** The composition +puts a property's definition in up to three places at once — a dictionary +entry, a kept property document's `property_settings`, and a type's +`property_definitions` — and 273 kept bundled-key relation documents in a +77-space export also carry an entry, so the pair is ordinary rather than +exotic. The order is: + +1. **The bundled table**, for a key it names. It ships with every reader and + is the same in every space (§3), so no document can redefine a + bundled property — an entry for one DOCUMENTS it, and the tools warn when + the two disagree rather than accepting the entry in silence. +2. **The dictionary entry**, for every other key. It is the bundle-wide + statement, and the one an author writes when there is no relation + document at all. +3. **A type's `property_definitions` entry**, which narrows nothing and adds + only `section` — what THIS type does with the property, not what the + property is. +4. **A kept property document's `property_settings`**, which is the same + `propertyDefinition` and should agree by construction; where it does not, + the dictionary is the bundle's answer. + +The redundancy is deliberate: a type document is a self-sufficient authoring +unit (§2a), and the dictionary is what a bulk reader consults (§15 #14). + +- **`properties`** — one `propertyDefinition` (§2e) per property the + bundle's objects actually REFERENCE. **Used-only, not + everything installed**: a space installs a median 125 bundled properties + and uses 57 (47%), and the 68 nothing touches buy a reader nothing a + restore does not already provide. Space-minted properties appear here in + full — the dictionary is where an author declares a property without + writing a relation document at all, in the same vocabulary as a type's + `property_definitions` entry. + +**The dictionary ANSWERS for stored keys, and its entries carry both +halves of the identity.** A document spells a property by its display name; +its `property_internal_keys` legend binds the spelling to the stored key; +the stored key is what the dictionary answers for. An entry states that key +as `internal_key`, verbatim, and its `property` in the CANONICAL SPELLING +every other slot uses — the display name from the shipped table for a +bundled key (`"Due date"`, and `"Format"` for the key stored as +`relationFormat`), the stored key verbatim for a space-minted one (nothing +is ever derived from a bson id, and the dictionary has no legend, so its +spelling must be a pure function of the key — the only pure spelling a +space-minted key has is itself). +Entries need no legend of their own: an `internal_key` never resolves at +all, and a `property` spelling recovers its stored key through the ladder +below. An author states any one identity — `property`, `internal_key`, or a +`name` — and a custom property with no `internal_key` gets a fresh minted +one from the import wiring, like everywhere else in the format (§2a). + +**The reader flow, in full, and the step that is easy to miss.** A spelling +resolves in this order — the document's own `property_internal_keys` +legend; then a verbatim match against a dictionary key; then **the shipped +name table over the dictionary's own keys** (the same table every §3 slot +resolves through), with the forgiving fold behind it for near-misses and +legacy derived-slug spellings. That third step is not optional +garnish: §3's exhaustive rule writes a legend line only for a spelling the +bundled table does not bind, so a bundled property's spelling never gets +one — measured before the re-spell, over a produced 77-space export of +503,919 property value slots, 69.4% of slots resolved only through the +shipped table's step. +**Look up, never transform** — the name and the key say different words +("Creation date" / `createdDate`), so no derivation in either direction +exists; a reader holds the shipped table and asks it. + +**Every entry carries its `format`, and the schema requires it.** +Self-sufficiency is the constraint that shapes the dictionary: a +third-party reader must be able to interpret a backup WITHOUT shipping +`bundle/relations.json` — tell a date from a string, an option name from +free text. Dropping bundled relation documents with *no* dictionary was +considered and rejected for exactly this reason; it is the same "stands +alone" property that keeps a space id off the envelope. (`installed` keys +carry no format because they are not interpretation, they are restore +instructions — the definitions a reader interprets by are the entries.) +`format` resolves per key exactly as everywhere else (§3): `"text"` on a +bundled short-text key stays short text. + +A dictionary entry is the **third home** of `$defs/propertyDefinition` +(§2e), referenced across files by the published URL the way the index +references `plainIcon`, and closed with `unevaluatedProperties`. Its layer +narrows `object_types` back to a real array — only a relation's STORED +value can hold a null (§2d), and a dictionary describes a property rather +than mirroring a store slot. `section` is refused: it is the one type-owned +member, meaningless off a type document. One key, one slot: a key stated +twice — in `installed` or in `properties` — is refused on read and on +write alike, with the first occurrence named. + +`version` follows the same rule as every other file in a bundle (§2c, §10). + +The tooling knows this is not an object document, the way it knows +`index.json` is not: `anyblockbatch.DiscoverJSONFiles` excludes it, +`anyblockvalidate` validates it against its own schema (and warns — tools +warn, the codec tolerates — on an `installed` key the local table cannot +name), and `anyblockconvert` reads it as a declaration source (§3, the +import wiring). + +## 2g. The authoring subset (`authoring/*.schema.json`) + +The three published schemas serve two audiences at once, and they pull in +opposite directions. One is a **backup**: a full-fidelity round trip of a +real space, which needs ids, attribution, legends, minted internal keys, +provenance, derived state. The other is an **author** — layer 2 of §1, +increasingly an LLM agent — generating a use case from nothing: a few types, +some properties, a handful of objects. An authoring agent reading the full +object schema reads 56 KB of which most is noise it must actively ignore — +and worse, it IMITATES what it sees: a 251-run evaluation of small models +against this format showed them inventing bson ids because every example +carried one, and writing key fields whose only correct values a real space +mints. + +So each grammar also publishes an **authoring subset**, one schema beside +each full one: + + schema/authoring/object.schema.json https://schemas.anytype.io/anyblock/2/authoring/object.schema.json + schema/authoring/index.schema.json https://schemas.anytype.io/anyblock/2/authoring/index.schema.json + schema/authoring/properties.schema.json https://schemas.anytype.io/anyblock/2/authoring/properties.schema.json + +**A subset, not a different format.** Same `version`, same reader, same +wire: an authored document imports through the same `Unmarshal` an exported +one does, and the authoring URLs keep the trailing file names `DocumentKind` +dispatches on, so declaring one routes to the same reader. The invariant +that keeps the subset honest is that **every document valid under an +authoring schema is valid under the corresponding full schema and full +reader** — and it is a TEST (`authoring_test.go`), not a claim: a fixture +per structure the subset can express, an enum sweep that builds one document +per value the authoring schemas state, and a worked example, each pushed +through the full `Validate` and the real codec. The semantic rules of §12 +apply on top, unchanged — the subset narrows the grammar, never the checks. + +**What the subset removes** is everything whose value only a live space can +produce, or that a reader derives: ids on blocks, views, sorts and filters; +the three legends (§9a — an author writes spellings, and the legends exist +to bind spellings to STORED keys the author does not have); attribution and +timestamps; `internal_key` where it is app-minted; provenance and derived +state (`origin`, `revision`, `snippet`, `backlinks`, sync state — the +`properties` schema node refuses the spellings small models actually write, +so the phantom-member failure of §2d is an error at generation time, not an +imported phantom); the output-only surfaces of §4a (`store`, `root`, +`fields`, `source`, `groups`, `object_orders`); the input aliases +(`heading_4`, `equation`, `group`); and every kind an author never writes — +the subset's `kind` enum is `page`, `object_type`, `template`, nothing else. +Property documents are gone whole: the dictionary (§2f) is where an author +declares a property, and the import wiring mints the stored identity, which +is exactly what that split was built for. + +**Two survivals are deliberate, and both are the SPEC's own rulings.** The +envelope `id` stays — it is the bundle-local slug every cross-file reference +resolves through (§1: "the envelope `id` is not part of that trade"), so the +subset requires its shape instead of dropping it: no leading `_`, none of +the six reserved bare words (§1, §2c). And `internal_key` stays on TYPE +documents only, required there: a bundle's own type key is a slug the author +mints (`habit`), objects name the type by writing that key in `type`, and +the batch wiring resolves the two against each other by exactly that member +(§2a, `anyblockbatch.TypeIds`). + +**Each authoring schema is self-contained** — no `$ref` crosses a file, +where the full dictionary and index reference into the object schema (§2e, +§2b). That is a deliberate trade of the one-shape-one-statement rule for the +subset's whole point: an agent handed one file has the whole grammar for +that surface, with no 56 KB import. What keeps the restatements honest is +the same subset test that keeps everything else honest. One narrowing is +semantic rather than surface: the two counting date presets +(`number_of_days_ago`/`_now`) are not in the subset's enum, because where +they apply they REQUIRE a day count in `value` (§6.2), and a subset +admitting them bare would admit documents `Validate` refuses. + +**Two subset rules are stated on the RESOLVED property key, not in the +schema**, and they have to be: JSON Schema matches a member name literally +while the format resolves a property key case- and separator-insensitively, +so a literal rule over a key the codec spells many ways is not a narrower +rule — it is a rule with holes in it. Both had one. + +- **A type document names itself.** Written as `required: ["name"]`, it + REFUSED the canonical `{"Name": "Habit"}` and accepted only the retired + lowercase spelling. Any spelling that resolves to the name property + satisfies it now; a type document with no name at all still fails. +- **The subset refuses the app's own derived keys in `properties`.** The + schema's literal list still names the pre-raw-name spellings, and nine + keys — `creator`, `createdDate`, `lastModifiedBy`, `lastModifiedDate`, + `addedDate`, `revision`, `internalFlags`, `featuredRelations`, + `isArchived` — are exactly the ones the FULL format DROPS rather than + refuses, so nothing downstream caught them under a display-name spelling: + an author's value disappeared without a word where it used to be refused + at authoring time. Those nine are enforced on the resolved key. The + `layout_align` narrowing (an alignment NAME, not the stored number) had + the same defect and takes the same treatment. + +The schema keeps its literal list — it is what an agent actually reads, and +it carries both spellings of each — and a test pins the list against the +enforced set in both directions, so neither can rot again. + +`ValidateAuthoring`, `ValidateAuthoringIndex` and +`ValidateAuthoringPropertyDictionary` (§13) run the FULL validation first — +so refusals carry §12's curated wording — then those semantic rules, then +the subset schema, whose verdicts name themselves subset verdicts. The +semantic rules run before the schema because they can say which key was +written and why the app owns it, where a literal `not`/`enum` can only say +that some member matched. A nil return means the document is valid AnyBlock +JSON, not merely subset-shaped. + +**The worked example** lives at `testdata/authoring/habit_tracker/`: an +index, one type, a three-property dictionary, a welcome page and two +objects. It validates against the authoring schemas and the real codec, +warning-free, and its cross-file references are asserted coherent — it is +the bundle an authoring agent should be shown first, and the test is what +keeps it worth imitating. + +Sizes: object 56,105 → 33,690 bytes, index 8,845 → +4,003, properties 6,727 → 5,691 — the three surfaces together 71,677 → +43,384 (−39%), with every remaining `description` rewritten for an author: +short, concrete, saying what to write. The byte count understates the +narrowing where it matters to a generator: 3 authorable kinds where the full +enum offers 31, 23 block types of 39, 12 envelope members of 19, and no +output-only member anywhere — the test asserts that literally. + +## 3. Properties + +`properties` is a JSON object keyed by **property key**, always spelled by +the property's **display name** — `"Due date"`, `"Plural name"`, +`"Manual property"`, `"Publish Date"` — NFC-normalized, otherwise verbatim; +bundled, API-created and UI-created keys alike. One uniform rule, no derived +identifier anywhere in the format, no table a writer must classify against: +*a key is the property's name*. A reader never has to know which kind of key +it holds, and a writer never has to transform anything — the measured hazard +of key writing is the derivation step (models normalize names improvisationally +and inconsistently across documents; copying a name byte-exactly is a solved +behavior), so the format deletes the derivation instead of policing it. +(The api slug lives on as the API surface's own +addressing convention — a separate decision — and `apiObjectKey` is never +read by this format.) + +The mapping is a **table, both directions, never a string transform**: for +bundled keys the name table derived from the shipped +`relations.json`/`types.json` (which travels with every reader, so documents +still resolve offline), and for every other key the space's own display +names, which a node-backed reader primes from the space — and which the +document carries the inverse of, entry by entry (`property_internal_keys`, +below). "Creation date" says a different word than `createdDate`; no case +transform in either direction exists, and the package's tests pin that the +reverse is a lookup, never a derivation. + +**The spelling, and the two authorities it comes from.** A key's spelling is +its display name, and there are two places a name lives: + +1. **A bundled key spells the name in the shipped table** — `createdDate` + spells `"Creation date"`, `tag` spells `"Tag"`, and the relation TYPE + (stored key `relation`) spells `"Property"`, because that is its bundled + name. The table ships with every reader, so these spellings resolve + offline with no legend entry. Names in the shipped table are unique over + the wire-reachable population and never byte-equal another entry's stored + key — a CI guard holds that as the condition under which a bundled entry + may be added or renamed (the `audioGenre` "Genre" → "Audio genre" rename + is exactly that guard firing early). The nine hidden transients sharing + the name "Underlying file id" are the tolerated remainder: all nine are + stripped internal keys, and a shared name is refused as a spelling + outright, so the tolerance can never leak into a document. +2. **A space-minted key spells what its space names it.** NFC of the stored + display name, and nothing else: no case fold, no separator collapse, no + transliteration, no grammar escape. Only three inputs still yield no + spelling — an empty name, a name over the 128-character writable bound + (refused, never truncated: a truncation invents a spelling nobody chose), + and a name carrying control characters — plus the two member names §2 + refuses before any resolution (`id` and `type`, byte-exact). Each of + those degrades through the collision rule below to the stored key + verbatim, which is always its own address. A name that merely repeats the + stored key is no spelling either; the verbatim key already says that. + +Raw naming has no normalization step, so `"#"`, `"☕"`, `"C++"` and +`"50% done"` are each a valid property key exactly as written — a rule that +cannot fail needs no repair path. (The one normalization surviving in the package, +`refNameNormalize`, serves the informative `#name` reference suffix (§9), +which is a different surface with a `#`-free grammar to keep.) + +**A name is carried exactly as the space holds it** — edge whitespace and +invisible characters included (`'Email 📧 '` is a real production name). +Validation warns about both (§12) and never refuses or trims: one stored +name must not make an object unexportable, and a cleanup belongs where a +user creates or renames the property — one normalization, applied once, at +authoring time — not at the export seam on every write. The forgiving fold +below bridges the near-misses either way. + +**Collisions are resolved per DOCUMENT, not per space.** Names are not +unique, and the format does not pretend they are: a document carries a map, +and a map already guarantees its own keys are distinct, so a name that is +ambiguous space-wide but appears once in this document spells its plain +name. Measured, genuine in-document collisions are 60 of 28,560 documents +(0.21%), across five names. Where a document does collide — two properties +claiming one spelling, or a name equal to a stored key the document names +(verbatim-first: a stored key always keeps its own term) — **every claimant +degrades**, deterministically, through one ladder: + +- **(a)** the stored key verbatim, when it is itself readable (not a minted + 24-hex bson id) — the `producer_region` / `wine_region` shape; +- **(b)** else ` ()`, tail6 = the stored key's last six hex — + deterministic, immutable while the key lives, visibly synthetic; +- **(c)** a residual tie (two claimants minting one suffix, or a suffix the + document already speaks for) falls to the full stored key, which is always + its own address. + +All claimants degrading — rather than first-claim keeping the plain name — +is what makes the suffix stable across exports and the plain name +trustworthy: a plain spelling in a document is never one of two same-named +claimants. A suffixed spelling never moves while its neighbours live; +deleting one claimant un-suffixes the other on its next export — cosmetic +churn, correct via the legend. + +**A claimant is a key this document actually writes**, and two populations +look like claimants without being one. Both are carved out for the same +reason: a claimant that will not be there next time must not decide anybody +else's spelling, or a second export of the same object differs from the +first and the round trip stops being a fixpoint (§11). + +- **A key the `properties` emit drops** — a type document's install + provenance, a participant's load timestamp, an admitted system-stamped key + whose value is empty, a name-over-number key holding a string its + vocabulary cannot name — is written nowhere, so it is not counted at all: + it claims no spelling and reserves no stored key. `isHidden: false` beside + a custom property named "Hidden" used to write `Hidden (b90aa1)` on one + export and `Hidden` on the next. +- **The attribution keys** are the opposite case: export WRITES them and + import drops them, so they occupy a member of this document and none of + the next one. They **yield** — alone on a spelling they take it as usual; + contested at all they take their own stored key, which is always readable, + and the normal claimants keep the verdict they will re-derive once the + attribution line is gone. + +**The map-less reader resolves a shared name within the declared type.** An +authored document need carry no legend, so a reader handed a bare name that +several live properties answer to resolves it against the declared type's +own property list first. Unambiguous there — the overwhelming case, measured +at 1 ambiguous type of 1,753 — and it is resolved. Ambiguous even within the +type, or absent from it, and the reader raises a loud, actionable error +naming the term and asking for the `property_internal_keys` entry that would +settle it. It never guesses between live properties and never mints a +phantom key while two live properties bear that exact name. (A term NO live +entity answers to still resolves verbatim — chain step 5 below — with a +warning; that is the price of any name-addressed scheme, stated in §11.) + +Three consequences worth stating outright: + +- **Non-Latin scripts are kept, never transliterated.** `Тоггл` is `Тоггл` + and `日本語のプロパティ` is itself. The api slug's transliteration exists + because a slug is a URL path segment there; it would answer `toggl` and + `ri_ben_yu_nopuropatei` here — unguessable and unreadable at once, which + is strictly worse than either the name or the key. The measured + degradations are not merely lossy but wrong: `作業内容` (Japanese) + transliterates through Chinese readings. +- **The name is the address, so a rename moves the spelling — and the + legend keeps every written document resolvable.** A spelling derived from + a name changes when the name changes, and the next export writes the new + one; the stored key never moves, and the `property_internal_keys` line + every non-bundled key carries binds the exported spelling to it, so a + document written under "Budget" imports correctly after the property + becomes "Cost", and a new property later named "Budget" cannot capture the + old document's values. What the legend cannot protect is the legendless + (hand- or agent-authored) document, whose stale name misses silently and + mints a phantom key — accepted as the price of any name-addressed scheme, + mitigated by the unknown-term warning (§12) and by one measured + consolation: the likeliest bundled guesses land through the fold + (`created_at` misses under every scheme, but a guessed `"Created Date"` + folds onto `createdDate`'s class and resolves). +- **A spelling that is already answered is not up for grabs.** A live stored + key outranks any name (verbatim-first, below), so a name byte-equal to + another live stored key degrades through the collision ladder; and `id` + and `type` are never minted as property spellings because §2 refuses those + two member names before any resolution. A custom property MAY share a + bundled name — "Description", "Priority" and "Emoji" all have real custom + twins in production — because the legend and the per-document ladder keep + both addressable; a shared spelling with no legend is exactly what the + type-scoped resolution above answers, loudly when it cannot. + +An **absent** `format` in either slot that carries one (`property_definitions[]`, +a dataview's `properties[]`) says the document did not speak, and the §3 +chain answers — the bundled table, then the caller's resolver. It is NOT a +declaration of `text`: that reading silently overrode the table, so +`{"property": "Due date"}` in a dataview's list pinned a bundled DATE +property to longtext and its filters stopped being dates, while omitting the +list entirely resolved correctly. Canonical export always writes a format, so an +absent one only ever arrives from a hand-written document — the population +that means "I did not say". + +**Resolution — one rule, stated once, covering both namespaces.** The +format names keys in two namespaces — property keys and TYPE keys — and +every key slot lands its term on a stored key through the same chain, first +answer wins, run against the slot's own namespace: its legend, its half of +the bundled table, its stored-key set. + +1. **The document's own legend** (`property_internal_keys` for property slots, + `type_internal_keys` for type slots — identity entries included) — the only + statement the *document* makes about its spellings. +2. **An exact stored key — verbatim-first.** A term that names a stored key + means that key, always; the name tables apply only to terms that are + *not* stored keys. A node-backed reader answers this step from its store + (`storeresolver`, both namespaces); a package-only reader has no + stored-key set and knows a term is a stored key only when the legend says + so — which is why export owes the identity entry below for every term the + bundled table does not bind to the key being written. +3. **The name tables**: the bundled name table, which ships with every + reader, and — for a node-backed reader — the space's own names, where + EXACTLY ONE live entity answers to the term. Several answering is an + ambiguity this step refuses: the type-scoped resolution above, or the + loud error, is what happens next — never a guess. +4. **The forgiving fold**, answering only when exactly one candidate + remains: NFC, casefold, trim, strip default-ignorable code points, drop + `_`, `-` and spaces. This is the near-miss layer, and it is also the + whole of legacy continuity: ToSnake only inserts `_` and lowercases, so + fold(ToSnake(key)) == fold(key) by construction and every pre-change + derived-slug spelling (`created_date`) lands in its stored key's fold + class with no compatibility table; `due_date_2` bridges to "Due Date 2" + the same way. A DENIED key's fold class answers nothing, deliberately — + forgiveness toward a key import refuses would turn the phantom-member + warning on `format` and `include_time` into a refusal. (The sixteen + retired alias spellings — `featured_properties`, … — are outside this + proof and are cut, not kept: pre-freeze, no back-compat is owed, and + existing bundles re-export under the names either way.) +5. **Verbatim** — the term *is* the stored key, which is what keeps a + package-only reader — with no space to ask — lossless on custom keys. + With a space-backed vocabulary in force, a verbatim term that is no live + entity's stored key draws a warning (§12): the stale-or-guessed-name + phantom, every naming scheme's shared hole, named at the seam. + +A conforming document resolves identically in every conforming reader: +steps 1, 3(bundled), 4 and 5 need nothing but the document and the shipped +table, and wherever the shipped table cannot answer for a term the document +itself uses, the document carries the entry that moves the answer into step +1 — so step 2 and the space half of step 3, the steps that need a store, are +never load-bearing for a document's own spellings. Every other statement of +resolution order in this document is shorthand for this chain. + +The namespaces are **disjoint claim domains**: a property and a type may +share a spelling without conflict (a space may name a relation and a type +one word, and `objectType` the stored type key coexists with `object_type` +the layout value below), which is why the legends are two maps and export +runs one term ledger per namespace — a shared domain would back a key off a +spelling the other namespace owns. + +**The document carries its own inverse: `property_internal_keys`.** The name +layer is a re-spelling of key identity, and like every compaction in this +format it has to be invertible from the document alone — the rule §9a +already states for object ids. A name the space minted is not: +`6a32d485…` spelled `"Priority"` reads back through the bundled table — a +different relation — in any reader that cannot ask that space, silently. So +export writes the entry: + +```json +"property_internal_keys": { "Priority": "6a32d4856761631534b22f85" } +``` + +- **Emitted for every spelling the bundled table does not bind to the key + being written.** One condition, two halves, and they ask different + questions: the bundled table must **bind** this spelling to this very key + (it ships with every reader, so `"Due date"` → `dueDate` owes nothing), + *and* the vocabulary in force must **invert** it (a reader may bind a + spelling the bundled table binds correctly, and the writer's own space is + the reader most likely to read the document back — a space holding a + custom twin of a bundled NAME cannot uniquely invert that name, so the + bundled key's own usage carries the entry there too, which is what keeps + the document self-resolving in the one space that is confused about it). + + The asymmetry is what makes the rule exhaustive. A term that is a stored key + written verbatim trivially *inverts* through any table, because a table that + does not know a term answers the term itself (chain step 5) — so asking the + bundled half as an inversion let every custom key pass with no entry at all, + and the document said nothing about the one population no reader can resolve + without it. That silence is the **corpse-after-export** hole: the key is + live and unambiguous the day it is written, and the moment the relation is + UI-deleted its stored key stops being live while its freed NAME becomes + another live relation's spelling. Every legendless line already written + re-points, offline, and no writer could have warned about it — the delete + happened afterwards. Only the document itself can close that, so a spelling + the bundled table does not bind owes an entry, verbatim or not. + + **The identity entry is therefore the ordinary line, not the exception.** + Every custom key names itself: `{"customStatus": "customStatus"}`. Two + shapes that used to be called out as special are just instances of the one + rule now. + + The first is the bundled *shadow*: a space whose relation is keyed with + the literal string of a bundled key's fold class — `due_date`, beside + bundled `dueDate` — exports + `"property_internal_keys": {"due_date": "due_date"}`: the document's only + way to tell a reader with no store that the term is a stored key (chain + step 2). Without it, the fold silently moved the value onto the bundled + twin in every package-only reader. + + The second is **the vocabulary in force**, which is the half that stays an + inversion, and it stays for a measured reason: dropping it — "ask one table, + not two" — loses `{"task": "task"}`, and a template comes back pointing at + an unrelated custom type; and loses the entry that keeps a bundled name + addressable in a space holding its custom twin. A vocabulary is consulted + *before* the bundled table (chain steps 2–3 are a node-backed reader's + store), so a term the bundled table inverts correctly can still be bound + elsewhere by the reader most likely to read the document back: the + writer's own space. This is not a hypothetical about hand-written + vocabularies — it is what a **delete** produces. A UI-deleted type or + property vacates the name namespace while every object it ever named + keeps its stored key, and its freed name becomes another live entity's + spelling: `initiative` stops being a live stored key while a live type is + NAMED "initiative", so `"type": "initiative"` written with no entry came + back as that other type, silently. The property namespace produces the + same fault one ladder rung later — the live twin takes the suffixed + spelling and both terms carry their entries. Export therefore asks both + tables, and writes `{"initiative": "initiative"}` when either would + answer something other than the key being written. The entry is + authoritative for *every* reader, which is the point of a legend; what it + cannot cover is a reader whose vocabulary disagrees with the bundled + table in a way the writer never saw, and that is the `KeyVocabulary` + precondition (§11), not a legend rule. + + The legend is therefore empty for a document whose every spelling is + bundled, and costs one line per non-bundled key otherwise. **Size**: the + four golden documents, which each carry two custom keys, grow 93 bytes — + about 2%. The adversarial corpus, where every document carries five or more + custom keys, grows up to 15%; that is an upper bound, not an estimate. The + product's store-backed path pays close to nothing new, because a + store-minted relation key is a 24-hex bson while its spelling is the + display name, so spelling ≠ key and the entry already existed. +- **Consulted first, before any vocabulary.** The legend is the only statement + the *document* makes about its own spellings; a vocabulary belongs to the + reader, and two readers disagreeing about a spelling is exactly how a + property ends up naming a different relation than it was exported from. +- **It covers every key slot, not just `properties`.** Wherever the format + names a property — a `property` block's `property`, a link block's + `properties` list, a dataview's `properties[].property`, a view's + `group_by`/`cover_property`/`end_property`, a filter's, sort's or + column's `property`, a property-definition entry's `property` — the + spelling is written through the same recording step and read back through + the legend first. A slot that writes the spelling without recording the + entry inverts only when some *other* slot in the same document happened to + record it, which is luck rather than a guarantee; a slot that reads + without the legend never inverts at all, even when the entry is right + there. +- **One term, one key — document-wide.** Export claims every spelling + through a single term ledger, exactly as ids go through one id domain + (§4): a stored key the document names *anywhere* always keeps its own term + (verbatim-first — no other key's name may take it), an uncontested + spelling goes to its claimant, and a contested one degrades EVERY claimant + through the collision ladder above — computed once from the document's own + key census, so which spelling a key gets never depends on which slot + happened to claim first. The discipline covers every key slot, not just + `/properties` — a `property` block whose spelling collided with a + `/properties` spelling used to record a legend entry that rebound the + term, so that property's value landed on a different relation, silently; + and two blocks sharing one spelling collapsed into naming one key. +- **A legend value is a stored key, and is admitted like one.** It obeys the + writable-key rule — non-empty, no control characters, at most 128 + characters, the same shape rule property names carry, enforced by the + schema — **and the §3 deny rule**: a value naming an internal key + (`uniqueKey`, `oldAnytypeID`, `spaceId`, `id`, …) is refused, by + validation and import alike, whether or not any member spells the entry. + The legend is step one of resolution, so an unchecked value was a + laundering primitive: it could bind any harmless spelling onto a key + admission refuses — in a key slot outside `/properties`, without admission + ever seeing it. + + **Export admits an entry before it records one**, and drops the entry, with + a warning, when it cannot. Two guards were supposed to cover this and both + had the same hole: a denied key never takes a spelling, and an unwritable + spelling is never written — but a key with *no* spelling at all skips both + checks, and the term that reaches the ledger is then the raw stored key. So a stored + key of 140 characters, or one carrying a newline, or an internal one, + reached the legend as an identity entry the moment the vocabulary in force + bound its spelling elsewhere; `Marshal` emitted a legend its own `Validate` + and `Unmarshal` reject, and the object became unexportable with nothing + said. Reachable through `Options.Keys` alone, which this format accepts + from anyone. + + Dropping the entry is the smaller loss, and it is not a loss of content: the + term is written **verbatim** either way — the ledger backed it off to the + stored key long before this point — so the object still round-trips through + any reader that reaches chain step 4. What it gives up is *portability for + that one key*: a reader whose vocabulary binds that spelling elsewhere has + no statement in the document to override it with. Such a key has no writable + spelling anywhere in this format, so no legend entry could have been written + for it under any rule; the warning names it. +- **A property key slot carries the writable-key rule wherever it is, + including where it is a JSON string VALUE.** `/properties` and the legends + are member names, so the schema states the rule as `propertyNames`; a + property-definition entry's `property` (§2a) is an ordinary string value the + schema can only reach as one, and for a while `minLength: 1` was the only + bound it had — a 140-character key, or one carrying a newline, validated + clean and then failed to import. The rule is the namespace's, not the + slot's: `/properties` is the property namespace's home surface and cannot + express a key that is not a member name, so a property with such a key + cannot appear in a document at all, and a slot that could carry one would + be offering an address the rest of the format has no way to use. (The type + namespace answers the same question the other way, and for the same reason + — its home surface is `type`, a value. See its own rules below.) Export + drops a type-property entry whose stored key is unwritable, with a warning, + rather than emit one the seam refuses. +- **A key slot has to name something — at every slot, through every door.** + This is the one rule that binds *all sixteen* key slots (twelve property, + four type), and it is the minimum: it says nothing about length or charset, + only that a slot which names nothing names nothing. Three doors carry it. + + **The document.** Every key-slot string is `minLength: 1` in the schema. + Only `/properties`, `property_definitions[].property` and `property_definitions[]. + object_types[]` used to be; the other thirteen took an empty spelling from a + plain document, no vocabulary needed, and then LOST the slot on the way back + out, in silence: a column and a sort vanish, a property block and a link's + shown-property list come back nameless, a filter re-exports as a node that + filters on nothing, and `"type": ""` costs the object its type. A dataview + filter also has to *carry* the member — `required: ["property"]`, as its + sibling sort and column always have — and validation states that rule in its + own words, because the schema can only state it inside a `oneOf` and the + branch that fails takes the other branch's whole verdict with it. + + **Export.** A filter and a `property` block whose stored key is empty are + **dropped**, with a warning, which is what the sort and the column beside + them have always done with the same input. Written out they were nameless + nodes: the schema accepted them, import stored the empty key, and the next + export wrote them again — forever, meaning nothing. + + **The import seam.** A vocabulary answering `("", true)` for a non-empty + spelling is refused at every slot. `/properties`, `type`, `template_for` and + `object_types[]` refused it from the start; the other nine stored it. The + refusal names the *spelling*, because the fault is the reader's table rather + than the document, and that is the fact a caller can act on. + + What this rule deliberately does NOT do is bound length or charset at these + slots. See the two bullets below: `/properties` cannot express such a key + because it is a member name, and the primary type slots stay unbounded on + purpose — bounding them would make a stored key unexportable, which is a + larger loss than the one it would prevent. +- **The legend cannot launder a spelling onto an internal key.** Entries are + honored during validation and admission exactly as during import — the + legend is step one of key resolution — so `{"prio": "uniqueKey"}` does not + smuggle a `uniqueKey` write past the §3 deny rule twice over: the entry + itself is refused (previous bullet), and the *resolved* key is what + admission judges regardless (see below). Conversely, a legend entry that + binds a denied SPELLING to a harmless stored key (a custom property may be + NAMED "Format", and an identity entry for a shadow stored key is exactly + this shape) is honored: nothing lands on the internal key, so nothing is + refused. + +**The type namespace carries the same inverse: `type_internal_keys`.** Everything +above holds with `type_internal_keys` for the legend, the type half of the bundled +table, and the type slots — the envelope `type` and `template_for`, and +`property_definitions[].object_types` (§2, §2a). Export claims type spellings +through a term ledger of the namespace's own, seeded by the same census +(every stored type key the snapshot or the resolved type-property +definitions name), and writes identity entries under the same trigger: a +custom type stored as `object_type`, beside bundled `objectType`, exports +`"type_internal_keys": {"object_type": "object_type"}` or a package-only reader lands +on the bundled twin. Four rules are the namespace's own, each from what a +type key is — and one rule above that deliberately does **not** carry over. + +- **No duplicate-binding refusal.** `/properties` refuses two spellings that + bind one stored key; the type namespace admits them. + `{"kind": "template", "type": "a", "template_for": "b", "type_internal_keys": + {"a": "template", "b": "template"}}` validates, and yields `ObjectTypes: + ["ot-template", "ot-template"]`. The property refusal exists because two + members collapse into one details field, so one of the two values is lost + with nothing to say which — a document that means two things and stores + one. Two type entries collapse into nothing: `ObjectTypes` is an ordered + list, a repeated entry is a repeated entry, and no value is displaced. + Refusing here would buy nothing and would refuse documents that lose + nothing. +- **No deny rule** — and the reason is not that the type namespace is + harmless. The property deny rule is *import refuses exactly what export + strips*, and export strips no type KEY: what it drops is positional (the + entries past the slots §2 models, and keyless entries, both below), never a + particular key, so the derived set is empty. The stronger reason is that a deny + rule here would guard nothing: **every effect a document-chosen type key + can produce is separately, and more directly, writable through the + property namespace.** Layout — `"type": "participant"` reaches + `resolvedLayout` through that type's own `recommendedLayout` — is + reachable as `{"properties": {"layout": …}}`, and `layout` is the FIRST + thing the resolver that computes `resolvedLayout` consults, above the + type's answer. There is one place a type key selects a code path in the + import wiring — a legacy `sub_object` document, whose first object type + picks which real kind it migrates into — and all that path does is set the + smartblock kind, which is `kind`, and fill in `sourceObject` when the + document left it empty, which is `{"properties": {"Source object": …}}`. + And merge resolution never reads the type list at all: the importer + derives a document's identity from `kind` plus the envelope `internal_key`, and + from `unique_key` — never from the object types. + + Merge resolution *is* steerable, but through the **document's own + fields**, not through type keys. `name`, `relationKey` and + `sourceObject` are ordinary writable properties, and the relation's + format — now the envelope's `format` (§2d), where it is + just as writable and lands on the same stored detail — travels beside + them; the importer uses them to pick which existing object a document + merges into: a relation matches on its format together with `name` or + `property_key`, and a TYPE document matches on `name` alone, since this + format strips `unique_key` and the name is then the only filter left. + They stay writable deliberately — the §2d lift moved a spelling, never a + capability, exactly because a stripped value that import refuses is a + lossy export and "Marshal never emits a document its own Validate + rejects" (§11, I1) is the stronger promise. The guarantee that an + imported document cannot rewrite an EXISTING relation's or type's + identity therefore belongs at the object layer, which every writer passes + through, rather than in this format, which is one writer among several. A + `type_internal_keys` value is admitted by shape alone — the writable-key rule the + schema enforces on both legends. +- **The primary type slots are unbounded, on purpose.** A `type_internal_keys` + spelling and a `type_internal_keys` value both carry the writable-key rule (1–128 + characters, no control characters) — the first because it is a JSON + member name, the second because it is a legend value like any other. The + envelope `type` and `template_for` carry neither: no pattern, no length + bound. A term written there is a JSON string *value*, so the member-name + shape rule does not bind it, and a non-empty stored type key of any shape + round-trips verbatim. One consequence is worth naming rather than fixing: + a type key containing `-` yields the object-type unique key `ot-a-b`, + which does not parse — a unique key is at most two `-`-separated parts — + so such a type is invisible to a space-backed vocabulary, which reads its + stored type keys back out of `unique_key`. It still round-trips through + this format verbatim, and that is the point: the format carries what the + store holds; which of the store's keys the rest of the system can address + is not its ruling to make. The one thing refused here is the **empty** type + key, in both its forms — the literal `"type": ""` (schema `minLength: 1`) + and a vocabulary resolving a non-empty spelling onto nothing — because it + would store the unwritable `ot-` and re-export as no type at all, silently. + That is not an exception to "unbounded on purpose": an empty string is not + a stored type key of any shape, it is the absence of one. +- **No reserved spelling.** No type key is a reserved word — including + `template` — because *which type an object has* (`type`) and *what kind + of smartblock it is* (`kind`) are two separate fields, and the §3 chain + never touches `kind`. Two checks remain, resolving nothing on their own: + export keeps `kind` explicit whenever the type term it is about to write + is literally `template`, and `Validate` refuses a document with no `kind` + whose `type` is literally `template` (§10). +- **Export writes only the slots §2 models, and says what it drops.** The + envelope carries one type, plus — on a TEMPLATE — the target type; entries + past those are not written. An entry with **no key** — a stored `ot-`, + which older builds wrote whenever a vocabulary resolved a spelling onto + the empty key — has no spelling at all, so it is dropped and the entries + behind it move up. Written in place it was contagious: it silenced the + slot it landed in, and a silent `type` slot makes `template_for` + inexpressible, so `["ot-", "ot-task"]` exported as no types at all and the + good entry died beside the bad one. **Both** kinds of drop are reported + through `OnWarning`, as an unwritable property key is — the keyless entry, + and the keyed entry the positional truncation leaves nowhere to go, each + naming the position it stood in among the snapshot's object types. The + truncation is the format's shape rather than a fault, but it is still a + type the caller holds and the document does not, and nothing in the + document says so. And only the slots actually + written claim a term, so the legend names only types the document + mentions: claiming a term is what records the legend entry it owes, and + slugging an entry no slot writes published a space's slug→key mapping in a + `type_internal_keys` line naming a type the document never spells. + + **The census sees the same list.** Verbatim-first reserves every stored type + key the document NAMES, and reserving more than that is not merely + wasteful: a key no slot spells backs another key's slug off, so the same + object exported before and after a round trip through this format produced + two different documents — one with the stored key, one with the slug and a + legend line to invert it. `["ot-custom1", "ot-cust"]`, with a vocabulary + spelling `custom1` as `cust`, is the whole shape. So the census runs the + reduction above, and asks of a type property exactly what the emit asks: + will this entry be written? Nothing is lost by the narrower reservation — + a key the document never names cannot be taken as another key's spelling by + a reader who never sees it. + +**What is not a key slot.** The vocabulary applies where +a document NAMES a type or property, and nowhere else. Envelope and DTO field +names, enum *values* (`kind: "object_type"`, layout and view-type names), the +`index.json` envelope, view field names like `default_template_id`, and — the +one most easily mistaken for a key — **block attribute names**: a callout's +`icon` and its `format`/`emoji`/`file` members are attributes of a block, not +property keys. They are the format's own vocabulary and follow the format's +own rule (§1 Naming, all snake_case); the vocabulary never touches them, so +they would keep their spelling whatever any *property* were called one +section over. The layout VALUE `object_type` coexists with the type key +spelled `object_type` — one is an enum this format defines, the other is a +name in the space — and that is intended. + +Values are encoded by the property's format: + +| Format | JSON encoding | +|---|---| +| `text` (default), `url`, `email`, `phone` | string | +| `number` | number | +| `checkbox` | boolean | +| `date` | RFC 3339 date-time string, UTC (`"2026-07-06T15:04:05Z"`); import converts back to unix seconds. Import also accepts date-only strings (UTC midnight), non-UTC offsets (converted to UTC), and fractional seconds (truncated to whole seconds). Export always writes the full UTC form — **except** for a stored value outside the years RFC 3339 can express (0000–9999), which export writes as the **raw number**, with a warning. There is no string form for such a value, and writing one anyway (`"57482-01-22T22:43:20Z"`, from milliseconds stored where seconds belong) would not parse back, so the value would return as a *string* on a date property and stay one. A reader must therefore accept a number here; the number is a stored value it cannot interpret as a date, not a second date encoding. | +| `select`, `multi_select` | array of option **names** (strings) — see below | +| `objects`, `files` | array of object ids (strings). A resolver-wired export drops an entry the SPACE does not hold — the stored `_missing_object` sentinel included — and the emptied list stays `[]`, because the key's presence is meaningful; a package-only export drops nothing (§9) | +| unresolvable format | value passes through verbatim in both directions | + +**Enum-valued properties are named, not numbered.** Seven stored keys hold +numbers whose meaning is a proto enum (their bundled relations have format +`number`), and the format writes the enum **name** — a bare integer would +be an opaque enum in an otherwise self-describing format. Each key's +vocabulary, one table per concept (`namedEnumProperties`): + +- `recommendedLayout`, `layout`, `resolvedLayout` — the object layout: + `basic · profile · todo · set · object_type · property · file · + dashboard · image · note · space · bookmark · property_options_list · + property_option · collection · audio · video · date · space_view · + participant · pdf · chat_deprecated · chat_derived · tag · notification · + missing_object · devices · discussion` (`$defs/objectLayout`). +- `layoutAlign` — the object's own page alignment: `left · center · + right · justify`, the SAME vocabulary a block's `align` and a view + column's `align` spell (`$defs/blockAlign` — one definition, three + slots, §15 #14). +- `origin` — how the object entered its space: `none · clipboard · + drag_and_drop · import · webclipper · sharing_extension · usecase · + builtin · bookmark · api` (`$defs/objectOrigin`). Real provenance, kept + on ordinary objects (the §2a admission dropped it from TYPE documents + only, as install provenance) — and all ten values occur in real data. +- `importType` — which importer created an import- or usecase-originated + object: `notion · markdown · external · pb · html · txt · csv · + obsidian` (`$defs/importType`). Named or refused, never a stray string: + the underlying enum's ZERO is notion, so an unchecked string here read + back as a false claim that the object came from Notion. +- `imageKind` — what an image object is used AS: `basic · cover · icon · + automatically_added` (`$defs/imageKind`). Stored on 4,079 corpus + documents; named for the same reason as the rest, since a bare integer + would be an opaque enum in a self-describing format. + +Import maps a name to its number and still accepts a raw number, so older +documents keep working; export always writes the name for an in-vocabulary +number and the raw number for anything else — a stored value outside the +vocabulary round-trips as its number rather than being lost. An +unrecognized NAME is a validation error stating the vocabulary, because +the silent alternative was measured and bad: the string imported onto the +number-format detail and every consumer reading it with an int getter saw +the enum's zero. The property slots' vocabularies are enforced by the +semantic pass on the RESOLVED key, not by the schema — a property SPELLING +is not fixed to its stored key (a legend may rebind it, above) — so the +schema states each vocabulary in `$defs` for the reader and the semantic +pass owns the refusal. On the way out the same rule binds export: a stored +STRING a vocabulary does not name has no written form and is dropped with +a warning (written verbatim it was a document Marshal's own `Validate` +rejects, §11 I1), while a stored string that IS a name survives and reads +back as the number. + +The remaining layout-ish bundled keys stay numbers deliberately: +`layoutWidth` is a fraction, not an enum, and `widgetLayout` / +`headerRelationsLayout` hold enums too marginal to earn a name vocabulary — +13 and 51 occurrences across 28,604 real exported documents. (The 51 was +first miscounted as 0; the corrected count changes the evidence, not the +verdict — a name table is bought for keys models actually write, and +neither key is one.) + +Format names follow the public REST API (`select`, `multi_select`, …); +internally they map to `model.RelationFormat` (`status`→`select`, +`tag`→`multi_select`, `longtext`→`text`, +`object`→`objects`, `file`→`files`; `emoji`, `properties` and `map` exist +for internal formats). The vocabulary is **total** over the model enum +(shorttext's fold aside), and that is load-bearing rather than tidy: a +relation document states its format as a required NAME (§2d), so a stored +format without a name is a relation object that cannot be exported. `map` +earned its name that way — the API does not serve it, but 72 production +relation documents carry format 102 (the bundled `templatePlaceholders` +relation), and a required name over real data may not have holes. The one +statement of the list lives in the published schema (`$defs/propertyFormat`), +referenced from every slot that speaks it. + +**There is one text format, `text`.** The editor offers a single Text +property type; the stored `longtext`/`shorttext` split is legacy, carries no +meaning an author could act on, and is **not part of this format** — +`shortText` is not a valid format name and is rejected by the schema. + +The collapse is not lossy, because `text` resolves per key rather than +blindly: + +- **Export** writes `text` for both stored formats. +- **Import** reads `text` as the key's *existing* format when that key is + already known to be `shorttext` — bundled properties (`name`, + `plural_name`, `source`, …) and anything the wiring's `ResolveFormat` + recognizes. So a + short-text property keeps its stored format across a round-trip even + though the document never names it. +- Otherwise `text` means `longtext`, which is what a **new** property + declared as `text` becomes. + +Any other format name is taken literally — the document is authoritative +about its properties, and only the text/text collapse needs a key to +disambiguate. + +**Properties are space-wide, not per-type.** Two types whose +`property_definitions` name the same select share one option pool, so their +vocabularies merge into a single dropdown. That is the point for a property +whose values are genuinely common (`tag`) and a defect for the lifecycle +selects a schema reaches for, where the same word means different things per +type. Documents that want distinct vocabularies must use distinct keys. + +**Select options are names, not ids — everywhere.** This rule covers +property values here, filter `value`s, and sort `custom_order` entries +(§6.2). Export writes option names (`"status": ["In progress"]`); import +resolves names against the property's existing options and **creates +missing ones** (the behavior of the CSV and Notion importers, and of the +public API's tag endpoints). Names, not ids, because a bundle carries no +option objects — unlike a linked object, which the bundle carries and the +importer relinks, an option id from another space would dangle — and because +opaque option ids are unwritable by agents and unreadable by humans. + +**The document carries the id beside the name: `option_ids`.** +Name-addressing alone loses identity in two ways a live account shows, and +both were measured on a 34 339-object sweep: two options of one property may +share a name, and name resolution answers the FIRST, so an object sitting on +the second came back pointing at the other one (7 objects); and an option +renamed between export and import stops resolving at all, so the wiring mints +a NEW option carrying the stale name — resurrecting the duplicate and +orphaning the object from the renamed option. So export writes the id beside +the name, in a legend keyed by the property that owns the option (§9a): + +```json +"priority": ["High"], +"severity": ["High"], +"option_ids": { + "Priority": { "High": "bafyrei…opt1" }, + "Severity": { "High": "bafyrei…opt2" } +} +``` + +The outer key is the property **spelling this document writes**, the inner +key the option **name** exactly as the value spells it; §9a states the shape +and the emission rule. It is written wherever export substitutes a name for +an id — property values, filter values, custom orders — and behind no option +at all, because it is identity rather than compaction. + +**Reading one option value: three steps, first answer wins.** + +1. **`option_ids[][]`** — honoured + only when the id it names is a **live option of that relation** in the + target space. There is no reachability precondition left to state: a + reader indexes the legend by the spelling the slot in hand wrote, so an + entry under any other spelling is simply never looked up (§9a warns about + one). The liveness check is the whole reason the entry is safe to write + unconditionally: an id from a space the reader never had is not an answer, + and the document falls through as if it carried none. +2. **Name resolution** against the property's existing options, as before. +3. **The value unchanged** — creating the missing option is the wiring's job. + +A reader with no option resolver (§13) has no space in which to ask either +question and stops at step 3, exactly as it did before this legend existed. + +**The legends do not answer to one rule, and the difference is deliberate.** +A `property_internal_keys` or `type_internal_keys` value is **authoritative**: the reader takes +it as the stored key, unchecked. Liveness-checking it would re-open the fault +the legend exists to close — a slug vacated by a deletion and reclaimed by a +new entity, where the key the document names is precisely the one the target +space no longer serves under that spelling. An `option_ids` value is a +**hint**, checked, because an option id names exactly one option of exactly +one relation, so the target space can answer whether the id is that; and +where the answer is no, the name is a better address than a dead id. The two +rules differ because the two questions do: a stored key IS the address, while +an option id is a shortcut past a name that is already one. + +What the authoritative rule costs, stated precisely: a legend value is a key +as the writing space holds it, so a reader in another space lands it +verbatim. That is *not* the same as saying legend values are +source-space-only. A **bundled** key is identical in every space, and a key +that arrived through an older pb-format import of the same data is reproduced +identically in every space that imported it — for both, a legend value +travels as well as the document does. The caveat is exactly the +**space-minted** key: a bson `6a32d485…` one space minted for its own +relation names nothing in another, so a document carrying it lands on a key +the target space has never seen instead of merging onto that space's +equivalent property. A bundle survives this because it ships the entity's own +document under that key; a document lifted out of a bundle does not. (The +import *wiring* narrows this further — `core/block/import/pb` re-homes a +non-bundled key onto an existing relation of the same format bearing the same +display name — but that is the wiring's behaviour, not the codec's: the codec +binds the slot to the stored key and hands it on.) + +What remains normalized, and what no longer is: **one object holding two +same-named options of one property** still collapses — the document spells +`["books", "books"]`, and two identical strings have no way to say which entry +means which option. Export keeps the first writing, so the collapse is +deterministic and a second export reproduces the first byte for byte (§11 +guarantee 3); +it is no better than name resolution here, and no worse. A rename, and a +duplicate name an object touches only once, are no longer lossy (§11). + +**Format resolution.** The format does not carry per-property formats; +`Marshal` and `Unmarshal` accept an optional resolver (§13). Property keys in +`bundle` resolve built-in; other keys resolve via the caller's resolver or +fall back to verbatim passthrough. Export and import must be wired with +equivalent resolvers for custom date/select properties to round-trip in +their pretty form; with no resolver the value still round-trips losslessly, +just unprettified. + +**Well-known properties** (the magic keys every generator needs). The +spelling is the display name (§3); the stored key is what it resolves to: + +| Spelling | Stored key | Format | Meaning | +|---|---|---|---| +| `Name` | `name` | text | the object's title | +| `Description` | `description` | text | subtitle/description line | +| `Done` | `done` | checkbox | completion state on task-like types | +| `Due date` | `dueDate` | date | due date on task-like types | + +The icon and the cover are **not** in this table: they are envelope fields of +their own (§2b), and the nine stored keys behind them are refused here. + +**Canonical key order in `properties`** (implementation decision): the +well-known keys `name`, `description` first (in that order, when present), +then all remaining members alphabetically BY SPELLING — the reader sorts +what it sees, so the order is over the display names, while which two go +first is decided on the stored keys. Both `icon_emoji` and +`icon_image` are lifted above `properties` entirely — a stronger +version of the same idea, since a reader now meets the icon before the +property list rather than at the top of it. + +**Presence is meaningful.** A key's presence in `properties` records that the +property was set on the object — clients use it to show the property even +when its value is empty. The §4 omit-empty canon therefore does **not** +apply to property values: every key present in the snapshot is written, with +its value verbatim — including `false`, `0`, `""`, `[]`, and explicit +`null`. Import preserves them all (an explicit `null` stays a null value). +Omitting a key and writing an empty value are different statements: absent = +property not set; empty value = property set, currently empty. + +**Seven system-stamped keys are the exception** (§15 #12). `isHidden`, +`isHiddenDiscovery`, `isArchived`, `relationReadonlyValue`, `revision`, +`relationMaxCount` and `relationDefaultValue` are written only when their +value is NOT empty. Nothing sets them but the system, and for each the empty +value IS the semantic default — `false` is visible, `0` is unlimited, an +empty default value is no default — so no reader distinguishes absent from +present-and-empty: every one reaches the value through a typed getter that +answers the same either way. Measured over 36,967 production documents, +their empty values are 1.13% of all bytes but the distribution is bimodal +(p50 1.21%, p90 13.55%, max 23.22%): they cluster on RELATION and TYPE +documents, so an agent reading a space's SCHEMA reads exactly the documents +that pay ~20%. + +It is a **whitelist, not a category**. The blanket form — every key in +`bundle.SystemRelations` minus an exception list — was declined: it admits +every system relation added in future sight-unseen, and buys almost nothing, +since the saving is top-heavy (these seven carry ~50% of it; the thirty-key +tail carries 0.04% of all bytes). The keys that FAILED admission are as +important as the ones that passed: `relationFormat` is excluded because its +`0` is `longtext`, a real format rather than "unset" (§15 #14), and +`relationFormatObjectTypes` and `featuredRelations` because they are +list-valued and user-intent-bearing — an empty list is how a CLEARED set is +expressed, the same reasoning GO-7451 settled for a type's recommended +lists. (The two `relationFormat*` keys have since moved to a relation +document's envelope, where the same verdict holds: the §2d fields mirror +stored presence, empty values included.) This is a state normalization, +recorded in `N(S)` (§11). + +**Value shape** (implementation decision): select/multi_select and +objects/files values are always JSON arrays; import stores them as lists, so +internally scalar-stored values (e.g. `assignee` holding one participant) +normalize to single-element lists on round-trip (§11). The two attribution +properties are the exception and are plain strings — see below. + +**Stripping.** Export removes internal/derived properties +(`bundle.LocalAndDerivedRelationKeys`) **except** those the importer +meaningfully preserves (mirroring `core/block/import/pb`): `createdDate`, +`lastModifiedDate`, `isFavorite`, `isArchived`, `resolvedLayout` — spelled +"Creation date", "Last modified date", "Favorited", "Archived" and +"Resolved layout". +Those five are **output-only** (§4a): export writes them, generators should +not — with one deliberate exception. **`isFavorite` is authorable**, because +the pb importer reads it to choose a space's root objects +(`core/block/import/pb/space.go`), which is how a generated bundle +designates the object a user should land on. A bundle with no favourite, no +`homepage` and no `spaceDashboardId` imports as an undifferentiated list. `id` is lifted to the envelope and `type` to `type`. Everything else +round-trips. + +**A participant document does not carry `createdDate`** (the +transient-key policy scoped by kind, like the type-provenance drop in §2a — +the verdict lives on `participantProvenanceKeys`). A participant is derived +from the ACL and has no creation change, so the store stamps `createdDate` +with `time.Now()` on every cold build. Measured, which is what admitted the drop: two exports of the +same 7 spaces, 1,164 documents compared field-by-field — the ONLY drifting +kind is participant (22 of 22) and the ONLY drifting field `createdDate`; +on a full 155-space run, 2,322 drifts against 2,492 participants, every +other kind byte-stable. Export omits the key on participants whatever it +holds; import drops it there (stale, not wrong); the §11 comparator +consults the same predicate. `creator` and `lastModifiedBy` STAY on +participants by decision, although both read `_anytype_profile` on 2,492 of +2,492 corpus participants: that placeholder is upstream's bug to fix — a +participant's creator should be the real identity — not this format's to +paper over by omission. + +**Attribution: `creator` and `lastModifiedBy` are the member's RESOLVABLE +id, named by the informative suffix — `#`, as a plain +string.** + +```json +"Created by": "A6eK73JmBUM9Aar2BJ4Pd6VkLW7cjhoWL7tJHDM9gk8fhpkc#roma_kha", +"Last modified by": "A6eK73JmBUM9Aar2BJ4Pd6VkLW7cjhoWL7tJHDM9gk8fhpkc#roma_kha" +``` + +Not an array. Both relations are `maxCount: 1` and 0 of 36,966 production +values were multi-valued, so the list wrapper the other object-format +properties take is definitionally wrong here. + +The spelling is the general §9 reference shape: the stored participant id +through the participant fold (48 characters instead of 135), the member's +display name riding after the `#` as a caption. An earlier design wrote the NAME alone: it broke API v2, whose consumers need an id to resolve a +member (avatar, profile), and **two members of one space can carry the same +display name** — 76 of 2,478 production participants do — so the name +identified nobody. The suffix keeps what the name-only form bought (a reader sees WHO, +not an address) and the id restores what it traded away. + +Both are `source: derived, readonly: true`: their value is recovered from the +object tree root's own cryptographic signature on every rebuild +(`treeSource.GetCreationInfo` → `NewParticipantId(spaceId, identity)`), and +four independent seams discard whatever a document supplies — +`state.StructCutKeys(details, LocalAndDerivedRelationKeys)` +(`core/block/editor/state/change.go`), the pb importer's preserve-list, which +names neither, `changeBlockDetailsSet`, and the API's "cannot be set +directly". **Import drops both keys**, whatever they carry. That reasoning +does **not** extend to `assignee`, `author`, `stakeholders` or any custom +`objects` property: those are `source: details`, chosen by a person; they +keep the array shape and the ordinary §9 reference rules. + +The name comes from a `ParticipantResolver` (§13), which export asks and +import does not have — and unlike the ordinary reference suffix it is NOT +behind `RefNames`: both keys are dropped on import, so no byte-stability is +at stake, and the name is the reason the line is worth writing at all. +**Without a resolver, or for a member this space has no name for, the id is +written BARE** — never a dangling `#`, and never an omitted property: the id +is the resolvable half and is complete without its caption. Only a value +holding no id at all omits the property — and so does the one degenerate id +production data actually holds: 9,103 of 37,429 corpus objects store +`lastModifiedBy = _participant__`, the composite built from a BLANK +identity. Eighty-six characters that address nobody are the id-shaped +analogue of a blank name and get the blank name's verdict. + +**The name is not an address, and nothing resolves it back.** It is the §9 +informative suffix: trimmed unread, never required, never unique. There is +deliberately no `option_ids`-style legend for it: the legend exists where a +name has to invert (§9a), and here nothing may. + +**Admission is symmetric with one documented exception: import refuses what +export strips, except for the keys it DROPS in silence.** Two families +qualify, and each entry owes the same two answers: what the key means in the +app, and why nothing downstream of an import can act on it. + +- **Transient keys** describe the *moment* an object was written rather than + the object. `internalFlags` carries editor state (`editorDeleteEmpty`, + `editorSelectType`, `editorSelectTemplate`: "this object was just created, + offer the type picker"), and a restored object is never mid-creation. + Export removes them like everything else on the stripped list. (Measured + across 36,967 real objects it was the single largest source of exported + noise — present on 18,647 of them, and empty on every one.) + `fileBackupStatus` and `fileIndexingStatus` are the same family from the + file machinery: which sync/index state THIS device last observed, stamped + on every file object (all 10,248 in a 28,604-document corpus), and the + destination's machinery determines its own — `fileIndexingStatus` carried + ONE distinct value across all occurrences and, imported, told the + destination's indexer the restored file needed no indexing. +- **Attribution keys** — `creator`, `lastModifiedBy` — name the member who + wrote the object. Their stored VALUE is stripped like every other derived + key; what export writes is the `#` spelling above, which no + write path could honour (the value is re-derived from the tree on every + rebuild). This closes an asymmetry with no reason behind it: `creator` + used to be accepted (it sat on the preserve-list, so the deny rule never + saw it) and landed a detail the next rebuild overwrote, while + `lastModifiedBy` — an identical relation definition — was refused + outright. + +Either way, import drops instead of refusing because a document carrying one +is *stale*, not hostile, and refusing it would make an older export +unimportable for no gain. Everything else on the stripped list is derived +state or a merge-resolution vector, and those stay errors. + +The rest of the rule, unchanged: **import refuses exactly what export strips.** The +list above is the only list — the reader derives its deny-list from it rather +than restating it, because a restated list drifts, and the drift ran one way: +import used to accept every key an author supplied, so `isArchived`, +`isDeleted`, `spaceId`, `restrictions` and `uniqueKey` all landed on details +while export removed them. Setting one is an error naming the key. Two more keys +are refused with them, because they are how the importer decides which +*existing* object a document merges into +(`core/block/import/common/objectid/existingobject.go`): `oldAnytypeID` and +`sourceFilePath`, alongside `uniqueKey` from the list. Those two are bundled +relations like any other — each has an api slug — but they are absent from +`bundle.LocalAndDerivedRelationKeys`, which is the list the deny-rule derives +from, so they have to be named by hand. Export strips those +three too, so the symmetry holds in both directions. `id` and `type` are +refused by name as well — they are the envelope's (§2), and dropping them in +silence left an author with no explanation for why the id they wrote had no +effect. (Those two are refused as *spellings*: the importer lifts them into +the envelope before any resolution runs, so the legend cannot re-purpose +them.) + +**Admission runs on the resolved stored key, not on the raw spelling.** The +document spells slugs, so a reader first lands each `properties` key on its +stored key through the §3 resolution chain, and *then* applies the deny +rule, the enum-name check and the format-shape warning to the result. +Checked against the raw spelling instead, all three were dead for exactly +the documents this format produces: `unique_key` walked past the rule that +`uniqueKey` tripped, and a `property_internal_keys` entry could rebind any harmless +spelling onto any internal key — including `id` itself, which overwrote the +envelope id from inside `properties`. `Validate` resolves with the chain +steps it has — legend, bundled table, verbatim; it holds no store, so chain +step 2 reaches it only through the identity entries export owes, and it +takes no resolver (§13). A reader whose vocabulary resolves *further* — a +node-backed caller whose space maps a slug to a stored key the bundled +table never knew — must re-run admission on **its** final resolved key, +which import does at the seam where details are written (`importer.build`). +Admission at that seam is three refusals, and validation mirrors every one +of them: a **denied** resolved key; an **unwritable** resolved key (a wider +vocabulary can resolve a spelling onto the empty string, which used to land +`details[""]` in silence and vanish on re-export); and **two spellings +binding onto one stored key** (refused only at import for a while, so a +hand-written `{"pluralName": …, "plural_name": …}` validated clean and then +failed to import; the original repro was the icon pair, which the icon rule refuses +one step earlier). The two halves agree exactly whenever no wider +vocabulary is in force, which is what keeps Validate and Unmarshal +accepting the same documents (§12). + +**A property key has to be writable.** Non-empty, no control characters, at +most 128 characters (`propertyNames` in the schema, restated in the reader so +the issue can name the offending key — §12). This is a *deny* rule and +not an allowlist on purpose: real stored keys are bundled camelCase keys, bson-hex +ids, and bare names from old accounts, and an allowlist could only be trusted +after checking every key in every account — while the shapes ruled out here +(the empty key, a key with a newline in it) are keys nothing can read. Export +drops such a stored key with a warning, since there is no way to write it. + +The rule binds the **spelling**, and the spelling is whatever the vocabulary +answers: the shipped label rule enforces it (§3 — a name outside the +writable bound is no label at all), but `Options.Keys` accepts an +implementation from anyone, and the raw material underneath is a display +name that is arbitrary user text with no length bound and no reserved-word +check — so nothing upstream *guarantees* a spelling this format accepts. +Export therefore checks the spelling it is about to write, and one it +cannot honor falls back to the stored key — always its own address +(verbatim-first) — with a warning naming the vocabulary's answer. Three +answers export cannot honor: an **unwritable** spelling (over-long, empty, +control characters — on either side of a legend entry); a spelling the deny +rule refuses before any resolution (`id`, `type` — the envelope's, which +the legend cannot re-purpose and therefore cannot rescue; a property +literally named "id" really mints this spelling); and any spelling for a +**denied key**, whose legend entry would carry a value admission refuses. +Checking the stored key and +then emitting the slug unchecked made `Marshal` produce a document its own +`Validate` rejects, on `/properties` and `/property_internal_keys` at once, which +§11 rules out. + +**A value whose shape its format cannot hold is a warning**, not an error, and +only for keys the bundle resolves after the resolution chain runs (`Validate` +takes no resolver, §13): `"Due date": "next Friday"` is stored as written and +then read as no date at all, which nothing else would ever report. It stays a +warning because the same check as an error would make one already-corrupt +stored value enough to make an object unexportable, and "Marshal never emits +what Validate rejects" (§11) is the stronger promise. + +Validation: the schema types `properties` loosely (`object` with scalar/array +values). Strict per-type validation against the object-type schemas generated +by `pkg/lib/schema` is a possible future layer (it would need a key↔name and +id↔name translation, since those schemas key by display name); v1 does not +provide this. + +## 4. Blocks — common structure + +`blocks` is a **flat array in pre-order**: a parent precedes its descendants +and a subtree is a contiguous run. Nesting is expressed by the per-block +`indent` integer — there is no `children` key (a document containing one +fails schema validation). Every block is an object: + +```json +[ + { "id": "b1", "type": "bulleted_list_item", "text": "top level" }, + { "indent": 1, "id": "b2", "type": "bulleted_list_item", "text": "nested" }, + { "indent": 2, "id": "b3", "type": "paragraph", "text": "deeper" } +] +``` + +| Field | Type | Req | Notes | +|---|---|---|---| +| `indent` | integer ≥ 0 | no | Nesting depth. Absent = `0` (top level); canonical form omits `indent: 0`. Values above **32** fail validation (adversarial-input bound). Real documents reach **6** — that is the deepest nesting anywhere in a 36,967-object corpus once transparent containers are lifted (§7a); before the lift the same corpus reached 26, all of it wrapper. See the nesting rules below. | +| `type` | string | **yes** | Discriminator; full inventory in §5. Unrecognized values fail schema validation (see §10 for forward compatibility). | +| `id` | string | no | `[A-Za-z0-9_-]{1,64}`. Uniqueness is enforced over the whole document, including derived table cell ids `-` — the whole grid, written cells and unwritten ones alike (§6.1) — so a non-table block id that collides with a derived cell id is a validation error. Dataview **view** ids are the one exception: they are unique **within their dataview block**, not document-wide (§6.2). Export writes ids by default — the `OmitIds` option drops them (§9); import generates missing ids with the editor's standard id generator. | +| `align` | `left · center · right · justify` | no | Omit when default (`left`). | +| `vertical_align` | `top · middle · bottom` | no | Omit when default (`top`). | +| `background_color` | string | no | Anytype color name. Omit when empty. | +| `fields` | object | no | Verbatim internal per-block key-value data **minus** keys lifted into first-class props (e.g. `lang` §5.1, `width` §6.1). Output-only escape hatch (§4a) that keeps unknown data lossless. | + +### Nesting + +- **Reconstruction** (import semantics, normative): walk the array with a + stack seeded `(root, indent = −1)`. For a block with indent *k*: pop the + stack until the top's indent is *k − 1*; the top is the parent; append the + block to the parent's children; push `(block, k)`. +- **Validity** (strict, the default): the first block's indent MUST be 0, + and a block's indent MUST be at most one greater than its predecessor's. + Violations are **errors**, path-addressed and naming both indents + (`/blocks/7: indent 3 follows indent 1 — a block can be at most one level + deeper than its predecessor`). Every prefix + of a valid `blocks` array is itself valid — a truncated document parses as + a well-formed prefix of blocks (enforced by test). +- **Lenient mode** (`Options.NormalizeIndent`, import only, default off): + an over-deep indent (jump > +1) is **clamped to the previous block's + indent + 1** — CommonMark's list rule: a level that hasn't been + established cannot be opened — and a first block with indent > 0 is + clamped to 0. Every clamp is reported as a warning-grade issue with the + block's JSON path (`Options.OnWarning`). Indents outside [0, 32] are + errors even in lenient mode. +- **Containment** (semantic checks, §12): leaf block types cannot be + parents — a block indented under one is an error naming the parent type + (the leaf types are marked in §5); a block whose parent is a `row` must + be a `column`. + +Block restrictions are **not** part of the format: they are runtime policy, +reconstructed by the editor on import. + +**Serialization canon** — what export produces; `Export ∘ Import` is +byte-stable over it (§11): + +- UTF-8, LF, two-space indent. +- **Key order = spec order.** Envelope keys in the §2 table order. Block + keys: `indent` first, then `id`, `type`, then the + type-specific props **in the order listed for that type in §5** (`text` + always last), then `align`, `vertical_align`, `background_color`, `fields`. + Nested dataview/table objects: the order listed in §6. `property_internal_keys`, + `type_internal_keys` and `option_ids` entries sorted by key, and each `option_ids` + inner map sorted by option name. +- **Omit empty and default.** Canonical form never writes an empty string, + empty array, or empty object (envelope included — no `"properties": {}`), + nor a default scalar (`"indent": 0`, `"checked": false`, `"align": + "left"`, `"hidden": false`…). Absent `text` means empty text. Import + accepts explicit empties/defaults and canonicalizes them away. + +### 4a. Output-only fields + +Some fields exist purely so that export → import loses nothing. Export +writes them; **generators should omit them** — import accepts documents +without them, and where a supplied value would not be safe to take it is +refused rather than quietly used: the internal property keys are a deny-list +in the reader (§3), which is where "authoritative only where semantically +safe" is actually implemented. Most output-only fields carry +`x-output-only: true` in the JSON Schema so tooling can warn; the one kind +that cannot is the preserved internal properties, which live inside the +free-form `propertyMap` and so have no schema node of their own to annotate. + +Output-only surfaces: `fields` (any block), `root`, `store`, `source` +(dataview), `groups`/`object_orders` (views, §6.2), `id` on sorts/filters, +filter `nested_property` (reserved), `cover.source` and the `emoji` +carry-over on `icon`'s named-icon branch (§2b), the five preserved internal +properties listed in §3, and the two attribution properties +`creator`/`lastModifiedBy`. + +The attribution pair is output-only in the strictest sense on the list: +export writes it and import does not merely ignore a supplied value, it +drops the key. Everything else here at worst round-trips. + +## 5. Block type inventory + +Text styles are promoted into `type`; every proto content type maps to one or +more JSON types. The "Proto origin" column is informative (for implementers), +not part of the format. Prop lists are in **canonical order** (§4). Complete +mapping: + +| JSON `type` | Proto origin | Type-specific props (canonical order) | +|---|---|---| +| `paragraph` | Text/Paragraph | `color`, `text` | +| `heading_1` … `heading_3` | Text/Header1..3 | `color`, `text`. Input aliases `heading_4`/`header_4` map to `heading_3`; stored deprecated Header4 blocks **export as** `heading_3` (§11) | +| `quote` | Text/Quote | `color`, `text` | +| `code` | Text/Code | `language` (from `fields["lang"]`), `text` (**literal**, §8.4) | +| `title` | Text/Title | — structural, see §7 | +| `description` | Text/Description | — structural, see §7 | +| `checkbox` | Text/Checkbox | `checked`, `color`, `text` | +| `bulleted_list_item` | Text/Marked | `color`, `text` (common block-editor naming) | +| `numbered_list_item` | Text/Numbered | `color`, `text` (numbering is derived from position among consecutive siblings; never stored) | +| `toggle` | Text/Toggle | `color`, `text` | +| `callout` | Text/Callout | `icon` (§2b, `emoji` or `file` only), `color`, `text` | +| `toggle_heading_1` … `toggle_heading_3` | Text/ToggleHeader1..3 | `color`, `text` | +| `file` `image` `video` `audio` `pdf` | File (Type enum promoted; `Type_None` → `file` with no `object_id`) | `object_id` (target file object), `name`, `mime_type`, `size` (bytes), `style` (`auto · link · embed`), `added_at` (RFC 3339; omitted with a warning when the stored timestamp is outside the representable years, §3 — unlike a property value there is no number form to fall back to). Legacy `hash` accepted on input. On export, a block with only the legacy `hash` set writes it as `object_id` (the hash migrates on round-trip, §11); when both are set, `object_id` wins and the hash is dropped. `state` is not serialized: import sets `Done` when `object_id`/`hash` is present, `Empty` otherwise. File blocks are leaves in the editor, but legacy data can nest real blocks under them — indented descendants are allowed and round-trip verbatim | +| `bookmark` | Bookmark | `url`, `object_id` (target bookmark object). `state` handled like file blocks. Deprecated preview fields and `type` (derivable) are dropped — preview data lives on the target object | +| `link` | Link | `object_id` (target object), `card_style` (`text · card · inline`), `icon_size` (`none · small · medium`), `description` (`none · manual · content`), `properties` (string array: property keys shown on the card). Deprecated `style` and legacy `fields` are dropped | +| `divider` | Div | `style` (`line · dots`, default `line`) | +| `row` / `column` | Layout/Row, Layout/Column | — (descendants carry content; a `row` contains only `column`s — §4 containment, read on the lifted tree, §7a) | +| `group` | Layout/Div (legacy) | — **accepted on input only; lifted** (§7a). No export ever writes one | +| `table` | Table (+ structural children) | `columns`, `rows` — see §6.1 | +| `embed` | Latex | `processor`, `text` (**literal**, §8.4) — see §5.2 | +| `table_of_contents` | TableOfContents | — | +| `property` | Relation | `property` (the property's spelling, the member every property-naming slot uses; renders the property inline) | +| `dataview` | Dataview | fully specified in §6.2 | +| `widget` | Widget | `layout` (`link · tree · list · compact_list · view`), `limit`, `view_id`, `auto_added`. Appears only inside a widget object — and a bundle carries no widget document: its sidebar is `index.widgets`, which states these members flat beside the link child's (§2c) | +| `chat` | Chat | — (rare) | +| `featured_properties` | FeaturedRelations | — structural, see §7 | +| `icon` | Icon | `name` (legacy profile objects only) | + +Enum values serialize as snake_case strings (§1 Naming); defaults are omitted. + +**Leaf types.** `embed` (and its `equation` alias), `bookmark`, `link`, +`divider`, `table`, `property`, `dataview`, `icon`, `table_of_contents`, +`featured_properties`, and `chat` cannot be parents: a block indented under +one is a validation error naming the parent type (§4 containment, §12). +Every other type may be a parent. + +Normalization notes: + +- `checked` on styles other than `checkbox` is dropped (the editor only + honors it there). +- Stored marks on `code`/`embed` blocks are dropped on export (their `text` + is literal). + +### 5.1 Code blocks + +`language` is lifted from the internal `fields["lang"]` (the storage location +used by the editor and all importers). On import it is written back; a `lang` +key inside an explicit `fields` object is an error when `language` is also +set. + +### 5.2 Embed blocks + +`processor` selects the embed kind — full enum, snake_case: `latex` +(default), `mermaid`, `chart`, `youtube`, `vimeo`, `soundcloud`, +`google_maps`, `miro`, `figma`, `twitter`, `open_street_map`, `reddit`, +`facebook`, `instagram`, `telegram`, `github_gist`, `codepen`, `bilibili`, +`excalidraw`, `kroki`, `graphviz`, `sketchfab`, `image`, `drawio`, +`spotify`. + +`text` carries **source code** for renderer processors (`latex`, `mermaid`, +`chart`, `graphviz`, `kroki`, `excalidraw`, `drawio`) and a **URL** for +service processors (everything else); for service processors import also +accepts the URL under a `url` key as an input alias. + +Standalone math is `{ "type": "embed", "processor": "latex", "text": "…" }`; +import accepts `equation` as a type alias for it (what Notion-trained +generators will write). + +## 6. Complex blocks + +Two content types carry structure beyond text and props; both get first-class +mappings rather than raw protojson — the format is meant to be fully +readable/writable, not only its Markdown-shaped primitives. + +### 6.1 Tables + +Anyblock stores tables as a block subtree (table → row/column layout wrappers +→ cells with composite ids `-`). The JSON format hides this +machinery: + +```json +{ + "type": "table", + "columns": [ { "id": "col1" }, { "id": "col2", "width": 120 } ], + "rows": [ + { "id": "row1", "is_header": true, "cells": [ "Name", "Status" ] }, + { "id": "row2", "cells": [ "Export", + { "type": "checkbox", "checked": true, "text": "done" } ] }, + { "id": "row3", "cells": [ null, "spec" ] } + ] +} +``` + +- `cells[i]` corresponds to `columns[i]`; `null` = empty cell. A row with + **fewer** cells than columns is padded with trailing empties; **more** + cells than columns is a validation error. +- A cell is a plain string, `null`, a block object, or an array of flat + blocks. The string form is shorthand for a plain paragraph and is + **canonical** whenever the cell qualifies (a `paragraph` with only `text` + set); a block object is used otherwise. A bare cell block carries no + `indent` (validation error if present). The **array form** exists for the + legacy case of a cell block with descendants: the cell block first at + indent 0, its descendants following per the §4 rules; export uses it only + when descendants exist (single-block cells stay bare — canonical). Cells + **never carry `id`** — cell ids are derived (`-`); an `id` + on a cell block (bare, or first element of the array form) is a + validation error. Cell blocks (and their array-form descendants) **cannot + be `table` blocks**: cells use a dedicated non-recursive block definition, + which is what keeps the whole block schema recursion-free (§12). +- Column/row `id`s are optional; when present they must match + `[A-Za-z0-9_]{1,64}` — **no `-`**, which is the composite-cell-id + separator. Import generates missing ids. +- `width` on a column entry (pixels) is first-class (lifted from the + internal `fields["width"]`); other column data round-trips via `fields`. +- **Generated row/column ids obey the same charset as authored ones.** A + cell's id is `rowId + "-" + colId`, and the editor recovers the column with + `SplitN(id, "-", 2)` (`table.ParseCellID`), + so a `-` anywhere in a row or column id silently reassigns cells to the + wrong column. `Options.GenerateId` belongs to the caller and need not + respect that, so import + sanitizes generated ids into `[A-Za-z0-9_]{1,64}` and disambiguates + collisions rather than trusting the generator. Both apply only where they + are needed: a generated id that already fits the charset and collides with + nothing keeps the name the generator gave it, as every other minted id does + (§9). Export sanitizes stored ids + the same way, since data predating this rule contains dashes and `Marshal` + must never emit a document its own `Validate` rejects. +- **A table owns its whole grid of derived ids, written cells or not.** The + id `-` belongs to the table for every row×column pair, + because the editor materializes a missing cell at exactly that id the + first time it is filled — an unwritten cell's id is reserved, not free. + All three surfaces claim the same set: validation over the grid, export + before it labels any other block, import before it generates any id. + **The plain block is the side that yields.** A derived id has no spelling + of its own — it is whatever the row and column ids make it — so a block + whose stored id collides with one is written under a disambiguated label + (`r1-c1` → `r1-c1_2`) while the row and column keep theirs. The reverse + would rename two authored ids to move one grid, and move every other + derived id in the table with it. +- Header rows must come first (editor invariant); import reorders + (normalizes) rather than rejects, same as the editor does. +- Export normalizes before flattening, mirroring the editor's own table + normalization: cells sorted into column order, orphan cells dropped. Only + a structurally unrecognizable subtree (missing row/column wrappers) is an + export error. +- An empty plain-paragraph cell and an absent cell are the same thing: + export writes `null` for both, import creates no cell block for `null`, + `""`, or a bare empty paragraph (normalization, §11). Trailing empty + cells are omitted (import pads). + +### 6.2 Dataview + +Dataview blocks embed a queryable view over objects — a *set* (live query) +or a *collection* (curated list, `is_collection: true`) — that they reference +but do not own. +Field-for-field from `Content.Dataview`, with cleaned names, snake_case +string enums, and defaults omitted: + +```json +{ + "type": "dataview", + "object_id": "bafyrei…targetSet", + "properties": [ + { "property": "Name", "format": "text" }, + { "property": "Status", "format": "select" }, + { "property": "Due date", "format": "date" } + ], + "views": [ + { + "id": "v1", + "type": "kanban", + "name": "By status", + "group_by": "Status", + "sorts": [ + { "property": "Due date", "direction": "asc", "empty_placement": "end" } + ], + "filters": [ + { "property": "Due date", "condition": "less", "date_preset": "current_week" }, + { "property": "Done", "condition": "equal", "value": false } + ], + "columns": [ + { "property": "Name" }, + { "property": "Due date", "width": 120, "align": "right" }, + { "property": "Status", "aggregation": "count_distinct" } + ] + } + ] +} +``` + +**Dataview props** (`Content.Dataview`), canonical order as listed: + +| Prop | Proto field | Notes | +|---|---|---| +| `object_id` | `TargetObjectId` | the set/collection object this view queries; empty for original set/collection objects and detached inline sets | +| `is_collection` | `is_collection` | | +| `source` | `source` | legacy, detached inline sets only; output-only (§4a) | +| `properties` | `relationLinks` | array of `{ "property", "format" }` — the properties available to this view, with formats per §3's vocabulary; `property` is the same member name the columns, sorts and filters use to refer to one (one spelling per concept). **This field is live** (maintained by the dataview editor), unlike the deprecated snapshot-level relationLinks | +| `views` | `views` | see below | + +Dropped (normalization): `activeView` (local UI state; the proto itself +excludes it from changes) and the deprecated proto `relations` field. + +**View props** (`Dataview.View`), canonical order: `id`, `type` +(`table · list · gallery · kanban · calendar · graph`, omit `table` — the public API currently says `grid`), +`name`, `group_by` (property key; from `groupRelationKey`), `cover_property` +(from `coverRelationKey`), `end_property` (from `endRelationKey`; the end +date of a range — **inert today**, see below), `hide_icon`, `card_size` (`small · medium · large`, +omit `small`), `cover_fit`, `colored_groups` (from `groupBackgroundColors`), +`page_size` (from `pageLimit`), `default_template_id`, `default_type_id` (from +`defaultObjectTypeId`), `wrap_content`, `list_size` (`compact · regular`, +omit `compact`), `alternate_rows`, then `sorts`, `filters`, `columns`, +`groups`, `object_orders`. + +**View id uniqueness is scoped to the dataview block.** Two views of ONE +dataview may not share an `id` — that is a validation error naming both +positions — but two views in *different* dataview blocks may. This is the +only id domain in the format that is not document-wide (§4). Across blocks, each view is reached through its own block and +nothing is ambiguous — and the app itself produces that case: the default +view of every set, collection and type is minted with the literal id +`default`, and creating an inline set from an existing object copies that +object's views verbatim, so a page with two inline collections legitimately +holds two views called `default`. + +Editor state nested per view, both output-only (§4a): + +- `groups`: `[{ "id", "hidden", "background_color" }]` — kanban group + display order (array order; the proto's per-group `index` is derived). + From `Dataview.groupOrders`, matched by view id. +- `object_orders`: `[{ "group_id", "object_ids": […] }]` — manual object order + within groups. From `Dataview.objectOrders`. + +**Column** (`View.Relation`), canonical order: `property` (the property +key), `hidden` (inverse of proto `isVisible`; omitted = visible, so the +common case costs nothing), `width` (displayed column width in **pixels**, +see below), `aggregation` +(`count · count_value · count_distinct · count_empty · count_not_empty · +percent_empty · percent_not_empty · sum · average · median · min · max · range` +— from proto `formula`; omit `none`), `align`. Deprecated per-column +date/time fields are dropped. + +**Column `width` is in pixels**, a non-negative **integer** (the proto stores +an `int32`, and the schema says so, so `33.3` is a validation error rather +than a silent truncation to `33` — a fractional width is almost always a +percentage the author meant), the same unit as the proto's `width` — not +a percentage, and not a share of the table. A row of columns summing to +`100` produces four unreadable slivers, not four proportional columns. +Serialization passes the number through unchanged: the client owns +rendering, and this package neither clamps nor defaults it — so **omitting +`width` is the better default than guessing one**: the client already applies +sensible per-format defaults and never clamps a non-zero value on render, so +the choice tracks the client rather than freezing here. Write a number only +to pin a deliberate layout. + +**There is no timeline/Gantt view, and `end_property` currently does +nothing.** The proto's view type enum ends at `Graph = 5`; the client +carries a sixth, `Timeline`, but it is gated behind `config.experimental` +and has no proto value, so it cannot be described here. `endRelationKey` is +read by that timeline component and by nothing else — a calendar view does +**not** use it, and shows single dates from `group_by` alone. `end_property` +round-trips faithfully for data that already carries it, but setting it on +any expressible view type has no effect. A view stored with the +experimental type reads back as `table`, since an out-of-range enum is +omitted rather than emitted as a schema-invalid empty string. + +**Sort** (`Dataview.Sort`), canonical order: `property` (from +`RelationKey`), `direction` (`asc · desc · custom`, omit `asc`), +`custom_order` (for `custom`; select values by option **name** per §3, other +values verbatim), `empty_placement` (`start · end`, omit unspecified), +`include_time` (include time-of-day when comparing dates), `no_collate` +(disable locale-aware collation; compare raw strings), `id` (output-only). + +**Dates are not empty-safe.** An object with no value for a date property +matches `less` and `less_or_equal` regardless of the threshold. An "overdue" view +must therefore pair the comparison with a `not_empty` on the same property +inside an `and` group; a `not_empty` under an `or` guards nothing. +`greater`/`greater_or_equal` are unaffected. Import warns on an unguarded +comparison rather than rejecting it — including undated objects is a legal +thing to want, and stored data contains such filters. + +**Filter** (`Dataview.Filter`) — a filter node is either a **group** or a +**leaf** (schema `oneOf`); the top-level `filters` array combines its nodes +with an implicit **AND** (canonical form uses bare leaves at the top level; +a group exists only for `or` or nesting): + +- group: `{ "operator": "and" | "or", "filters": [nodes…] }`. Export maps a + proto node with non-empty `nestedFilters` to a group and drops its leaf + fields; import writes `operator` only on groups (leaves get the proto + default). +- leaf, canonical order: `property` (**required** — a leaf filter names the + property it filters on, like the sort and the column beside it; export drops + a filter whose stored relation key is empty rather than write a node that + filters on nothing, §3), `condition`, `value`, `date_preset`, + `include_time`, `nested_property` (reserved, output-only), `id` + (output-only). `condition` values: `equal · not_equal · greater · less · + greater_or_equal · less_or_equal · contains · not_contains · in · not_in · + empty · not_empty · all_in · not_all_in · exact_in · not_exact_in · exists` + (`contains`/`not_contains` from proto `Like`/`NotLike` — the public API + agrees). `date_preset` from proto `quickOption` (`yesterday · today · + tomorrow · last_week · current_week · next_week · last_month · current_month · + next_month · number_of_days_ago · number_of_days_now · last_year · current_year · + next_year`, omit `exactDate`). `value`: for select/multi_select properties, + option **names** per §3; dates stay unix numbers in the structured form; + everything else verbatim. `value` is **dropped** on + `empty`/`not_empty`/`exists` leaves (§11). + + **Dynamic values.** A `value` entry of the form `_filter_template__` is + a placeholder the *client* substitutes for a real object id before issuing + the query (`Dataview.valueTemplateMapper`): `_filter_template_2_` is the + current user, resolving to `_participant__`, and + `_filter_template_1_` is the object hosting an inline dataview, resolving + to its id. They are stored verbatim and are **opaque to the middleware** — + nothing in Go resolves them, so a query evaluated server-side compares + against the literal string and matches nothing. They are not object ids, and nothing in either direction rewrites them — + object references are never compacted, so there is no legend one could be + swallowed into (§9a). They are meaningful only on `objects`/`files` + properties, since they resolve to an object id; on any other format the + placeholder is stored UI state that matches nothing, and the mismatch is a + **warning, not a refusal** — the same severity the neighbouring date-preset + rule takes, for the same reason. Both doors warn — the fragment surface + too, or one filter would validate on one door and be refused on the other. + + **A preset applies on a date property, under six conditions only.** + `transformDateFilter` returns a filter whose format is not `date` before it + computes anything at all; on a date filter it computes the range but + substitutes it into the query for `equal`, `in`, `less`, `greater`, + `less_or_equal` and `greater_or_equal` only. Fail either half — a preset on + a text or select property, a preset under `not_equal` — and the preset is + stored UI state with no effect on what the view matches. A preset resolves + to a day *range*, and the condition picks the endpoint: + `less`/`greater_or_equal` compare against the range start, + `greater`/`less_or_equal` against its end, and `equal`/`in` expand into a + pair bracketing both. + + A preset under a condition that does not apply is a **warning**, not an + error: the author wrote "verified this week" and the view means "verified, + ever", which is worth saying, but export writes the pairing because stored + filters carry it, and refusing it would make one stored filter enough to + make an object unexportable (§11, I1). The format half is not warned about, + because the format of a filter's property usually comes from outside the + document — the bundled table, the space — so "not a date" is as often "not + known here", and a warning that fires on a correct filter makes every + warning cheaper to ignore (§12). + + `number_of_days_ago` and `number_of_days_now` are the two presets that **take an + operand**: `getDateRange` reads the day count from `value` + (`pkg/lib/database/quickoptions.go`), so they are the one case where a + preset and a `value` legitimately coexist, and **where the preset applies** + — both halves of the gate above — a leaf carrying such a preset without a + **day count** in `value` is a validation error: the count would default to + `0`, silently meaning today. The rule reads the operand, not the member: + `getDateRange` reads it with `domain.Value.Int64`, which answers `0` for a + `null`, a string, a list — for every kind that is not a number — so those + are the same silent "today" a missing member is, and a presence-only rule + refused one and admitted the others. A day count is a **whole number in + `[0, 36500]`**, the bound the compact grammar already puts on `daysAgo(n)` + (§6.2.1): two forms of one filter language admit the same filters. Because + the count is meaningful data rather than an absent field, export writes it + even when it is `0`, overriding the usual empty-elision (§4) — and writes + the count the query engine reads out of a stored operand that is not one + (`0` for a non-number, the truncation for a fraction, the bound for + anything past it), with an `OnWarning`, because the slot has one written + form and a document carrying the junk verbatim is one this package's own + `Validate` refuses (§11, I1). Anywhere the preset does not apply the rule does not + either, because the count is never read: nothing is silently anything. + +Sorts and filters do **not** carry the proto's cached per-node `format`: +import rehydrates it from the dataview `properties` list and `bundle` +(unresolvable keys get format 0, which the query engine tolerates). + +Proto-default edge cases (implementation decisions): a leaf whose proto +condition is `None` (0) omits `condition` — absent means `None`; a proto +group node with operator `No` (0) exports as `"and"`; contentless filter +nodes (groups with no live children, leaves carrying at most an id) and +sorts without a property key are no-ops and are dropped on export; +out-of-range proto enum values are omitted rather than serialized (an +unknown *text style* is an export error — silently restyling content would +be worse). + +#### 6.2.1 Compact filter syntax — shipped grammar, reserved document field + +**Status: split scope.** The grammar below and its parser ship +**now**, as the library subpackage `pkg/lib/anyblockjson/filterstring` +(§13): parse a filter string → the §6.2 structured filter tree +(`model.BlockContentDataviewFilter` nodes), with **offset-addressed +errors** naming the offending token and its position. Its consumer is the +API v2 request surface (`POST …/search` and the `filter` field of +`POST …/sets`), where the string is the documented +small-model form and both request forms land on one internal tree. The +grammar is thereby pinned by the parser and served via the API's discovery +surface. + +The **document** side is unchanged and stays reserved: v1 documents ship +the structured `filters` array only. The view field name `filter` +(singular, string) is **reserved** for a post-v1 extension: v1 schemas do +not define it, so introducing it later is a version bump (§10 — a +v1.0 reader encountering it reports "produced by a newer version"; export +keeps writing the structured array; the `CompactFilters` export option +stays reserved in `Options`). When that lands, the two forms coexist +permanently — `filter` and `filters` mutually exclusive per view, import +accepting both, export choosing via option. One consequence of raw-name +addressing (§3) is already known for that future field: a display name is +not a bare identifier in this grammar, so the document-side form will need +a quoted-key production (`"Due date" < currentWeek()`); the bare-key +grammar below is the API request surface's, whose key convention is a +separate decision. + +The design, normative for the parser (and unchanged for the future +document extension): a view carries its filter as a single SQL/JQL-flavored +query string: + +```json +{ "type": "kanban", "group_by": "Status", + "filter": "done = false AND (due_date < currentWeek() OR due_date IS EMPTY)" } +``` + +Grammar (informal here; the `filterstring` parser is the normative +artifact, and the EBNF it pins is what the API discovery surface serves): +`OR` over `AND` over parenthesized groups over leaves; `AND` binds tighter, +parentheses group. There is deliberately **no free-standing `NOT (…)`** — +the internal model has no NOT-group; negation exists only in negated +conditions, keeping string ⇄ structured 1:1. + +| Condition | Syntax | +|---|---| +| equal / not_equal | `priority = 3` / `priority != 3` | +| greater / less / greater_or_equal / less_or_equal | `> < >= <=` | +| contains / not_contains | `name CONTAINS "report"` / `NOT CONTAINS` | +| in / not_in | `status IN ("In progress", "Blocked")` / `NOT IN (…)` | +| all_in / not_all_in | `tags HAS ALL ("urgent", "q3")` / `NOT HAS ALL (…)` | +| exact_in / not_exact_in | `tags = ("a", "b")` / `!= (…)` — set literal on the right | +| empty / not_empty | `assignee IS EMPTY` / `IS NOT EMPTY` | +| exists | `assignee EXISTS` | + +Values: double-quoted strings, bare numbers, `true`/`false`, RFC 3339 dates +in quotes (`due_date < "2026-08-01"`), and date-preset **functions** — +`yesterday() · today() · tomorrow() · lastWeek() · currentWeek() · +nextWeek() · lastMonth() · currentMonth() · nextMonth() · lastYear() · +currentYear() · nextYear() · daysAgo(n) · daysFromNow(n)` (the parameterized +pair maps to `number_of_days_ago`/`number_of_days_now` with the value as `n`; +parens distinguish presets from string literals). + +**Property keys are bare identifiers, and they reach a spelling through the +fold.** The grammar has no quoted-key form, so a key is written with +identifier characters only (the exact charset is given below) and must not be one of +the grammar's reserved words. That is narrower than what a property may be +SPELLED, since a spelling is a display name and names carry spaces: `Due +date` cannot be written here. It does not have to be, because resolution +folds away case and separators, so the bare `due_date` addresses it — and +`Дата_выполнения` addresses "Дата выполнения" the same way. What no +identifier folds onto — `C++`, `50% done`, a name colliding with `AND` or +`IS` — has no compact form at all, and the parser says so and names the +structured `filters` array as the way to express it. +Select/multi_select values are option **names**, per §3 (the structured form +agrees; only date values differ — RFC 3339 here, unix numbers +there). The RFC 3339 → unix conversion is **format-driven, not +string-driven**: it happens only for keys whose format resolves to `date` +through the consumer-wired `Options.ResolveFormat` (a date-looking string +on a text property stays a string; a non-RFC-3339 string on a date +property is a parse error steering to the presets). A consumer that wires +no resolver gets string values verbatim — executing such a filter against +date properties matches nothing, so query surfaces MUST wire the resolver. + +Parser interpretation calls (normative, matching the shipped parser): +keywords match **case-insensitively** (`and` ≡ `AND`) and are **reserved** +— none can be a bare property key (a colliding key is reachable only +through the structured form); property keys are Unicode identifiers +(`identStart identPart*` — letters of any script, digits, `_`, and the +combining marks the vowels of Indic and SE-Asian scripts are written with), **the +grammar §3 mints every label through**, so a key a document spells can +always be written here — the reason `50% done` labels `_50_done` and a +bson-keyed property labels its name rather than its key; presets are **excluded +from value lists** and from conditions the engine does not transform +(`notEqual`, `contains`, …: only `= > < >= <=` take a preset); set +literals require `=` / `!=` (a list after an ordering operator errors); +the counting presets take a whole day count in `[0, 36500]`. Bounds: the +input is capped at **4096 bytes** and parenthesis nesting at **32** (the +§4 document nesting bound) — both are ordinary offset-addressed parse +errors. + +Canonical rendering: uppercase keywords, `", "` separators, double quotes +with backslash escapes, parentheses only where precedence requires. Export +will keep writing the structured array by default; a future `CompactFilters` +option will emit the string form for any view whose filter is fully +expressible (every leaf free of output-only fields like `nested_property`, +every option name resolvable), falling back to the structured array per +view otherwise. Import will accept both forms; string-parse errors report +the view's JSON path, the offending token, and its position. + +## 7. Structural blocks + +The following blocks are **derivable** and are dropped on export: + +- the root block (implicit, §2), +- the header wrapper and its children `title`, `description`, + `featured_properties` — their content duplicates `properties.name` / + `properties.description`. + +Import does **not** attempt to rebuild them: which structural blocks an +object gets depends on its layout (note objects have no title block at all, +todo objects bind `done`, …), which the editor resolves from the type's +recommended layout at first open (`template.InitTemplate`). The package +preserves `resolvedLayout` in `properties` (§3) and leaves structural blocks +absent; the editor regenerates them on open. `N(S)` in §11 is defined +accordingly. + +A document that nevertheless contains such blocks at indent 0 is accepted +(agents will produce them): import merges `title` / `description` text into +the corresponding properties when those are unset and drops the blocks +otherwise — together with any blocks indented under them; a top-level +`featured_properties` block (which carries no content) is simply dropped. + +**The primary dataview** is the one structural id import *does* rebuild. +Object types, sets and collections keep their own dataview at the fixed +block id `dataview` (`state.DataviewBlockID`); the editor recreates it on +open only *if absent* (`template.WithDataviewIDIfNotExists`), so a document +whose dataview lands on a generated id gets a second, empty dataview +alongside the configured one. Unlike `title`/`description`, the block cannot +simply be dropped and regenerated — its views, columns and widths are the +author's configuration, not derivable — so import **pins the id** instead: + +> the first indent-0 `dataview` block with neither an explicit `id` nor an +> `object_id` becomes `dataview`. + +`object_id` is what separates the two cases: an inline view of *another* set +or collection has it set (§6.2) and keeps its generated id, as does any +dataview nested below indent 0, and any dataview after the first. If some +block already claims `dataview`, that block wins and nothing is pinned — an +explicit id stays authoritative and cannot collide (§13). Export is +unchanged: it emits the id verbatim, and under `omitIds` (§9) the rule +restores it on the way back in. + +**Content-less blocks** (legacy data): old accounts hold blocks whose +content oneof is unset — relation objects wrap their "used in" dataview in +one, and pages can contain orphaned empty leaves. They are transparent +containers (§7a): the block is dropped either way, and a subtree under one is +lifted into its place. + +## 7a. Transparent containers + +A block is a **transparent container** when it contributes containment and +nothing else: + +- its content is `Layout` with style **`Div`** (`model.BlockContentLayout_Div` + — the editor's fan-out wrapper, minted by `state.wrapChildrenToDiv` when a + parent exceeds `maxChildrenThreshold` children), or +- its content oneof is **unset** (legacy data), with or without children. + +The test is on **content**, never on the `div-` id prefix the normalizer +mints. Keying on a prefix would make id *spelling* semantically load-bearing, +and it would leave an authored +`{"type": "group"}` round-tripping into a permanent wrapper. + +**Export** writes nothing for a container and emits its children at the +container's **own indent**, with the container's own top-level status. In +JSON terms: `group` is a type no export ever produces, on any surface. +Consequences, stated so they are not re-derived: + +- A **childless** container emits nothing at all. +- **Nested containers collapse fully**: a chain of *n* removes *n* levels. +- **Every attribute the container carried goes with it** — `align`, + `vertical_align`, `background_color`, `fields`. No conditional + preservation; recorded in `N(S)` (§11) and reported through + `Options.OnWarning` when there was anything to lose. +- The lift runs **before** the depth bound is checked, so both the value + compared against 32 and the emitted `indent` are post-lift. +- A container at indent 0 is transparent for §7 too: a structural block + underneath it is at the document's top level and is dropped there, rather + than being preserved by the accident of a wrapper standing over it. +- The rule applies on every export surface — the document, a table cell's + descendants, and a block subtree (§13.1) **including its root**, since no + read surface ever serves a container id, so no caller can address one + except out of a stale cache. A subtree rooted at a container marshals as + its lifted children; rooted at a childless one, as an empty run. + +**Import** does the inverse, as a pre-pass over the flat run: a `group` entry +contributes no block, and every following entry indented deeper than it +re-bases one level shallower — recursively, for nested containers. Any +attribute on the entry is ignored, and so is its id. The lift runs **before** +the primary-dataview pin and before top-level structural absorption (§7), +which is what lets a wrapped dataview be seen at the indent-0 position the pin +requires and a wrapped `title` be absorbed into `properties.name`. Because the +lift is positional, a lifted structural block is at indent 0 for every +purpose, on both sides. + +Monotonicity survives by construction, so the lift can never manufacture an F6 +violation: a container at indent *g* satisfied *g ≤ p+1*, and its first child, +at *g+1*, lands at *g*. + +**The two positions that address exactly one block cannot lift, and say so +rather than resolving to nothing:** the single-block fragment entry point +(§13.1) refuses a lone container, and a table **cell's own block** cannot be +one — a cell is a position, not a run — which `Validate` refuses too, so the +two agree. That holds for **both cell spellings** (§6.1): the array form is +refused at index 0 of the run, the object form on the cell itself. They are +separate checks because they are separate readers (I2, in the one shape §7a +cannot lift). A cell whose +stored block *is* a container renders as an empty cell. + +**Containment (§12) is judged against the lifted tree**, because that is the +tree import builds. `row > group > column` is **valid**: it says +`row > column`. `row > group > paragraph` is invalid and is reported against +the row, naming the container in between (`nested under a group inside a row +— a row block can only contain column blocks, got paragraph`), or the message +reads as wrong to whoever wrote the `group`. A container is itself exempt from +the check: it becomes nothing, so there is nothing to place. + +**What comes back.** Nothing in this format re-creates a container, and no +importer wiring is needed: the editor's own normalization re-wraps on +`ApplyState`, which runs on creation and on every cache load. It puts back a +**different** partition — the split point is a function of arrival order, not +of the document — and for a document whose content has since shrunk below the +threshold it puts back nothing at all. Both are covered by `N(S)` (§11). + +**The re-wrapping is an obligation on the wiring, not the format:** the +re-wrapping is the editor's, not the format's. A writer that builds a +snapshot and stores it WITHOUT going through the object-creation path that +enables layouts (`EnableLayouts`) will land a thousand-child object in front of a +renderer the threshold exists to protect. Every path that writes an imported +document has to run the editor's apply, exactly as the import wiring does +today. + +**The other five layout styles are unaffected**: `Row` and `Column` are +author-created and grammar-bearing (a column carries `fields.width`), +`Header` is structural (§7), and `TableRows`/`TableColumns` belong to a +table's internals (§6.1). A stray `TableRows`/`TableColumns` outside a table +still drops its whole subtree, deliberately: folding it into this rule would +put table cells at top level. + +## 8. Rich text: inline markup + +Text-bearing blocks carry a single `text` string. Formatting is expressed +inline — **offsets never appear in the format.** + +```json +{ + "type": "paragraph", + "text": "Ship the **new export** by Q3 with Roman" +} +``` + +### 8.1 Grammar + +A CommonMark-inline subset plus a small whitelist of inline tags for marks +with no Markdown equivalent: + +| Syntax | Proto `Mark.Type` | Notes | +|---|---|---| +| `**text**` | Bold | | +| `*text*` | Italic | canonical form; `_text_` accepted on input | +| `~~text~~` | Strikethrough | | +| `` `text` `` | Keyboard | inline code; content is literal (CommonMark code-span rules, §8.2) | +| `[text](url)` | Link | external URLs | +| `[text](anytype://object?objectId=)` | Object | inline link to an Anytype object — Anytype's standard deep-link shape. The form is **exact**: scheme `anytype`, host `object`, and a single `object_id` parameter, with the id percent-encoded. Any other `anytype://` destination — a second parameter, a different host, a path — is **not** an object reference and stays a plain Link, preserved verbatim (§10) | +| `text` | Mention | decorated object reference (icon + name in UI) | +| `text` | Underscored | standard HTML | +| `text` | TextColor | Anytype color names | +| `text` | BackgroundColor | coincident color+background ranges combine into one tag: `` | +| — | Emoji | not writable: export **materializes** the mark by splicing its emoji over the covered text (the mark's semantics are replacement; this matches the Markdown export and the chat renderer). On import emoji are plain text | + +Inline tags: a tag name and an attribute name are `[A-Za-z][A-Za-z_]*` — +snake_case like every other identifier the format defines (§1 *Naming*), which +is why `object_id` is an attribute name and not the attribute `object` +followed by a stray character. Import accepts any attribute order, single or +double quotes, and surrounding whitespace; canonical form is double quotes, +single spaces, `color` before `background`. An attribute the tag does not +define is an error naming it (§12), so a document written against an older +draft fails loudly rather than dropping the mark. Zero-length tags (e.g. +``) are dropped on input. + +Everything else is literal text. No other Markdown constructs are recognized +inside `text` — no block syntax, no images, no autolinks, no HTML beyond the +whitelisted tags. `\n` inside `text` is a soft line break within the block +(Shift+Enter), encoded as the JSON `\n` escape. + +### 8.2 Escaping + +CommonMark backslash escapes apply: literal `*`, `` ` ``, `[`, `]`, `~`, `<`, +`\` in prose are written `\*`, `` \` ``, `\[`, `\]`, `\~`, `\<`, `\\`. Input +additionally accepts HTML entities (`<`, `&`). Canonical export uses +backslash escapes, applied minimally (only where the character would +otherwise be parsed as markup). + +Code spans follow CommonMark, where backslash escapes do **not** apply: +content containing backticks is delimited by the shortest backtick run not +present in the content, space-padded when the content starts or ends with a +backtick (`` `` `code` `` ``). + +**Canonical escaping, made precise** (implementation decision — "minimally" +is defined as the following deterministic rule set; at internal mark +boundaries the unseen neighbor is treated as punctuation, conservatively): + +- `*` — escaped unless whitespace on both sides. +- `_` — escaped iff it could open or close under underscore flanking + (intraword underscores stay literal). +- `` ` `` — always escaped in prose. +- `~` — escaped when adjacent to another tilde or sitting at a mark + boundary. On input, only runs of exactly two tildes are strikethrough + delimiters; other run lengths are literal. +- `[` — always escaped in prose (a bare `[` could assemble a false link + with text from a later mark segment; no local lookahead can rule it out). +- `]` — escaped only inside link labels. +- `<` — escaped before any **tag-shaped** sequence: `<`, an optional `/`, + then at least one ASCII letter. Deliberately wider than the three tag + names version 2 knows: `x` in prose exports as + `\x\`. This is the tag namespace's **reserved syntax space** + (§10) — see the note below. +- `&` — escaped only where a valid entity follows. Recognized entities: + `lt gt amp quot apos nbsp` and numeric (`A`, `A`). +- `\` — escaped when followed by ASCII punctuation (input accepts a + backslash before any ASCII punctuation as an escape, per CommonMark). + +**Reserved syntax space** (the one escaping rule that is not minimal, and +why). A `text` string carries no version marker, so bytes are the only thing +a later version has to work with. If canonical output escaped `<` only for +`u`/`font`/`mention`, a version-2 document could contain a literal +`x`, and the day a version adds `sub` those same bytes read as +markup — the reader cannot tell version-2-literal from version-3-markup, and +because a malformed instance of a *known* tag is an error (§8.3), a stored +document that was valid could become invalid. Escaping the tag *shape* +closes that: in canonical output, an unescaped `<` is never followed by a +letter, so the entire `` space is free for any future version to +define, with no text-rewriting migration and no ambiguity. The cost is a +backslash on prose that looks like markup (`a\`) when the URL contains whitespace (brackets and +backticks escaped there too — raw ones would join the enclosing label or +code-span scan when links nest); entities are decoded in destinations and +attribute values on input. `_` delimiter runs parse exactly like `*` runs +(so `__x__` is bold — liberal input; canonical output always uses stars). + +**Resource bounds** (implementation decision — deterministic local rules +that keep parsing linear on the untrusted-document boundary): link +destinations longer than 2048 UTF-16 code units, destinations surrounded by +more than 32 whitespace characters, and link labels nested more than 32 +deep are not recognized — the `[` stays literal. Export drops Link/Object +marks whose rendered destination would exceed the bound, and Emoji marks +whose param exceeds 64 code units, as invalid (§8.3 step 1), so round trips +stay byte-stable. + +### 8.3 Canonical rendering (the round-trip contract for marks) + +Internal marks are ranges over UTF-16 code units and may overlap arbitrarily. +Export: + +1. Materialize Emoji marks (§8.1). Drop zero-length and invalid ranges. +2. Normalize boundaries: Markdown-delimited marks (`**`, `*`, `~~`, `` ` ``) + shrink past leading/trailing whitespace at their boundaries — whitespace + at a boundary carries no visible styling, and CommonMark's flanking rules + reject delimiters against whitespace. Tag-delimited marks (``, + ``, mention) and links are unaffected. +3. Resolve same-type overlaps: two marks of the same type with different + params (two links, two mentions) cannot wrap one segment — the + earlier-starting mark wins the overlap and the later range is truncated + to start where the earlier ends (zero-length results dropped). +4. Split the text at every remaining mark boundary; each segment carries its + mark set. +5. Emit segments left to right, opening/closing delimiters so that nesting + is deterministic — fixed order outermost→innermost: Mention, Object, + Link, ``, `` (coincident ranges combine into + one tag), ``, `~~`, `**`, `*`, `` ` ``. Delimiters shared by adjacent + segments stay open (maximal runs). + +Implementation decisions: + +- **Step 1 details**: "invalid" ranges are out-of-bounds, inverted, + zero-length, or splitting a UTF-16 surrogate pair; a param-carrying mark + (link, mention, object, colors, emoji) with an empty param is dropped; + a param on a param-less mark type is cleared (so equal ranges merge); + params beyond the §8.2 resource bounds are dropped. A **Link mark whose + param is exactly the `anytype://object?objectId=` deep-link (§8.1 — + one parameter, nothing else) normalizes to an Object mark** — the two + render identically, and without the normalization the parse-back type flip + would change same-type overlap resolution. A Link carrying any *other* + `anytype://` destination is left alone: reinterpreting it would have to + guess which part is the id, and guessing wrong is unrecoverable, whereas + preserving it verbatim always round-trips. +- **Step 2 extension**: emphasis-family marks (`**`, `*`, `~~`) additionally + exclude any whitespace run touched by a *stack-outer* mark's endpoint — + the outer change forces the emphasis delimiter to close/reopen inside the + run, and an emphasis delimiter against whitespace cannot re-parse + (flanking). Whitespace styling is invisible for these types, so the split + is a rendering no-op. +- **Step 3 tie-break**: at equal starts the longer range wins; the shorter + same-type range is truncated to nothing and dropped. + +Import parses the grammar back to ranges: each maximal contiguous run of a +mark becomes one range; offsets are computed in UTF-16 code units (matching +editor semantics; `util/text` helpers). + +**Parser discipline** (implementation decision): the parser is the exact +inverse of the canonical renderer — a deterministic delimiter stack (close +the top entry while it matches, open with the remainder, demote what can do +neither to literal text), *not* CommonMark's delimiter-run algorithm. The +rule-of-three resolution is not invertible, and §11's byte-stability over +arbitrarily overlapping ranges requires an exact inverse; the grammar stays +syntax-compatible with CommonMark/anymark for well-formed input. Unmatched +Markdown delimiters demote to literal text (CommonMark spirit); malformed, +unclosed, or misnested *whitelisted tags* are validation errors (§12) — +once ``, bundled table or a declared format — a + space-minted key it cannot resolve passes unremarked), and the value is + stored as written, addressing nothing. +- **An id containing `#` still loses its tail on read**, since the reader + cannot tell that `#` from a caption's. It is the format's one reference + normalization, listed in `N(S)` (§11), and it converges after one + generation. +- **Round trip**: byte-stable given the same resolver — import trims, the + next export re-derives the same names. Absent a resolver the suffix is + absent, the same class of resolver-dependence as option names (§3). + +### The participant fold + +`_participant__` is a derived id +(`core/domain.NewParticipantId`): the space half restates the document's own +space, and the 48-character identity is the whole of the content. Every +reference slot above folds it to the bare identity on export, and import +rebuilds the composite against `Options.SpaceId` (§13): + +- **The trigger is the VALUE's shape, never the property name.** The + heaviest participant slots in production are space-minted custom + properties (`owner`, `voters`, …) with no declared target type, and + `assignee`/`author` may legitimately hold a contact. The classifier is the + identity's own strkey checksum (`crypto.DecodeAccountAddress`): no CID, + bson id or `_`-prefixed derived id can pass it, so unfold cannot fire on + anything else. +- **The participant document's own envelope `id` folds too** — otherwise a + reader could not textually join a folded reference to the document it + points at. This makes participants the documented special case in the + envelope `id` slot, which otherwise always holds a real object id; import + rebuilds the composite as the object id and the root block id. +- **`Options.SpaceId` arms it, in both directions at once.** The format + carries no space id anywhere in the envelope, so the wiring supplies one + exactly as it supplies resolvers (`storeresolver` wires the index's own). + With no SpaceId nothing folds and nothing unfolds. **Any reader of a + folded document MUST set it** — it is the space the document is being + read INTO, which every importer necessarily knows, since an import lands + in a space. A reader that names none stores the bare identity where a + composite belongs, addressing no object; because the classifier is exact, + the reader knows this has happened and reports it through the warning + sink, once for the document (§13). The one caller with genuinely no + target space is a converter that does not import — `cmd/anyblockconvert` + — and it is the path the warning exists for. +- **An empty identity is not an identity.** `_participant__`, built + from a blank identity, addresses nobody; 9,103 of 37,429 production + objects store one in `lastModifiedBy`. It does not fold — and only the + classifier refuses it, since `NewParticipantId(space, "")` rebuilds that + exact string, so the round-trip recheck cannot. Without the classifier it + would fold to the empty string and the reference would be deleted. +- **Only this space's composites fold.** A composite embedding a DIFFERENT + space passes through whole in both directions: folding it would silently + re-home the member on import. (A document carried into another space + re-homes deliberately and correctly, because its folded references + rebuild against the READER's SpaceId.) +- Measured (37,429 production objects): 3,446 same-space composite + occurrences across properties, `items`, block `object_id`s, filter + values, object orders and the participants' own envelope ids — all fold, + none remain. The corpus held zero cross-space composites. + +### References the space cannot serve + +A reference to an object that does not exist in the SPACE is not written as +if it did. The space stores already state this for the references +their importers resolved — `_missing_object` +(`pkg/lib/localstore/addr.MissingObject`) stands 1,089 times across a +28,617-document corpus — and export now applies the same honesty to ids +that dangle without the sentinel, and a consistent policy to the sentinels +it re-exports. + +**The split is by what the slot can express.** + +- A **singular slot** — a block's `object_id` (link, bookmark, file, image, + video, audio, pdf, dataview) and a `` target (§8) + — REWRITES the id to `_missing_object`. Omission cannot express "no + target" there: only deleting the block (or the mark) could, and that + would lose the fact that a link or mention existed — the mention's text + stays, only its address is gone. A stored sentinel is kept as-is. +- A **list slot** — an objects/files property value (§3), a property + document's `object_types` (§2d) — DROPS the entry: a list expresses + absence by being shorter. A stored sentinel drops too. The emptied list + stays `[]`, never omitted: the key's presence is meaningful (§3), and for + `object_types` an empty list is a cleared target set (§2d). +- Everything else is deliberately out of scope: collection `items`, filter + values, custom orders, `object_orders`, a type's `default_template_id`, + and object-link marks keep their ids verbatim. Each of those can be + extended later on this section's precedent; none was in the evidence. + +**"Missing from this export" and "missing from the space" are different +facts, and only the second may cause a rewrite.** An export of a single +object references its neighbours in the space; those objects exist and were +simply not exported, and rewriting them would corrupt a perfectly good +export. The exporter never sees the export set — it works one document at a +time — so the only question it can ask is of the STORE, which is the right +question: does this space hold a row for this id? The answer comes through +the `ObjectExistenceResolver` capability (§13), asked affirmatively — +`known && !exists` — so a store failure moves nothing. And it is a NEW +capability because the resolver already standing in the object namespace +cannot answer it: `ObjectNameResolver`'s ok is `name != ""`, which reads +"exists but untitled" as "no". Untitled objects are common; conflating the +two questions rewrites live references. + +**The question only reaches ids the space index is the authority for**: +CID-shaped ids (`isObjectIdShaped` — `cid.Decode` behind a length gate). +A `_date_…` id is virtual, `_ot…`/`_br…` resolve against the bundled +tables, a participant composite against the fold, a bare type key against +the key vocabulary, a widget link target against the editor's constants — +a store that was never an id's authority cannot declare it missing, so +none of those are ever asked about, let alone rewritten. A deleted +object's tombstone is a row: its id still means something in this space, +and references to it are untouched — with ONE deliberate exception, the +icon. An icon is optional where a link or mention target is not, so an +`iconImage` whose target the space DELETED is dropped rather than kept or +rewritten: export asks the narrower question through the +`ObjectDeletionResolver` capability (§13, `DroppedDeletedIconRef` — the +predicate is exported so the comparator applies the same rule), and the +document falls through to whatever icon channel is left, exactly as an +image that is not an object id already does (§2b). Measured before the +rule: 134 corpus bookmark documents shipped an icon pointing at a favicon +whose file object was a tombstone in their own space's store. A store +failure (`known == false`) drops nothing, and no other reference slot asks +about deletion at all. + +**With no capability wired, nothing moves — the sentinel included.** A +package-only export passes every reference through verbatim, exactly as +before this rule existed: the absence of an answer is not evidence of +absence, and the offline round trip stays byte-exact. + +**Warnings follow what is lost.** A rewrite or a real-id drop destroys the +stored id — the warning is that id's last appearance anywhere — so both +warn, naming the id. A stored sentinel kept or dropped says nothing: which +object it was is already gone, and ~990 silent sentinel drops per corpus +would drown the channel §12 just reclaimed. + +**Round trip**: the change converges in one generation and is a fixpoint +after — the first export rewrites and drops, import stores what was +written, and `Export(Import(Export(S))) = Export(S)` holds (§11 guarantee +3). The comparator applies the same exported predicate +(`DroppedMissingObjectRef`) to both sides, so a dropped-by-design entry is +a normalization, not loss (§11). + +### 9a. The legends, and compact ids + +The envelope carries **three legends** and no other indirection. Each answers +one question the rest of the document cannot: + +| legend | maps | question | +|---|---|---| +| `property_internal_keys` | property spelling → stored relation key | which relation does this spelling name? (§3) | +| `type_internal_keys` | type spelling → stored type key | which type does this spelling name? (§3) | +| `option_ids` | property spelling → (option name → option id) | which option does this name mean? (§3) | + +Three maps rather than one, and `option_ids` nested rather than flat, for one +reason stated twice at two scales: **a name in this format is arbitrary user +text, so no character can be reserved to join it to its scope.** The property +and type namespaces are disjoint claim domains and a space may slug a +relation and a type onto one term (§3), so a single spelling→key map would +hold two answers for it. One step down, an option name may contain anything a +JSON string may, and so may the property spelling that owns it — under raw +naming a property really is named `C#`, and its spelling is exactly that. A +flat map keyed +`#` therefore had no representable entry at all for an +option of a property named `C#` — the escape hatch was unreachable exactly +where it was needed — and re-opening that after the freeze costs a version +(§10). +Nesting removes the separator, and with it the split rule, the key admission +rule, the two charsets, and the joined key's length bound. + +**`option_ids`.** + +```json +"properties": { "Priority": ["High"], "Severity": ["High"] }, +"option_ids": { + "priority": { "High": "bafyrei…opt1" }, + "severity": { "High": "bafyrei…opt2" } +} +``` + +- **Outer key**: a property **spelling as this document writes it** — the + reader that resolves the entry is reading the document, not the store — so + it carries the writable-key rule every property spelling carries (1–128 + characters, no control characters, §3), and the property it names inverts + through `property_internal_keys` like any spelling elsewhere in the document. The + reader does not invert the outer key itself: it indexes the legend by the + spelling the slot in hand wrote, and matches or does not. Export writes the + spelling the slot itself just used, so its outer keys are spellings the + document holds by construction. +- **Inner key**: the option **name**, character for character as the value + spells it, bounded only by being non-empty. It carries no charset rule, + deliberately: it is the same string the value slot already holds, and a + legend that cannot name a value its own document carries is the `C#` hole + again, one level down. +- **Value**: the full option id. +- **Written unconditionally**, wherever export substitutes a name for an id — + property values, dataview filter values, sort custom orders (§3). Behind no + compaction flag, because this is identity rather than compaction; and + behind no ambiguity test either, though one is computable: such a test sees + only the divergence that exists when the document is written, and the + rename it would guard against happens in the gap between writing and + reading. Nothing is pruned because nothing unused is written — the entry is + recorded at the substitution itself. +- **Read as a hint, not an address** — §3's three steps: the id, honoured + only where the target space still serves it as a live option of that + relation; then name resolution; then the value unchanged. A reader with no + option resolver ignores the legend entirely, having no space in which to + ask, which is what keeps a bundle carried elsewhere working exactly as it + does without it. +- **`OmitIds` drops it** (§9): the export and backup shape keeps the legend, + the prompt shape does not. §9 states what that gives up — the two losses above, back, + on the read/prompt shape — and why export does not warn about it. +- **An outer key naming a property this document never spells is a warning** + (§12) — a key-set comparison, not a parse. The entry can never be + consulted, since a reader indexes by the spelling the slot in hand wrote. A + warning rather than an error, because a legend may carry more than one + document needs; but an entry that degrades to name resolution in silence is + the kind of silence this format reports everywhere else. + +**Object references are never compacted.** Every object id — mention and +object-link targets in `text`, `object_id` props, a callout's `icon.file`, +the envelope `icon.file` and `cover.file` (§2b), `objects`/`files` property +values, `items`, +view `default_template_id`/`default_type_id`, `object_orders[].object_ids`, +and filter `value`/sort `custom_order` entries of `objects`/`files` +properties — is written in full, on every shape, with no legend. The §9 +`#name` suffix and the participant fold are not exceptions: the suffix adds +a caption to a full id and inverts by deletion (no table to carry, nothing +to keep in sync), and the folded identity IS the participant id's content, +rebuilt from the reader's own space rather than looked up anywhere. + +This is a deletion. The format used to carry a `refs` map of short labels to +full ids behind a `CompactObjectRefs` flag, and two independent measurements +retired it. API v2 removed the same legend from its read shape after +measuring a net token **loss** per document, and because the indirection +trapped write-back: an agent editing an object-valued property through a +label has to keep the legend in step, and one that regenerates the document +without it silently re-points every reference it held. The freeze review +measured the loss from the other end — a 200-item collection grew **32.7%** +under compaction, because a label used once costs more than it saves. Two +measurements, one verdict. + +**The compaction that survives is the legend-less one**, and that is the rule +this section has left. `CompactBlockLabels` relabels ids the document itself +defines, so a short label needs no table to invert: it is a placeholder +within its containing document, never an address outside it, and a write +endpoint resolves one against the live object by unique suffix. There is +nothing to carry, nothing to keep in sync, and nothing to read back. An +indirection table has all three obligations, and the object legend failed all +three at once — which is why the half sold as "lossless, because the legend +inverts it" is gone and the half documented as *lossy* stayed. + +With `CompactBlockLabels` (or `CompactIds`, which now selects that one half), +block/row/column/view ids are relabeled to their last 5 characters. Only +machine-minted opaque ids relabel: `dataview` is a documented constant, +`title`/`header` are structural, and an imported document's human-readable +ids carry meaning that relabeling would destroy for no benefit. Labels are +constrained to the schema charsets (the block-id charset `[A-Za-z0-9_-]{1,64}` +of §4; row and column relabels additionally dash-free, since `-` is the +derived-cell-id separator of §6.1), and an id whose label would collide with +another id in the document — relabeled or not — or that yields no valid +label stays uncompacted (implementation decision — fixed-width suffixes with +a full-id fallback, chosen over shortest-unique lengthening for simplicity; 5 +characters over CID/hex alphabets make collisions birthday-rare). + +The collision rule counts BOTH id populations, and that is not an accident of +implementation: the labeller's own census sees only the doc-local ids it may +relabel, so the object ids — every one of them now spelled verbatim in the +document, in the folded spelling where the §9 participant fold applies — +enter it as an avoid-set (both spellings: the document spells the folded +form, and a suffix-trimming reader recovers the raw one). A short object id spelled in a mention +and a minted block whose suffix equals it would otherwise both answer to one +name in one document. Deleting object compaction made this guard matter more, +not less. + +**The census counts the ids the document SPELLS, not every id the snapshot +holds** — the same principle the term census follows (§3). A block the +document does not spell — a transparent container (§7a), a structural block +(§7), a content-less leaf, anything unreachable — +is gone from the snapshot a round trip rebuilds, so reserving its suffix slot +makes the two reads disagree: the first keeps a paragraph's id full because +an invisible block shares its 5-char tail, the second compacts it, and +guarantee 3 (§11) fails on the API's default read shape. The protection given +up is illusory in any case: a container the editor re-creates gets a FRESH id +no census could have reserved against. + +**One unspelled id is reserved all the same: a cell's.** A cell carries no id +in the flat form (§6.1), but unlike everything else in that list it is not +gone from the rebuilt snapshot — import re-derives `rowId-colId` from row and +column ids the document DOES spell, so the same cell ids come back and +reserving them is stable across generations. It is also necessary: a cell id +ends with its column's id in full, so its last five characters ARE the +column's label. Leave cells out and the column wins that bucket alone and +compacts to a label its own cells share as a suffix in the live object — +which breaks this section's own promise that a served label is neither equal +to nor an ambiguous suffix of another served id, and makes the wiring's +resolve-by-unique-suffix allowance below unsound. Measured before the fix: +899 documents in a 36,966-object account served such a label. + +**The census costs a second block emit.** `emittedLocalIds` runs the emit +again on a throwaway exporter rather than re-deriving the drop rules, because +a second statement of "what export emits" would be a second thing to keep in +step with the first, and the census is correct only while the two agree +exactly. Measured on a 1,630-block document: 4.2 ms → 6.7 ms, +57%. It is +paid only where labels are minted — that is, on the API's default read shape, +and never on the export/backup shape or under `OmitIds`, which writes no id +for a plan to label. + +The two shapes the API serves are the two this leaves: API v2 default reads +use block labels (the server resolves them by unique suffix) and keep object +refs full inline, while its export shape — the backup/round-trip shape, whose +bytes re-import to the same document up to what the editor regenerates (§7, +§7a) — keeps block ids full (API spec C4). + +**A wiring may still shorten what the format does not.** Import wiring MAY +resolve an id it cannot find by unique suffix against the target space +(useful for hand-written documents naming known objects), and a write +endpoint MAY resolve a block-label reference the same way against the live +object. Both belong to the wiring, not to this package: they are lookups +against live state, not indirection a document carries. + +`CompactBlockLabels` and `OmitIds` compose: together they yield the most +prompt-friendly form (no block ids at all). Both are alternative +serializations — the canonical round-trip form (§11) remains the default +full-id export. + +## 10. Versioning and compatibility + +`version` is a **single integer with no minor axis**. It is required, it is +the sole authority on format identity, and it is checked before anything else +in the document is interpreted. + +- **A reader rejects any document whose `version` is greater than its own**, + with a dedicated error naming both versions rather than a generic schema + failure. There is no partial or best-effort read of a newer document and no + forward compatibility: a change an older reader cannot handle is exactly + what a version bump means. +- **A reader accepts any document whose `version` is less than or equal to its + own**, migrating older documents forward before parsing — with ONE + exception, stated here rather than left implied: **`version` 1 is refused**. + It is the pre-freeze draft integer (below), carried by every export made + while the grammar was still moving, so there is no single grammar to + migrate it from; the reader says so at `/version` and names re-export as + the repair. Version 2 is the first frozen grammar and the first this rule + will ever migrate FROM. Because `version` is required and unambiguous, a + later migration has complete information about the grammar a stored + document used. +- **Every format change bumps the version.** There is no additive-within-a- + version rule, because there is nothing additive to have: the schema closes + every object (`additionalProperties: false`) and every enum is exhaustive, + so a new block type, a new property, a new enum value, a new mark, or a + renamed key is rejected whole-document by an older reader regardless of how + it is introduced. Saying so plainly is cheaper than a reserved-field + mechanism that buys nothing under the rule above. +- **Two regimes, and every field belongs to exactly one.** The bullet above + is the CLOSED regime, and it is not the whole format. A **closed** slot — + an enum this document states as a fixed set of names, or a JSON object's + own membership — refuses what it does not recognize, whole-document, with + no degradation. An **open** slot — a property or type spelling, an option + id, a dictionary key, or a numeric detail the app itself stores and reads + as opaque data — degrades instead of refusing, because the entity it names + lives in a space or a bundled table this reader may not fully know: it + passes the value through verbatim, never inventing and never silently + coercing to a default, and warns exactly where the degradation would + otherwise be invisible. A field is closed when every value it can legally + hold is enumerable at freeze time and a wrong one cannot be repaired by + resolving it against a live space or an older bundle; it is open + otherwise. **A new field's author states which regime it joins, in the + same sentence that adds it.** + + The three open-regime behaviours, and why they differ: a stored number + outside a named-enum property's vocabulary passes through RAW and lossless + (§3), because the app treats it as opaque data; an out-of-range proto enum + on a struct-typed field is OMITTED, which reads back as that field's + default, because the slot has a safe default and no raw form (§6.2); and a + content discriminator — `kind`, a block `type`, a relation `format` — + REFUSES the whole document at export rather than misrepresent content. +- The `$schema` URL carries the same integer + (`https://schemas.anytype.io/anyblock//object.schema.json`) and is + **decorative**: it is optional, no reader gates on it, and the schema at a + version's URL is mutable in place — a correction that does not change the + format is republished there rather than given a new number. The frozen + grammar is `anyblock/2/`. Format identity lives in `version` and nowhere + else. +- `index.json` shares the same version number and the same rules (§2c), and a + bundle is versioned as one artifact: if the index or any document in it + declares an unsupported version, the whole bundle is rejected rather than + partially imported. +- **A pre-release grammar change left no version marker, and the freeze + closes that hole.** Every revision this document records — the three + legends replacing `refs`, most sharply — happened under `version` 1, so a + draft written against any of them is indistinguishable from a draft written + against the last. The integer therefore moved ONCE at the freeze: the + frozen grammar is 2, and 1 is refused outright at the version gate (§15 #9). + That is the whole of what the bump buys — not migration, which no single + grammar could define, but a clean refusal in place of a silent misread. + A superseded draft that somehow reaches the schema is still refused there + too, by the members the current grammar does not admit, and the reader + names the member (`/refs`) with the rule that replaced it and the repair, + rather than reporting a closed-set violation at the document root (§12). + + The relation lift (§2d) is the same shape, and the same decision — + **refuse, loudly, with the repair named**, never read-and-migrate. A + legacy relation document spells `relation_format` inside `properties` + and has no envelope `format`. It trips the missing-`format` refusal, which + carries the whole repair: the message lists the vocabulary and, when a + legacy spelling sits in `properties`, says outright that it is the + legacy form and where the value moved. Measured over all 10,617 legacy + relation documents in a 38,061-document corpus, every one trips exactly + that refusal and exactly one — the `/properties/relation_format` refusal + cannot also fire, because it lives in the semantic pass and a schema + failure never reaches it. It appears on the second pass, once the envelope + field exists and the old member is still there. The same message also + names `format` in `properties` when that is what the author wrote, which + is the commoner mistake and the one a missing-member verdict would + otherwise never mention. Reading the old spelling with a warning was + declined for the reason §2b records — this format is a draft with no + external consumers, so the refusal strands nobody. + + The `relation`→`property` rename moves the first refusal a legacy document meets, without + changing the decision: a legacy relation document spells + `kind: "relation"`, which the kind enum now refuses by name before any + member is read, and the vacated `relation_format` spelling resolves to + nothing at all any more. (The alias spellings are retired in turn: the + refusal-by-resolution now fires on the display name `"Format"` and on the + verbatim stored key, the two spellings that still name the detail — §3.) + +**Syntax inside `text` is versioned too, and the reader is exact about it.** +A `text` string carries no version marker of its own, so the only thing that +keeps a stored document readable across a bump is that the reader recognizes +*exactly* the syntax its version defines and treats everything else as +literal. This binds three namespaces: + +| namespace | version 2 recognizes | anything else | status | +|---|---|---|---| +| inline tags (§8.1) | `u`, `font`, `mention` | literal text, never an error — reported as a warning, since canonical output would have escaped it | **reserved**: canonical output escapes every tag-shaped `<` (§8.2), so the whole ``, one parameter | a plain Link, preserved verbatim | matched by exact form, so a second parameter is available to a later version | + +Being exact is what makes a later migration possible: when a version adds a +tag, a delimiter, or a deep-link parameter, the migration escapes or rewrites +the prior occurrences that a stored document meant literally, and it can only +do that if version 2's rule was unambiguous. A reader that guessed — matching +a deep link by prefix, say, and taking whatever followed as the id — would +have already destroyed the information a migration needs. + +The reservation is what keeps that migration from being needed at all for +canonical documents: because export escapes tag-shaped `<`, a version that +adds a tag can read version-2 documents as they are. Only hand-written +documents can carry an unescaped tag-shaped sequence, which is why import +warns about one instead of silently accepting it — the warning is the +author's notice that those bytes are only literal by virtue of the document's +`version`, and that canonical form spells them `\<`. + +**The cost this accepts.** When version 3 ships, a client still on version 2 +cannot open *any* document a version-3 client exported — refused, not +degraded. For an export and interchange format written by external tools and +agents that is the right trade: it buys a contract with exactly one rule, and +the alternative — readers that tolerate unknown constructs — obliges every +reader to carry a degradation behaviour for every construct that will ever be +added. It would be the wrong trade if AnyBlock JSON became a cross-device wire +format, so that is a deliberate constraint on where the format is used, and it +is recorded here rather than discovered later. + +## 11. Round-trip guarantees + +Let `N(S)` be state normalization (given export and import wired with +equivalent resolvers, §3): structural blocks dropped, to be regenerated by +the editor at first open (§7); **transparent containers dropped and their +children lifted to the container's own position, with every attribute the +container carried** (§7a) — the editor re-creates wrappers on the next +`ApplyState`, but conditionally and in a shape that is a function of arrival +order rather than of the document, so unlike `title`/`header` what comes back +is neither the same partition nor guaranteed to come back at all; +restrictions rebuilt (§4); properties +stripped per §3 (with the exemption list), the attribution pair +`creator`/`lastModifiedBy` among them — export spells them `#` +and import drops the key, so a round trip clears both (§3); informative +reference suffixes trimmed and participant composites folded/rebuilt (§9) — +exact inverses for every id either side WRITES, and the round trip is +byte-stable for them, but three residues remain because a snapshot's +reference slots are untrusted and may hold what the format cannot spell: +**an id containing `#` loses everything from the first one**, since the +reader cannot tell that `#` from the one a caption hangs on (this is the +only place the format silently narrows a value it was handed; export no +longer captions such an id, so the loss happens once and the value is a +fixpoint after — measured across two corpora, 81,696 documents, zero occur); +**a bare account identity already stored in an object or file slot comes +back as this space's participant id**, because unfold cannot know the fold +never fired (every bare identity in the corpus sits in a text-format +property, where the object arm never runs); and **a reader wired without a +SpaceId leaves folded identities bare**, which addresses no object — it is +told so through the warning sink, once for the document, since the fault is +the wiring and every such reference in the object shares it; +select/multi_select option ids +replaced by name resolution — in properties, filter values, and custom +orders (§3, §6.2) — which `option_ids` inverts exactly (§9a), leaving two +residues: **two same-named options of one property held by ONE object** +collapse onto the first, because the document spells one string twice; and a +reader wired with no option resolver ignores the legend and keeps the names, +having no space in which an id could be an option at all; the seven +system-stamped keys of §3 come back ABSENT when their stored value was empty +(§15 #12) — a whitelist, so every other key present-and-empty still survives; +deprecated snapshot +and block fields cleared (§2, §5); deprecated `Header4` re-styled to +`heading_3` (§5); `checked` outside checkboxes dropped and marks on literal +blocks dropped (§5); marks normalized — emoji materialized, whitespace +boundaries shrunk, same-type overlaps truncated, adjacent ranges merged +(§8.3); file/bookmark `state` recomputed (§5); empty strings/arrays/objects +and default scalars dropped from block attributes and envelope fields — but +never from property values, whose presence is meaningful (§3, §4); tables +normalized and ids canonicalized +(§6.1, including empty-paragraph cells collapsing to absent cells); dataview +`activeView`, cached sort/filter formats, deprecated per-column date/time +fields and `value` on `empty`/`not_empty`/`exists` leaves dropped, group +`index` derived from order (§6.2); scalar-stored select/objects/files +property values become single-element lists and the legacy file `hash` +migrates into `object_id` (§3, §5); object types reduced to the positions §2 +models — one type, plus, on a template, the target type — with keyless +entries (`ot-`, `""`) dropped first, so the remaining entries close ranks +rather than lose the slot a keyless one would have silenced (§3); +**icon and cover reduced to the single winning variant** (§2b), which is +seven clauses of its own: +(a) the four icon channels collapse under `iconName` → `iconEmoji` → +`iconImage`, with `iconOption ≥ 1` attached as `color` to whichever won and +standing alone as the `color` variant when none did; +(b) a source whose stored value is EMPTY (`""`, `[]`, `0`, `null`) is not a +source, so a key present and empty comes back ABSENT — the one place this +format overrides §3's presence-is-meaningful rule, and it rests on all nine +relations being `hidden: true`, so no property row exists for presence to be +meaningful to (1,358 production objects carry only empty sources and end up +with no icon and no cover at all); +(c) `iconOption: 0` is the proto zero, not a colour, and is dropped; +(d) `iconImage` entries beyond the first are dropped with a warning (never +observed — the relation is `maxCount: 1`); +(e) a `file` value that is not id-shaped is dropped with a warning, because +there is no way to write it (33 production objects, every one an absolute +filesystem path a Notion import left in `coverId`); +(f) a `coverType` outside `0..5`, a `coverType` of 0, or a `coverType` with +an empty `coverId`, produces no cover, with a warning where anything was +lost; +(g) `coverScale`/`coverX`/`coverY` with no image cover to frame are dropped. +A callout's icon reduces the same way, `emoji` over `file`; and a type +object gains an empty list for every recommended role nothing occupies — +`property_definitions` (§2a) collapses the four role lists into one labelled +array, and import rebuilds all four from it, so a role the store left absent +comes back as `[]`. An absent list and an empty one say the same thing, and +the empty list is the only way this format can express a role being +*cleared*, since `property_definitions` cannot name a section that exists with no +members. Whether the object state itself should carry all four consistently +is a question about the state, not the format (GO-7451). + +The §2d relation lift adds almost nothing here, by design — presence mirrors +presence, so `false`, `[]` and `null` all survive and the three keys are +otherwise untouched — but three residues are real and stated: **a relation +snapshot with no stored `relationFormat` comes back with an explicit 0**, +because `format` is required and absent-reads-as-longtext is what every +consumer of the detail already does (never observed: all 10,617 production +relation documents carry the key); **the §3 text collapse now reaches the +relation's own definition** — a non-bundled shorttext relation read without +a format resolver comes back longtext, exactly the residue §3 states for +every other format slot (53 of 10,617 under bare options in the corpus; +zero with the space's resolver, which knows every live relation's format); +and **`object_types` entries take the §3 list normalizations** — a +scalar-stored value wraps, empty-string entries drop — while the id↔key +translation is exact for every id the store actually speaks: ids out, ids +back under the `TypeResolver` capability, verbatim both ways without it. +One residue, measured at 27 corpus relations: **a legacy bare type KEY +stored where the store speaks object ids comes back as this space's type +object id** — export passes the key through verbatim (it is no id the +resolver serves), and import writes the id the key names, which is the +store's own spelling for the same type. A respelling, not a rebinding — the +comparator normalizes both sides to keys through the same capability, the +treatment the recommended lists already get, so only a change of the type +NAMED reports. + +The deleted-icon rule (§9) adds one normalization of its own, armed only +when the wiring supplies the `ObjectDeletionResolver` capability (§13): +**an `iconImage` reference whose target is a tombstone in the space's own +store is dropped**, and the document falls through to the remaining icon +channel. The predicate is exported (`DroppedDeletedIconRef`) and the +comparator consults it on the icon/cover comparison, the same-commit +discipline every owned predicate here follows — without it the comparator +reads every dropped icon as data loss, the drift class that once produced +1,344 false failures in a single sweep. + +The missing-reference rule (§9) adds one normalization, armed only +when the wiring supplies the `ObjectExistenceResolver` capability (§13) — +under bare options it adds nothing and every reference passes verbatim: +**a reference to an object the space's store holds no row for is rewritten +to `_missing_object` in a singular slot (block `object_id`s, mention +targets) and dropped from a list slot (objects/files values, +`object_types`), and a stored sentinel follows the same split — kept in a +singular slot, dropped from a list.** The movement converges in one +generation: the first export writes the sentinel or the shorter list, +import stores exactly that, and every later export is byte-identical — so +guarantee 3 below holds, with the rewritten id's warning as its last +appearance anywhere. The predicate is exported +(`DroppedMissingObjectRef`) and the comparator applies it to BOTH sides +of the objects/files and `relationFormatObjectTypes` comparisons, the +same-commit discipline every owned predicate above follows: a +dropped-by-design entry is not loss, a live entry that vanishes still +reports, and a comparator handed no capability excuses nothing. + +The §2a `type_settings` group adds three normalizations, all scoped +to TYPE documents and all owned by exported predicates the comparator reads +(`DroppedTypeProvenanceKey`, `DroppedEmptyTypeSetting`), so the two sides +cannot drift the way that once produced 1,344 false failures in one sweep: +**the seven install-provenance keys come back ABSENT** — `layout`, +`resolvedLayout`, `smartblockTypes`, `sourceObject`, `origin`, `addedDate`, +`setOf`, each admitted to the drop individually against 1,760 corpus type +documents (the verdicts live on `typeProvenanceKeys`, §2a; `revision` was +admitted and then failed — it guards a type's own name against the bundled +reviser) — +while the same keys on any other kind survive untouched; **the five lifted +settings come back ABSENT when their stored value was empty** (`pluralName` +`""` on 145 corpus docs, `defaultTemplateId` `[]` on 87), the §4 omit-empty +canon where §2d mirrors presence, because these are settings with defined +defaults rather than a property's definition; and **a `defaultTemplateId` +with a second entry keeps only its first**, with a warning — the member is +the one default template, and 0 of 1,760 corpus documents carry more. + +The §2f dictionary adds one normalization, and it is a COMPOSITION rule +rather than a document one — the per-document codec is untouched: **a +bundled-identical relation document is not written at all**. Its key travels +in the dictionary's `installed` list, and a reader reconstructs the object +from its own bundled table, across which trip (a) the install artifacts — +`createdDate`, `origin`, `addedDate`, `sourceObject`, `revision`, +`apiObjectKey`, `featuredRelations`, `scope`, `importType`, +`lastModifiedDate`, `layout`/`resolvedLayout`, the three recommended-list +stamps — come back ABSENT, re-stamped by the next install, and (b) a +definition member the copy never stored comes back as its explicit empty +default, because an install states the whole definition. Both movements are +owned by exported predicates the comparator reads +(`OmittedBundledRelation`, `RelationInstallArtifactKey`, +`InstallStampedDefault`) and both are scoped to snapshots the omission +predicate itself admits, so the ordinary document round trip keeps its full +sensitivity. The predicate is fail-closed on every axis — a divergent +definition member, an unclassified detail key, an alien-kinded value, a +block the format preserves (19 corpus relation documents carry a dataview +or free text) each keep the document — because omitting a document that +carried real data would delete it silently, which is disqualifying for a +backup format. Each admitted artifact key passed the §15 #12 test +individually against the 9,675 bundled-key relation documents; the verdicts +live on `relationInstallArtifactKeys`, and the keys that FAILED +(`isUninstalled`, `isFavorite`, `isArchived`, the bare `includeTime`) keep +their documents. + +Export emits `blocks` in pre-order with exact depths, so export can never +produce a monotonicity violation and the flat shape does not disturb +byte-stability. Strict inputs add nothing to `N(S)`; for lenient +(`NormalizeIndent`) inputs, the clamped indents are part of the documented +normalization. **Marshal never emits a document its own validation +rejects**: a snapshot nested deeper than the indent bound (32) fails export +with an error naming the block, as does a table anywhere inside a table +cell (§6.1). + +The snapshot's block graph is untrusted: export emits each block **once** +(the first parent listing it wins), which both terminates on cyclic +`ChildrenIds` and keeps duplicate/shared blocks from producing invalid +documents; duplicate table column/row ids are likewise dropped +(implementation decision). + +**What "equivalent resolvers" requires.** Both guarantees below are stated +for export and import wired with equivalent resolvers, and for the key +vocabulary that means three things, none of which follows from the one +before it (`KeyVocabulary`, §13). One: whatever `…Slug` emits, `…Key` must +invert. Two: **no answer, in either direction, may bind a spelling that the +bundled table binds to a different key.** Three: **a live stored key +outranks the vocabulary's own NAME binding** — chain step 2 as an obligation +on the implementation, so a term that is some live entity's stored key +answers "not a spelling", and no spelling is emitted that a live stored key +answers to. Without the third, a document naming a property by its stored +key lands on whichever other property carries that string as its display +name. + +The second is what the legend can only partly rest on. A document owes an +entry for every spelling a reader's chain would bind elsewhere, and export +asks the two chains it can see: the bundled table, which ships with every +reader, and the vocabulary it is running under (§3). A third reader's +vocabulary is not one of them — so a stored key both visible chains invert is +written with no entry, and a reader whose vocabulary disagrees with the +bundled table for that spelling silently resolves it elsewhere. A vocabulary +can satisfy the first rule completely and still turn a template for the +bundled `task` type into a template for an unrelated custom type. The +vocabulary this system ships (`storeresolver`) refuses such an answer in both +directions; the rule is stated because `Options.Keys` accepts an +implementation from anyone. + +1. `Import(Export(S)) ≡ N(S)` — state-level equality on the snapshot after + normalization. +2. `Export ∘ Import` is **idempotent and byte-stable**: for any valid + document `J`, `Export(Import(J))` is the canonical form of `J`, and + re-importing/re-exporting it is byte-identical. (Byte equality with the + *original* `J` holds only when `J` is already canonical — import mints + missing ids, merges marks, maps aliases like `heading_4`/`equation`, + absorbs top-level title/description blocks, and export spells every key + with the LABEL its authority gives it now, so a document written before + the property was renamed — or before this rule — comes back naming the + same stored key with a different term. That is a change of spelling and + not of state: the label resolves through the document's own legend + first, so `N(S)` is untouched and the object is the same object either + way.) +3. `Export(S) = Export(Import(Export(S)))` — the same guarantee anchored on + the SNAPSHOT rather than on a document, and the one an object exported + twice depends on: once directly, once after a round trip through this + format. It is what §9's "re-exports diff cleanly" means for everything + that is not an id, and it is why the term census reserves only the keys + the document spells (§3). Ids are the documented exception in the same + direction as (2): a snapshot carrying a block or view with no id exports a + document that is not canonical, and import mints one. + + **The attribution pair is the second documented exception, and the only + one that is not an id.** A snapshot carrying a `creator` exports a document + naming the member; import drops the value, so the next export has nothing + to write and `Export(Import(Export(S)))` is one property shorter. Nothing + there is recoverable and none of it was data: the value is derived from the + object tree root's signature, and an imported object gets the importing + account's own from its own new tree. What still holds — and is what a + re-export diff actually depends on — is that the loss happens **once**: + `Export(Import(Export(S))) = Export(Import(Export(Import(Export(S)))))`, + so every export after the first is byte-identical to the next. + +Both properties are enforced by tests in the package: golden-file tests for +representative documents plus property-based round-trip tests over generated +states (all block types, mark overlap/adjacency/whitespace-boundary cases, +emoji, tables, dataviews, UTF-16 payloads such as astral-plane characters). + +## 12. Validation + +**What earns a check.** A validation rule has to meet both of these, or it +does not belong here: + +1. **It catches something silent.** The document validates, converts and + imports, and is wrong somewhere the author will not look — a width read as + pixels when written as a percent, a `group_by` the view cannot honour, a + `less` on a date matching every record that has none, a target type that + resolves to nothing. If the defect is visible the moment the object is + opened, looking at the result catches it and a check only adds noise. +2. **It traces to a mechanism.** Every rule below points at the code that + makes it true. A rule justified by taste rather than by behaviour cannot + be argued with, and mixing the two is what turns warnings into something + readers skip. + +The cost of a marginal check is not the code, it is that every warning +becomes cheaper to ignore — including the ones that matter. Conventions that +fail neither test belong in authoring guidance and in review. + + +- Schema: JSON Schema **draft 2020-12**, hand-authored (the format + deliberately diverges from proto shape), one file, blocks discriminated on + `type`. The block definition is **non-recursive** — the flat encoding has + no `children`, and table cells reference a dedicated `cellBlock` + definition (same core, no table arm) so the block↔cell cycle is cut — + which is what makes the block schema usable under strict/constrained + decoding. The one remaining recursive definition + is the dataview **filter tree** (`filterNode` groups nest, §6.2) — it is + inherent to the filter model; a reduced core-profile schema (planned + follow-up) without dataview is fully non-recursive, and the compact + filter string (§6.2.1 — its parser ships as the `filterstring` + subpackage for the API query surface; the *document* field stays + reserved) removes it from the generation path. To keep validation errors usable for LLM + producers, validation dispatches on the `type` const first (per-type + `if/then` or programmatic pre-dispatch) instead of a flat 30-branch + `oneOf` whose error output is noise. **The same rule governs every + discriminated union in the schema**, and `icon`/`cover` (§2b) are where it + was measured rather than assumed: `oneOf` reported 10 issues for one wrong + member and never named the alternatives, `if`/`then` reports one and does. + Output-only fields carry + `x-output-only: true` (§4a). Annotated `x-app: Anytype` in line with + `pkg/lib/schema`. +- Published at a stable URL and embedded in the package (`go:embed`); + validated with `santhosh-tekuri/jsonschema/v6` (new dependency; the repo's + existing `gojsonschema` is draft-07 only). +- Import = schema validation first, then semantic checks the schema can't + express: **indent monotonicity** (§4 validity — errors name both + indents), **leaf containment** and **row→column** (§4 containment, judged + against the tree §7a's lift builds and naming the effective parent), id + uniqueness over the whole document (§4), table shape and cell rules + (§6.1, a cell block that is a transparent container included), envelope combinations (`items`/`template_for`/`kind`, §2), + **property-key admission on the resolved stored key** (§3 — each + `properties` spelling resolves through the §3 chain before the deny rule, + the enum-name check and the format-shape warning run; validation + mirrors the importer's details seam refusal for refusal — a **denied** + resolved key, an **unwritable** resolved key, and **two spellings binding + onto one stored key** are all errors — and a `property_internal_keys` *value* is + admitted like the stored key it is, deny rule included; import re-runs + the seam's checks on its own resolved key when a wider vocabulary is in + force; the TYPE namespace mirrors the same way, minus one thing it used to + need: the `/template_for` gate and the kind read `kind` alone, so neither + runs the §3 chain and `Validate` no longer keeps a private copy of it, a + `type_internal_keys` spelling or value gets the same writable-key restatement as + `property_internal_keys`, and the import seam refuses a term a vocabulary resolves + onto the empty type key, §3), **a typed field with no `format`** (§2b — the + schema's `required` says a member is missing but not that it is a CHOICE, + so the reader states the alternatives, reading them out of the published + schema rather than restating them, and the schema's own verdict at that + pointer is suppressed so the document still gets one fault, one issue), + `language`-vs-`fields.lang` conflicts, an **`option_ids` key naming a + property this document never spells** (§9a — a warning: the entry can never + be consulted and the value degrades to name resolution; a key-set + comparison against the document's property census, not a parse of the key), + and + **inline-markup parsing** (§8) — grammar errors report the block's JSON + path and the offending snippet. The indent bound [0, 32] lives in the + schema. +- **Validate and Unmarshal agree, in both directions.** Whatever Validate + accepts, Unmarshal decodes; whatever Validate rejects, Unmarshal rejects + too, with the same path-addressed issues. This is the promise that makes + Validate worth calling — "this document imports" — and it constrains the + reader in two places where JSON's value model is wider than Go's: + - JSON Schema counts `2048.0` and `1e3` as integers, so every + schema-integer field (`indent`, `size`, `limit`, `page_size`, column + `width`) is read as a JSON number and converted, never decoded straight + into an `int64`/`int32`; and each carries `minimum`/`maximum` for the + range its stored type can hold, so the conversion cannot truncate. + - a JSON number has no range, `float64` does, and every number in this + format ends up in one (proto `Struct` values are doubles). A number + outside `float64` — `1e400` — is therefore rejected wherever it appears, + including on the loose surfaces (§3 property values, block `fields`, + `store`, filter values) that have no schema bound by design. + Both are enforced by a corpus invariant test over hand-written documents; + a corpus generated from export cannot find these, because export never + writes them. +- These path-addressed errors are the guardrail for agent-generated + documents: generate → validate → feed errors back. With the flat schema + the generate step can also run under strict/guided decoding end to end. +- **One fault, one issue.** Because the errors are fed back to a generator, + an issue that is *confidently wrong* costs more than a missing one: an + agent told `property "type" is not allowed` removes `type`, and its next + attempt is further from valid. Two mechanics in the schema produce such + issues, and the reader prunes both rather than passing the validator's + bookkeeping through (implementation decision, `validate.go`): + - `unevaluatedProperties: false` is what closes a block to the fields its + `type` admits, but it can only see the properties that *successfully* + evaluated subschemas annotated. One bad field makes the type's subschema + fail, and then every property of that block is reported as not allowed. + So a "not allowed" verdict is **dropped** when the object it belongs to + failed for some other reason *and* the property name appears somewhere in + the schema. A name the schema never mentions cannot be admitted under any + reading, so that verdict stands — a hallucinated key is still reported + alongside the real fault, in the same round. + - an `anyOf` (table cells, §6.1) reports every branch it tried. Branches + that failed only on the instance's type never applied, so they are + dropped; if none applied, they merge into one issue naming every + admissible shape. + A reader that reports more than this is not wrong about the document being + invalid, but its extra issues are not statements about the document. +- **Three warnings watch the raw-name seams**, all cheap, none a refusal + (introduced with the raw-name re-spell). (i) A key spelling carrying edge + whitespace or an invisible (default-ignorable) code point — 8 of 767 + measured production names do — draws a hygiene warning at Validate: the + name is carried exactly as the space holds it, and an exact match must + reproduce bytes the eye cannot check; a cleanup belongs where the entity + is named, not at this seam. (ii) At import, a term that resolves verbatim + and EXTENDS a live or bundled name past a word boundary + (`Lists [in work] (text)`) is warned as a probable glued annotation — the + one real raw-name failure shape the generation eval produced. (iii) At + import under a space-backed vocabulary, a term that resolves verbatim and + is no live entity's stored key is warned as the stale-or-guessed-name + phantom — every name-addressed scheme's shared hole, named at the seam. + Warnings (ii) and (iii) fire once per term per document, at every key + slot, both namespaces: the diagnosis is a fact about the term, not about + any one slot. +- **A malformed `option_ids` entry is an error; an unconsulted one is a + warning.** The two look like degrees of one fault and are not. An outer key + naming a property the document never spells is **well-formed content the + document does not need**, and §9a permits a legend to carry more than one + document needs — refusing it would refuse a legitimate document. A value + that is empty or not a string is **not a legend entry at all**: the slot is + typed as an option id and holds something that is not one, under no reading + of the format. Three things keep it an error. The published schema types the + slot (`{"type": "string", "minLength": 1}`), and the promise above is that + an external validator running that schema *and nothing else* reaches the + same verdict — downgrading the reader would make `Validate` accept what the + schema we publish rejects, and the only way to close that divergence is to + loosen the schema until an id slot admits `null` and `12`, at which point it + no longer describes the format. `Marshal` never writes one (§11, I1), so the + sole source is authoring — the case that can still be fixed. And the cost is + bounded and already paid correctly: the fault is reported **once**, at the + member's own pointer (`/option_ids//`), not as a verdict + about the document. The objection this answers — that a whole document, + blocks and all, is refused over a field a reader may legitimately ignore — + is equally true of every typed member of the envelope, and singling out + `option_ids` would make it the one member whose type is advisory. +- **An issue names the member it is about.** The key slots are the one place + where the schema cannot: `propertyNames` — the writable-key rule on + `properties`, on `property_internal_keys` and `type_internal_keys` spellings (§3), and on + `option_ids` outer keys (§9a) — is checked by validating each name as a + *standalone string + instance*, so the verdict carries neither the enclosing object's location + nor, for a length bound, the name itself. A 200-character property key was + reported as `maxLength: got 200, want 128` at the document **root**, which + names no property at all. The rule stays in the published schema, because + an external validator runs that and nothing else, and the reader + **restates** it where the key is in hand: `/properties/`, + `/property_internal_keys/`, `/type_internal_keys/`, + `/option_ids/` and `/option_ids//`, with the + offending string in the message. A `property_internal_keys` *value* is covered the same way — the schema + addresses it correctly but describes the bound rather than the string. The + schema's own verdict is suppressed only for the members the restated check + spoke for, so if the two statements of a rule ever diverge the document is + still refused, with the schema's wording, rather than passed. Every issue + path is a JSON **pointer**, so a segment taken from the document is escaped + as one (RFC 6901: `~` → `~0`, `/` → `~1`); a stored key may hold either + character. Both statements of a rule build it that way, which is what makes + the suppression above possible at all — it is keyed by pointer, so one + unescaped spelling is one fault reported twice. + + The **envelope** was the other place the schema could not name the member, + for a different reason: it closes with `additionalProperties: false`, whose + verdict names every unknown member of one object *inside its own text* and + carries the **object's** location — `additional properties 'refs' not + allowed`, at the document root. Inside a block the same fault is addressed + correctly, because blocks close with `unevaluatedProperties`, which the + library reports per member; so the promise held everywhere except the + envelope, which is exactly where a document written against an older grammar + fails. The reader splits that verdict into **one issue per member, at its + own pointer**, in sorted order — the names are collected by ranging over the + instance's map, so unsorted they come back in a different order run to run. + Unlike an unevaluated-property verdict these are never pruned: + `additionalProperties` consults only its own schema object's `properties` + and `patternProperties`, which always evaluate, so its verdict never depends + on a sibling subschema having succeeded, and the unreliability the pruning + exists for cannot arise. +- **A removed key is told what replaced it.** `children` and `refs` are the + two names a document written against a superseded grammar brings, and for + both the bare "not allowed" points at the wrong repair: drop the subtree + rather than flatten it into `indent` (§4), and delete the legend rather than + expand what it inverted (§9a) — which strands every short label in the + document as an id that addresses nothing. Each is answered instead with the + rule that replaced it and the repair to make. This is the whole of the + migration story for a document written before a rule changed, because + `version` does not move for a pre-release grammar change (§10): the + diagnostic is the only notice such a document gets. +- **A superseded MEANING gets the same treatment, and needs it more.** A + removed key at least fails on its own — the schema has never heard of it. + A member whose meaning changed still validates, and imports as something + else. There is one: `{"type": "template"}` with no `kind` meant a template + in an earlier revision, and means an ordinary page whose type is the Template type + after it (§2). Both readings are well-formed, so nothing structural + separates them, and the failure is silent in the worst way available — the + object arrives, under the wrong kind, invisible to every template check + downstream. So the shape is refused outright, by name, with the repair + (`add "kind": "template"`), and export never emits it: a page whose type + term is literally `template` keeps its `kind` explicit, or `Marshal` would + be writing what `Validate` rejects (I1, §11). The refusal is a byte + comparison on the raw spelling — no legend, no vocabulary — because it + identifies a byte sequence a previous version of this format wrote rather + than resolving a type. It is deletable at the version bump. + +## 13. Package layout and API + +``` +pkg/lib/anyblockjson/ + SPEC.md — this document + PRINCIPLES.md — the design rules the format answers to, + with the priority order for conflicts + PRINCIPLES_SHORT.md — the same rules on one screen + ANOMALIES.md — real-world data anomalies found by prod + round-trip testing, and how the format + handles each + schema/object.schema.json — the published JSON Schema (embedded) + schema/index.schema.json — the bundle index's own schema (§2c, embedded) + schema/authoring/ — the authoring subset (§2g): one self-contained + schema per surface, embedded like the full + three; strict subsets, enforced by test + authoring.go — the authoring subset surface (§2g): + ValidateAuthoring and its index/dictionary + siblings — the full validation first, then + the subset schema + export.go — snapshot → JSON + import.go — JSON → snapshot + inline.go — marks ↔ inline markup codec (§8) + table.go — table subtree ↔ columns/rows + dataview.go — dataview content mapping (§6.2) + optionrefs.go — the `option_ids` legend and the whole of + option resolution (§3, §9a): the export site + that records an entry, the one import function + that resolves a select value, and the property + census Validate's reachability warning is + taken against + validate.go — schema + semantic validation + json.go — ordered canonical-JSON writer, enum tables, + proto value bridges, id helpers + typeproperties.go — property_definitions ↔ recommended lists (§2a); + GenerateSchema derived artifacts are planned + here (post-v1) + keyvocab.go — KeyVocabulary: the stored key ↔ spelling table, + both namespaces, and the bundled default (§3) + label.go — the label rule (§3): what a document spells + for a key the bundled table does not speak + for — the display name, NFC and verbatim. A + vocabulary calls it; the codec never does + bundledname.go — the bundled name tables (§3): key ↔ display + name, both namespaces, the forgiving fold, + and the collision-ladder helper + blockvocab.go — the block-type name tables (§5) + viewvocab.go — the dataview enum name tables (§6.2) + fragment.go — the FRAGMENT surface: one block, a flat run, + one property value, the §8 inline codec (below) + filters.go — the fragment surface for a §6.2 filter tree and + sorts array, standalone (query paths) + index.go — the bundle index (§2c) + storeresolver/ — the space-backed implementations of the four + resolvers, including KeyVocabulary; the only + place that reads a space's display names, and + the one that applies the §3 label rule to + them + snapshotdiff/ — snapshot ↔ snapshot diffing for the PATCH path + compose/ — the bundle-level composition (§2c, §2f): the + path plan, the concurrent-emit composer that + accumulates the dictionary, manifest and + index lift, and the used-key census — shared + by the production exporter + (core/block/export/anyblock) and the cmd + tools, so the sweep exercises the shipping + composition rather than a private copy + filterstring/ — compact filter string parser (§6.2.1): + string → §6.2 filter tree, offset-addressed + errors (planned with API v2 Phase 4; the + document-field integration stays post-v1) + markdownblocks.go — ParseMarkdownBlocks: block-level markdown → + a §4 flat run (id-less). Inline text passes + through verbatim as §8 markup source; only + the block slicing (headings, lists/indent, + fences, quotes, dividers, tables) lives + here. Never fails: unknown constructs + degrade to paragraphs, indents clamp per + the §4 lenient rule, and every output run + imports through UnmarshalBlocks (by test). + Built for API v2 Phase 5 (the insertBlocks + markdown payload and the create shortcut). + roundtrip_test.go — §11 property tests + state assertions + golden_gen_test.go — golden files (testdata/, -update to refresh) +``` + +```go +// FormatResolver reports the format of a property key, when known. +// Bundle properties are resolved internally; the resolver covers custom keys. +type FormatResolver func(key domain.RelationKey) (model.RelationFormat, bool) + +// OptionResolver maps select/multi_select option ids to names on export and +// names to ids on import (creating options is the import wiring's job). +// OptionName carries a second duty on the import side: it is the liveness +// question `option_ids` is checked against — it answers for an id exactly +// when that id is an option of that relation here (§3, §9a). +type OptionResolver interface { + OptionName(key domain.RelationKey, id string) (string, bool) + OptionId(key domain.RelationKey, name string) (string, bool) +} + +// PropertyDefinition describes a property object referenced by a type +// document (§2a). Options is the declared select vocabulary in display +// order; ObjectTypes restricts which types an objects/files property may +// point at, given as STORED type keys. +type PropertyDefinition struct { + Key domain.RelationKey + Name string + Format model.RelationFormat + Options []OptionDefinition + ObjectTypes []string +} + +// PropertyResolver maps property object ids to definitions on export and +// definitions back to ids on import; PropertyId receives the full definition +// so the wiring can create-and-return missing properties in one step (§2a). +type PropertyResolver interface { + PropertyById(id string) (PropertyDefinition, bool) + PropertyId(def PropertyDefinition) (string, bool) +} + +// ParticipantResolver names the space member a participant id stands for, +// for the derived attribution properties creator/lastModifiedBy — spelled +// # (§3, §9). EXPORT ONLY, and there is deliberately no +// inverse: a display name is a label, not an address — two members of one +// space may share one — and both properties are derived from the object +// tree's own signature, so an importer has nothing to do with the value +// even if it could resolve it. Answering false writes the id bare: the id +// is the resolvable half and is complete without its caption. +type ParticipantResolver interface { + ParticipantName(id string) (string, bool) +} + +// ObjectNameResolver names the object behind a reference, for the +// informative #name suffix (§9). EXPORT ONLY, behind Options.RefNames; +// import trims the suffix without asking anyone. Answering false writes the +// reference bare — never a partial or invented suffix. storeresolver +// implements it from the space index (one point lookup, cached). +type ObjectNameResolver interface { + ObjectName(id string) (string, bool) +} + +// ObjectExistenceResolver answers whether the space's store holds an object +// under an id — the missing-reference rule's question (§9). An optional +// capability of Options.ResolveObjectNames, discovered by type assertion +// (the TypeResolver pattern); storeresolver implements it off the same +// cached point lookup ObjectName pays for. It is a SEPARATE question from +// ObjectName deliberately: that seam's ok is name != "", which reads +// "exists but untitled" as "no" — using it as an existence check rewrites +// live references. known=false (a store failure) moves nothing; a +// tombstone row is a row, so a deleted object's references are untouched +// everywhere but the icon slot, whose optionality earns it the narrower +// ObjectDeletionResolver capability (§9). +// With no implementation wired, export rewrites and drops NOTHING. +type ObjectExistenceResolver interface { + ObjectExists(id string) (exists, known bool) +} + +// ObjectDeletionResolver is the narrower capability the icon slot uses: an +// icon reference to a TOMBSTONED object is dropped, where an ordinary +// missing reference is rewritten (§9, §11). Discovered by type assertion on +// ResolveObjectNames, like ObjectExistenceResolver. With no implementation +// wired, no icon reference is dropped for deletion. +type ObjectDeletionResolver interface { + ObjectDeleted(id string) (deleted, known bool) +} + +// Marshal serializes a snapshot into canonical AnyBlock JSON. +func Marshal(sbType model.SmartBlockType, snapshot *model.SmartBlockSnapshotBase, opts Options) ([]byte, error) + +// Unmarshal validates data and reconstructs a snapshot. +// Errors wrap *ValidationError with JSON-path–addressed issues. +func Unmarshal(data []byte, opts Options) (model.SmartBlockType, *model.SmartBlockSnapshotBase, error) + +// Validate checks data against the embedded schema and semantic rules +// without building a snapshot. +func Validate(data []byte) error + +// ValidateAuthoring checks data against the FULL validation above and then +// the authoring subset schema (§2g), so a nil return means valid AnyBlock +// JSON, not merely subset-shaped. ValidateAuthoringIndex and +// ValidateAuthoringPropertyDictionary do the same for the other two +// surfaces; AuthoringSchemaURL and siblings name the published locations. +func ValidateAuthoring(data []byte) error + +// DetectFormat reports the version and $schema markers without validating — +// the cheap dispatch probe for import wiring. +func DetectFormat(data []byte) (version int, schemaURL string, ok bool) + +// FormatVersion (= 1) and SchemaURL (the published schema location) are +// exported constants for the wiring's dispatch. + +// KeyVocabulary translates between the STORED keys a snapshot carries and +// the SPELLINGS a document writes, in both namespaces and both directions +// (§3). The default is BundledKeyVocabulary — the name table that ships +// with every reader, and nothing else. storeresolver supplies the +// space-backed one. Three preconditions on an implementation, none implied +// by the one before it: it inverts what it emits; it never binds a spelling +// the bundled table binds elsewhere; and a live stored key outranks its own +// name binding (§3, §11). +type KeyVocabulary interface { + PropertySlug(key string) string + PropertyKey(slug string) (key string, ok bool) + TypeSlug(key string) string + TypeKey(slug string) (key string, ok bool) +} + +// ScopedKeyVocabulary is an OPTIONAL capability a KeyVocabulary may also +// carry, discovered by type assertion on Options.Keys. Display names are not +// unique, so a map-less reader meeting a shared spelling needs the space's +// candidate lists to resolve it within the declared type instead of guessing +// (§3); without this capability an ambiguous spelling is an error naming the +// legend as the repair. storeresolver implements it. +type ScopedKeyVocabulary interface { + // Every live key whose exact document spelling is the term, as a sorted + // set. Says nothing about stored keys: verbatim-first is the caller's + // step, asked before this one. + PropertyKeyCandidates(spelling string) []string + TypeKeyCandidates(spelling string) []string + // The stored property keys a type declares — the disambiguating scope + // for a shared property name, counted once per key. + TypePropertyKeys(typeKey string) []string + // Diagnose one term for the verbatim-resolution warnings (§12). + PropertyTermFacts(term string) KeyTermFacts + TypeTermFacts(term string) KeyTermFacts +} + +// Legend carries the three legends of the document a FRAGMENT was cut out +// of, so a fragment entry point runs the §3 chain from step 1 rather than +// from the reader's vocabulary. Marshal and Unmarshal ignore it: a whole +// document carries its own. The zero value is "no legend". +type Legend struct { + PropertyKeys map[string]string // spelling → stored relation key (§3) + TypeKeys map[string]string // spelling → stored type key (§3) + OptionIds map[string]map[string]string // {spelling: {option name: id}} (§9a) +} + +type Options struct { + ResolveFormat FormatResolver // optional; nil = bundle-only resolution (§3) + ResolveOptions OptionResolver // optional; nil = option values pass through as ids + ResolveProperties PropertyResolver // optional; nil = type documents keep raw recommended-relation ids (§2a) + ResolveParticipants ParticipantResolver // optional; export only. nil = attribution ids written bare (§3) + ResolveObjectNames ObjectNameResolver // optional; export only. The #name suffix rides it behind RefNames; + // nil = references written bare (§9). An implementation may also carry + // ObjectExistenceResolver (type-asserted), which arms the + // missing-reference rule (§9) — without it nothing is rewritten or dropped. + SpaceId string // the space this run reads from / writes into; arms the + // participant fold in BOTH directions — empty disables it (§9). + // Supplied by the wiring exactly as resolvers are; the format + // itself carries no space id. + RefNames bool // export only: write the informative #name suffix on object + // references (§9). Default off — the backup shape stays minimal + // and rename-stable; read shapes opt in. + Keys KeyVocabulary // optional; nil = BundledKeyVocabulary (§3) + Legend Legend // fragment entry points only: the enclosing document's legends (§3) + OmitIds bool // export only: drop every id, the option_ids legend included (§9, §9a) + CompactBlockLabels bool // export only: relabel doc-local block/row/column/view ids (§9a; lossy, legend-less) + CompactIds bool // export only: alias for CompactBlockLabels — object refs are never compacted (§9a) + GenerateId func() string // import only: id generator for missing ids; + // nil = random 24-hex (editor-shaped). The wiring + // passes the editor's generator. + NormalizeIndent bool // import only: clamp over-deep indents instead of + // rejecting (§4 lenient mode) + OnWarning func(Issue) // optional sink for warning-grade issues + // (NormalizeIndent clamps, path-addressed) + // CompactFilters (reserved): filters as query strings — post-v1, §6.2.1 +} +``` + +### 13.1 The fragment surface + +The entry points above take a whole document. The **fragment surface** takes +a piece of one — a single block, a flat run, one property value, one filter +tree — for wiring that edits a live object op-by-op instead of round-tripping +it (API v2 PATCH). It is the same codec throughout: a fragment run is +validated by wrapping it in a synthetic document, so §4 monotonicity and the +§5 per-type shape rules apply unchanged. + +```go +// MarshalBlockSubtree serializes one block subtree into a fragment envelope: +// {"property_internal_keys": {…}, "option_ids": {…}, "blocks": […]} — plus "type_internal_keys", +// which no block slot can owe today, so it never appears in practice +// — the flat §4 run beside the legends its blocks owe, in the envelope's own +// member order. OmitIds and the compaction flags are REFUSED here. +func MarshalBlockSubtree(subtree []*model.Block, opts Options) (json.RawMessage, error) + +// UnmarshalBlocks converts a flat run into model blocks with the ChildrenIds +// graph wired; topIds names the run's top-level blocks, ready for a splice. +func UnmarshalBlocks(run []json.RawMessage, opts Options) (blocks []*model.Block, topIds []string, err error) + +// UnmarshalBlock converts one block object into its model block(s); forcedId, +// when non-empty, keeps a replaced block's identity. +func UnmarshalBlock(raw json.RawMessage, forcedId string, opts Options) ([]*model.Block, error) + +// MarshalPropertyValue converts one property value to its JSON form, plus +// this key's share of the option legend — {option name: option id} (§9a). +func MarshalPropertyValue(key string, v *types.Value, opts Options) (any, map[string]string) + +// UnmarshalPropertyValue is its inverse. `key` is a STORED key, not a +// spelling; the option legend arrives through Options.Legend.OptionIds. +func UnmarshalPropertyValue(key string, v any, opts Options) *types.Value + +// UnmarshalFilters and UnmarshalSorts convert a standalone §6.2 filter tree +// or sorts array — the query paths, which carry no document. +func UnmarshalFilters(raw json.RawMessage, opts Options) ([]*model.BlockContentDataviewFilter, error) +func UnmarshalSorts(raw json.RawMessage, opts Options) ([]*model.BlockContentDataviewSort, error) + +// BuildRecommendedLists is the PATCH-type door into the §2a array +// applyTypeProperties reads out of a document: it resolves a typeProperties +// array into the four recommended-relation id lists, refusing exactly what +// the document path refuses. +func BuildRecommendedLists(props []TypeProperty, opts Options) ([]RecommendedList, error) + +// ParseInlineText and RenderInlineText are the §8 inline codec, exported: +// the single-field pair Marshal uses for every text-bearing block. +func ParseInlineText(md string) (string, []*model.BlockContentTextMark, error) +func RenderInlineText(text string, marks []*model.BlockContentTextMark) string + +// ParseMarkdownBlocks slices block-level markdown into a §4 flat run +// (markdownblocks.go). Never fails; unknown constructs degrade to paragraphs. +func ParseMarkdownBlocks(md string) []json.RawMessage +``` + +**A fragment has no envelope, so it has no legend of its own — and that is +the one thing every entry point here has to be handed.** The §3 chain's +first and highest step is the document's own statement about its spellings, +and a fragment cut out of a document that said +`property_internal_keys: {"priority": "6a32d485…"}` carries the spelling and not the +statement. Resolved through the reader's vocabulary alone, `priority` lands +on whichever relation THAT space gives the spelling to — the exact +misresolution §3 wrote the legend to prevent, at the one seam that writes to +a live object. So: `MarshalBlockSubtree` and `MarshalPropertyValue` **return** +the legends their output owes, and every reading entry point takes them back +through **`Options.Legend`**. A caller that assembled the fragment itself +leaves the field zero. + +**`OmitIds` and the compaction flags are refused on a fragment, not +ignored.** This surface exists to address a live document, and both take the +addresses away: `OmitIds` drops every block id, the view id and the filter +id, so the run says what to write but not where; block-label compaction +rewrites doc-local ids to short suffixes that are local to the emitted run +and are not the object's ids at all. Either produced a fragment that reads +correctly and cannot be applied. + +Other exported helpers, in service of the same wiring: `ValidateWarn` +(validation with a warning sink), `SchemaJSON` (the embedded schema bytes), +`InternalPropertyKeys` (what §3 strips), `IsCompactLabelShaped`, +`LeafBlockType` / `TextBlockType`, `FormatName` / `FormatByName`, the four +vocabulary listers, and the `index.json` namespace helpers (§1, §2c): +`IsPlatformId`, `IsReservedWidgetTarget`, `IsImportableWidgetTarget`, +`ReservedWidgetTargets`, `IsReservedHomepage`, the translators between a +reserved name and the importer's own bare spelling (`WireWidgetTarget`, +`FormatWidgetTarget`, `WireHomepage`, `FormatHomepage`), and the widget +object's omission seam (§2c): `OmittedWidgetObject`, `IndexFromWidgetObject`, +`WidgetObjectResidualKey`, and `WidgetsSnapshot` — the one builder both +`cmd/anyblockconvert` and the round-trip verifier use, so the archive a +bundle installs from and the reconstruction the sweep verifies are the same +bytes by construction. + +The bundle index (§2c) has its own pair, since it is not an object snapshot, +and the property dictionary (§2f) another, on the same reasoning: + +```go +func UnmarshalIndex(data []byte) (*Index, error) +func MarshalIndex(idx *Index) ([]byte, error) +func UnmarshalPropertyDictionary(data []byte) (*PropertyDictionary, error) +func MarshalPropertyDictionary(d *PropertyDictionary) ([]byte, error) +``` + +The dictionary's Go surface is `[]PropertyDefinition` — the same struct the +resolvers speak and both doors of the §2a array build — rather than a +dictionary-local entry type: §2e's one-shape rule holds on the Go side too, +and a fourth field list is how a fourth spelling starts. + +The §2f composition predicates are exported beside them, for the wiring +that composes a bundle and the comparator that verifies one: +`OmittedBundledRelation` (may this relation document be omitted, and under +which `installed` key), `InstalledRelationDetails` (the reconstruction a +reader builds from that key), `RelationInstallArtifactKey` and +`InstallStampedDefault` (the two movements the omission trip makes, which +`snapshotdiff.Compare` reads rather than restates). + +The DROP predicates of §9 and §11 are exported for the same reason — the +comparator has to read the rule export applied, or a deliberate drop reads +back as data loss: `DroppedMissingObjectRef` and `DroppedDeletedIconRef` +(the two reference drops, §9), `DroppedTypeProvenanceKey` and +`DroppedEmptyTypeSetting` (the type-document admissions, §2a), plus +`DroppedParticipantProvenanceKey`, `DroppedEmptyIconCover` and +`DroppedEmptySystemProperty`. Each answers for one normalization `N(S)` +names (§11). + +The package is deliberately **pipeline-agnostic**: it depends only on +`pkg/lib/pb/model`, `core/domain`, `pkg/lib/bundle`, `util/text`, the proto +runtime (`gogo/protobuf/types`) and `santhosh-tekuri/jsonschema/v6` (§12). +It must not import anything from `core/block/import` or `core/block/export` +— including `anymark`; the inline codec is implemented in-package because +canonical, byte-stable rendering needs stricter guarantees than `anymark`'s +best-effort import parsing, while staying syntax-compatible with it (§8.1). + +Wiring status — export landed, import is the follow-up: +- Export SHIPS: `core/block/export/anyblock` is the production exporter, + wired straight into the export service's format switch + (`core/block/export/export.go`) rather than through a + `converter.Converter` shim — the earlier plan for a + `core/converter/anyblockjson` shim was superseded by that wiring and no + such package exists. It passes the storeresolver-backed format, option + and key resolvers (§13) and shares the `compose` composition with the + cmd tools, so the sweep exercises the shipping path. +- Import (follow-up, not this package): an entry that dispatches on the + `version`+`$schema` markers so `RpcObjectImportRequest` accepts the + format. This must be built on the ImportV2 engine (branch + `go-7349-import-refactor`), not the legacy import pipeline, and must + supply resolvers equivalent to the export side's (§3), including + create-missing-option behavior. + +## 14. Full example + +```json +{ + "$schema": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "version": 2, + "id": "bafyreieqh63jv…", + "type": "Page", + "icon": { "format": "emoji", "emoji": "🔥" }, + "cover": { "format": "gradient", "gradient": "pinkOrange" }, + "properties": { + "Name": "Project Phoenix", + "Status": ["In progress"] + }, + "option_ids": { + "Status": { "In progress": "bafyrei…opt1" } + }, + "blocks": [ + { "id": "b1", "type": "heading_2", "text": "Goals" }, + { "id": "b2", "type": "paragraph", + "text": "Ship the **new export** by Q3 with Roman" }, + { "id": "b3", "type": "bulleted_list_item", "text": "Flat JSON schema" }, + { "indent": 1, "id": "b4", "type": "bulleted_list_item", "text": "Validate in CI" }, + { "id": "b5", "type": "checkbox", "checked": true, "text": "Draft spec" }, + { "id": "b6", "type": "code", "language": "go", + "text": "func main() {\n\tfmt.Println(\"hi\")\n}" }, + { "id": "b7", "type": "table", + "columns": [ { "id": "c1" }, { "id": "c2", "width": 120 } ], + "rows": [ + { "id": "r1", "is_header": true, "cells": [ "Format", "Size" ] }, + { "id": "r2", "cells": [ "pb.json", "3020 B" ] } + ] }, + { "id": "b8", "type": "callout", "icon": { "format": "emoji", "emoji": "💡" }, + "text": "See the [ADF docs](https://developer.atlassian.com/cloud/jira/platform/apis/document/structure/) for the reference shape" }, + { "id": "b9", "type": "dataview", + "object_id": "bafyrei…tasksSet", + "properties": [ + { "property": "Name", "format": "text" }, + { "property": "Status", "format": "select" }, + { "property": "Due date", "format": "date" } + ], + "views": [ + { "id": "v1", "type": "kanban", "name": "By status", + "group_by": "Status", + "sorts": [ + { "property": "Due date", "direction": "asc", "empty_placement": "end" } + ], + "filters": [ + { "property": "Due date", "condition": "less", "date_preset": "current_week" }, + { "property": "Done", "condition": "equal", "value": false } + ], + "columns": [ + { "property": "Name" }, + { "property": "Due date", "width": 120, "align": "right" }, + { "property": "Status", "aggregation": "count_distinct" } + ] + } + ] + } + ] +} +``` + +## 15. Decisions and deferrals + +The draft kept its open questions here. At freeze the ledger is verdicts: +what was decided and where each rule now lives, what is deliberately +deferred past v1, and the one item still genuinely open. Item numbers are +stable — the rest of this document, the code, and `specclaims_test.go` +cite them as §15 #N — and the house style stands: a rejected design keeps +the decision, the overturned position, and the evidence that killed it, +the evidence pinned as assertions in `specclaims_test.go` so a rejected +design cannot come back after its counter-evidence has quietly stopped +being true. + +### Decided + +- **#1 Extension** — settled: `.anyblock.json`. A FAT bundle legitimately + carries blobs that are themselves `.json` files (12 corpus file objects + have `file_ext == "json"`), so "is this file a document" needs one cheap, + collision-free test, and the double extension is that test — the entire + skip rule for non-document files, at zero cost. `$schema`/`version` + disambiguate the three grammars only once a file IS a document. + +- **#2 `dataview` vs `database`** — kept `dataview`: ownership semantics + differ from a database table. A judgment call, recorded as one. + +- **#3 Option names vs `{id, name}` objects** — settled: names stay in the + value, generatable and readable, and the id rides beside them in + `option_ids`, under the property that owns the option (§9a). Three + alternatives were each proposed more than once; each is falsified by + evidence pinned in `specclaims_test.go`. + + - **A flat legend map with a separator** (`#`, deleted). No separator + survives real names: `bundle.ApiSlug("C#") == "c#"` and + `ApiSlug("#1 priority") == "#1_priority"`, so `#` appears inside both + halves of the joined key. The nested shape needs no separator (§9a). + - **A sigil in the value** (`"@opt-high"` marking a handle). A legal + property slug can BEGIN with the sigil — `ApiSlug("@home") == "@home"` + — and `Validate(data []byte) error` takes no resolver (§13), so it + must accept the sigil everywhere (breaking I2) or refuse it where + Marshal emits it (breaking I1). Export's deep links are not the + counter-example they look like: `objectLinkDest` percent-encodes, so a + leading `@` is written `%40`. + - **`{name, id}` value pairs.** Not the format-only change it was + believed to be: `model.RelationOption` is + `{Id, Text, Color, RelationKey, OrderId}` — no key field — so the + store cannot supply the stored keys the byte-cost argument rested on. + It also puts a second value shape in the slot small models write most + often. + + One argument that must not come back attached to any of these: the sigil + designs were largely defended as protecting `object_ids` against a + dropped legend. Object-reference compaction was deleted and `object_ids` + never shipped — the only `object_ids` in this format is the dataview's + `object_orders[].object_ids` (§6.2); object references print in full, + everywhere, and need no legend. The §9 `#name` suffix is not that legend + returning: a caption on a full id, inverted by deletion, split id-first — + the `#` inside an option NAME that killed the flat legend provably + cannot reach the id half. + +- **#3a Attribution spelling** — settled twice; the second answer stands. + The first spelled `creator`/`lastModifiedBy` as the member's display + name alone; the standing rule is a resolvable id with the name as the + informative `#name` suffix (§3, §9). Name-only broke API v2's need for a + resolvable id, and a display name shared by two members (76 of 2,478 in + production) identifies neither. `CREATOR_SPEC.md` (outside this repo) is + SUPERSEDED on this point. + +- **#4 Mention syntax** — `` (§8.1), implemented: + unambiguous and LLM-friendly. Client-side confirmation that the tag + renders well remains welcome and is non-blocking. + +- **#5 Emoji materialization** — lossy by design (§8.1): the mark + disappears, its rendering is preserved. No surface has claimed to need + the mark itself; that confirmation is likewise non-blocking. + +- **#6 Icon block** — mooted by the icon lift: the block round-trips on + the legacy profile objects that carry it (§5's table admits it, `name` + only) and appears nowhere else, so there is no drop decision left to + make. + +- **#7 `type_properties` naming** — settled (§2a): the group is + `type_settings`, the array `property_definitions`, and the `section` + enum won over three booleans — mutual exclusion for free. Not to + re-propose: `definition.properties` (extra nesting) and a `schema` field + (collides with `$schema`, and the section is more than a schema). + +- **#8 Property documents** — settled; §2d and §2f are that section. + `kind: "property"` documents carry the definition group + (`property_settings`), and the dictionary (§2f) is where a bundle + declares a property without a document at all, options resolved by name + with `internal_key` beside them. + +- **#9 The `version` bump** — resolved: the integer moves to **2** at + freeze. Everything written during the draft period is refused by the + version gate, with its dedicated both-versions error (§10, §12), rather + than by a member name. That makes the special-case refusal of + `{"type": "template"}` with no `kind` dead code — it existed only + because that one shape was well-formed under both readings — and it is + deleted along with the last use of the `template` string constant + outside export's emission rule. + +- **#10 `kind` as sole template authority; the `_` namespace** — settled, + with the cheaper alternatives declined (§2, §3; §1, §2c). Deriving the + kind from the type term when `kind` is absent costs no migration but + leaves the type term carrying structural meaning — two authorities, half + the incoherence kept; making `kind` REQUIRED everywhere costs ~16 bytes + on every page and contradicts §4's omit-every-default rule. Bare-word + reserved listings plus a ban on the six words is a word list — every + listing added later retroactively bans an id that was legal. The ban did + NOT go away: the `_` prefix makes the FORMAT unambiguous, and the ban on + the six wire spellings is still what makes the WIRE unambiguous, since + the importer's own spellings are bare. + +- **#11 The §3 chain's store step** — stays exactly as stated: step 3c is + optional, store-backed readers only, single-candidate-or-nothing. + Deletion and promotion were each proposed, and each is falsified by one + call (pinned in `specclaims_test.go`). Deletion: + `bundle.RelationKeysByApiFold("Severity") == []` — the bundled fold + knows nothing about a space's custom keys, so a store-less step 3 would + silently mint a second relation beside the one an agent just read. + Promotion: `bundle.TypeKeysByApiFold("Task") == [task]` — a mandatory + fold would overrule verbatim-first (§3 step 2) on a live stored key this + format itself can create. The asymmetry is the point: a reader may + resolve MORE than another, never DIFFERENTLY (§3). + +- **#12 System-property trim** — settled: a whitelist of seven keys, + spelled out in §3 and `systemtrim.go`. "The §15 #12 test" cited across + this document and the code means the per-key admission discipline: a key + is trimmed only where its empty value is both the proto zero and the + semantic default, verified individually against the corpus. The inverse + rule (`bundle.SystemRelations` minus an exception list) fails open — + every future system relation joins the trim set sight-unseen — and buys + almost nothing: the seven vetted keys carry ~50% of the 1.13% saving, + the thirty-key tail 3.6% of 1.13%, or 0.04% of all bytes. `done` was + never a candidate (not a system relation); the keys that FAILED + admission are in §3. `internalFlags` (24% of the measured saving) is + transient editor state and went to the `transientProperties` strip list + outright, independent of this item. + +- **#14, the spelling half** — taken (§2d), exactly as this entry + prescribed when only one more pre-freeze change fit: the raw + `relation_format: 100` became the envelope's required + `format: "objects"`, `include_time` and `object_types` lifted beside it, + and the flat spellings are refused with the repair named. The disease — + one concept spelled two ways (a raw number on a standalone relation + document, a name in a `type_properties` entry), and one word naming two + concepts — is what "the §15 #14 disease" means wherever this document + cites it (§2a, §2e, §3). The emptiness half is deferred, below. + +- **#15 `picture` stays flat** — deliberate (§3). It has the same relation + format as `iconImage` (`file`, `objectTypes: ["image"]`) and 1,946 + production objects carry one, so folding it into `icon` looks tempting — + but it is a bookmark's preview image, not the object's identity, and + folding it in would make one union mean two things. It reads correctly + as an ordinary `files` property. Written down so it is not re-litigated. + +- **#18 One statement of what a property KEY may be** — fixed. The + writable-key rule is enforced at every key slot with matching schema + bounds, `dataviewProperty` included, with export and the schema moved + together so Marshal cannot emit what its own Validate rejects (§11 I1) — + the coordinated change this entry said a lone `$ref` could not be. The + drift it closes was demonstrated: a 200-character key accepted in a + dataview block's `properties[]` and refused in a definition, in the same + document — the §2e one-shape rule violated invisibly until measured. + +### Deferred past v1 + +- **#14, the emptiness half** — deliberately not taken with the spelling: + `include_time` is still present-and-false on 8,375 documents and + `object_types` present-and-empty on 8,903, now on the envelope, because + presence mirrors the store (§2d); collapsing it is a separate decision + with its own snapshot-comparator cost. `file_variant_*` (7 parallel + arrays on every file object, 8.35% of corpus bytes), `space_invite_*` + and `widget_*` remain deferred with less at stake — machine-written, + never authored. + +- **#16 Reusing a key across spaces** — follow-up, and NOT a format + change. Measured: 39 spellings in a 77-space account already bind to + more than one stored key, `date` to three. The format already has the + answer, and it is the legend: mint the key ONCE, ship it in + `type_internal_keys`/`property_internal_keys`, and get the same key in + every space, deterministically and offline — using a RANDOM key, since a + readable one can collide with an unrelated property a space already has + and merge the two in silence. The tempting alternative — look the + type/property up in the user's OTHER spaces and reuse the key — is + declined for the format (non-deterministic, order-dependent, cross-space + reads on the creation path, a name-heuristic equivalence test that + silently merges exactly what §3's chain and the exhaustive legend rule + keep apart) and left as a possible import feature. If built, it should + be a **suggestion, never a bind**. + +- **#17 `order_id` → `sort_position`** — deferred to v2, recorded here so + the deferral does not freeze in by omission. `order_id` survived the + §2a admission test because it carries the user's own ordering, but what + it carries is a lexid coupled to store internals — 946 documents carry + one (603 `relation_option`, 343 `object_type`), every value exactly four + characters, commonest `VVVV` — which an author cannot compute and a + reader cannot sort on without the whole set. `sort_position: 2` is the + right document spelling, the same move as `relation_format: 100` → + `format: "number"` (§2d), but its own attack pass is unresolved — what + import does when two entries claim one position, and whether export + renumbers densely or preserves gaps — so it does not go in under freeze + pressure. The store keeps its lexid either way. + +- **#19 `layout` and `resolved_layout` follow the featured list into + deprecation** — follow-up. The type owns an instance's layout: the UI no + longer offers a per-object choice, so `layout` records a decision nobody + can make any more, and `resolved_layout` is a cache of + `type_settings.layout`. The corpus agrees from two directions: `layout` + restates `resolved_layout` on 18,515 documents and has never once + disagreed, and both are declared `number` in 76–77 of the 77 + dictionaries while every document writes enum-name strings — the largest + single class of the format-does-not-predict-shape problem, 45,369 slots. + Deferred because `resolved_layout` is load-bearing on the way IN — a + reader with no type document to consult still needs to render — so + retiring it means deciding what an importer does when the type is + absent: a question about the bundle, not one document. + `type_settings.layout` is untouched either way — the declaration, not + the cache. + +- **#20 A bundle that carries files BY REFERENCE** — follow-up, + deliberately not in v1. Today's bundle is FAT: the bytes travel, the + importing account uploads them under keys of its own, and §3 refuses to + carry `fileVariantKeys` and its siblings because a shared bundle + carrying the source's keys would hand its recipient the keys to every + file in that space, for no benefit. The thin bundle — each file named by + cid with the key that opens it, the importing account DOWNLOADS instead + of uploading — is worth having and is not being built now. It needs its + own bundle-level marker, so a reader knows an absent blob is intended + rather than missing; that marker is what makes carrying a key defensible + in that mode and only that mode. The keys are absent because today's + bundle is the FAT kind, not because a key can never appear in this + format. + +- **#21 Option documents vs the dictionary** — follow-up, after the + freeze. A bundle writes 2,641 `kind: "relation_option"` documents the + dictionary nearly restates: since the dictionary learned `internal_key`, + option identity is settled, and the api key need not travel at all — + measured over a 77-space export, all 514 real option api keys are + reproduced by the app's own mint-from-name rule (470 by the api slug, 44 + by the transliterate fallback), so not one survived a rename. The + obstacle is the used-only rule: 175 of the 2,641 options belong to + properties no document references, so dropping the documents today + silently loses those 175. The real question — should a dictionary state + a vocabulary nobody in this bundle uses — changes what the dictionary + MEANS (the properties the bundle exercises vs the space's schema), and + is not being decided under freeze pressure. + +### Open + +- **#13 The icon and cover assumptions the clients own** (§2b). Four, in + descending order of what a wrong guess would cost — each closeable only + with evidence from outside this repository: + + - **Icon precedence is unverified outside heart.** `iconName` > + `iconEmoji` > `iconImage` comes from `core/api/service/icon.go`, the + only precedence implementation in this repository — every other + converter (`dot`, `graphjson`, `publish/relationswhitelist`) emits all + four channels and lets the consumer decide. If the desktop client + renders the emoji over the named icon, the export picks a different + icon than the app shows for the 200 objects that hold both. **One grep + in the client repo settles it** — anyone with a client checkout can + answer — and the answer changes one line of the export rule. + - **`coverType: 4` (prebuilt) has zero instances** in 36,966 objects, + and the prebuilt id vocabulary exists nowhere in this repository. It + is modelled as `{"format": "image", "file": …, "source": "prebuilt"}` + because `state/details.go` and `cmd/usecasevalidator` both treat + `{1,4,5}` as file-backed. What closes it: a client engineer confirming + whether a prebuilt `coverId` is an object id or a client-side asset + *name* — if the latter, the `image` branch is wrong for it and fixing + it costs a version bump. + - **The gradient and cover-colour vocabularies live only in the + clients**, so `cover.color` and `cover.gradient` stay opaque names. A + document can say `{"format": "gradient", "gradient": "sunset"}` and + get a broken cover with no validation error — the one corner where the + typed shape does not do what it exists to do. The format cannot close + this alone; what closes it is the clients publishing the two lists, at + which point the API's discovery layer can serve the enum. + - **`icon.name` is an open string** (§2b) — where this design is weakest + for an offline generator, and open in the sense that only ownership + closes it: the ~397-name enum cannot be frozen into `pkg/lib/*` + without violating I1 the first time the app ships a new icon, so + closing it means someone owning lockstep maintenance with the client + icon set forever. The API's own list currently contains a stray + `t.txt` between `sync` and `tablet-landscape` — what that maintenance + looks like when nobody owns it. diff --git a/pkg/lib/anyblockjson/attribution_test.go b/pkg/lib/anyblockjson/attribution_test.go new file mode 100644 index 0000000000..21b1ca4690 --- /dev/null +++ b/pkg/lib/anyblockjson/attribution_test.go @@ -0,0 +1,455 @@ +package anyblockjson + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The real shape, 135 characters of it, on the account that produced the +// corpus these numbers come from. +const testParticipantId = "_participant_bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq_30afw2fe3tvff_AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA" + +// `creator` and `lastModifiedBy` are DERIVED: recovered from the object +// tree root's own signature on every rebuild, and discarded by every write +// path. A document naming one is telling a reader who wrote the object, not +// setting anything — so import drops the key instead of refusing it, in every +// spelling, and whatever shape the value arrives in. +// +// Before this, the two behaved differently for no reason anyone could state: +// `creator` was accepted (it sat in propertiesKeptOnExport, so the deny rule +// never saw it) and landed a detail that the next rebuild overwrote, while +// `lastModifiedBy` — the same relation definition, one word apart in the +// bundle — was refused outright, making 71 documents in a 36,966-object +// corpus unimportable for a line neither side could act on. +// +// How these can fail: drop the keys from derivedAttributionProperties and +// every case flips to an error (the deny rule reaches them the moment they +// are not exempt, since export strips both); wire the exemption so that it +// only skips the ERROR and not the write, and the NotContains assertions +// fail with the value on details. +func TestAttributionProperties_DroppedNotRefused(t *testing.T) { + for name, tc := range map[string]struct { + doc string + key string + }{ + "creator, as a member name (what export writes)": { + doc: `{"version": 2, "properties": {"creator": "Roman"}}`, + key: "creator", + }, + "creator, as the id array older exports wrote": { + doc: fmt.Sprintf(`{"version": 2, "properties": {"creator": [%q]}}`, testParticipantId), + key: "creator", + }, + "last_modified_by, the spelling that used to be refused": { + doc: `{"version": 2, "properties": {"last_modified_by": "Roman"}}`, + key: "lastModifiedBy", + }, + "last_modified_by, as an id array": { + doc: fmt.Sprintf(`{"version": 2, "properties": {"last_modified_by": [%q]}}`, testParticipantId), + key: "lastModifiedBy", + }, + "the stored spelling drops too": { + doc: `{"version": 2, "properties": {"lastModifiedBy": "Roman"}}`, + key: "lastModifiedBy", + }, + } { + t.Run(name, func(t *testing.T) { + // given the document above + + // when + validateErr := Validate([]byte(tc.doc)) + _, snap, err := Unmarshal([]byte(tc.doc), Options{}) + + // then + require.NoError(t, validateErr, "an attribution line makes a document stale, not wrong") + require.NoError(t, err, "Validate and Unmarshal agree (§11 I2)") + assert.NotContains(t, snap.GetDetails().GetFields(), tc.key, + "and it must not reach the snapshot: the value is derived, not input") + }) + } +} + +// The control that keeps this from spreading. `assignee` points at a +// participant too, and looks identical in a document — but it is +// `source: details, maxCount: 0`, chosen by a person, and the id is the whole +// of its meaning. Dropping it, or spelling it as a name, would be data loss. +// +// How this can fail: add assignee (or author, or any `source: details` +// property) to derivedAttributionProperties and the value stops arriving. +func TestAttributionProperties_UserChosenParticipantsAreUntouched(t *testing.T) { + for _, key := range []string{"assignee", "author", "stakeholders"} { + t.Run(key, func(t *testing.T) { + // given + doc := fmt.Sprintf(`{"version": 2, "properties": {%q: [%q]}}`, key, testParticipantId) + + // when + _, snap, err := Unmarshal([]byte(doc), Options{}) + + // then + require.NoError(t, err) + require.Contains(t, snap.GetDetails().GetFields(), key) + assert.Equal(t, []string{testParticipantId}, + valueStringList(snap.GetDetails().GetFields()[key]), + "a user-chosen participant reference keeps its full id") + }) + } +} + +// The legend resolves a spelling onto a stored key before admission runs +// (§3), so it is the way a document could smuggle a key past a rule keyed on +// the spelling. It must reach the same verdict here: dropped, not landed and +// not refused. +func TestAttributionProperties_LegendCannotLandThem(t *testing.T) { + for name, tc := range map[string]struct{ doc, key string }{ + "creator": {`{"version": 2, "property_internal_keys": {"who": "creator"}, "properties": {"who": "Roman"}}`, "creator"}, + "lastModifiedBy": {`{"version": 2, "property_internal_keys": {"who": "lastModifiedBy"}, "properties": {"who": "Roman"}}`, "lastModifiedBy"}, + } { + t.Run(name, func(t *testing.T) { + // when + validateErr := Validate([]byte(tc.doc)) + _, snap, err := Unmarshal([]byte(tc.doc), Options{}) + + // then + require.NoError(t, validateErr) + require.NoError(t, err) + assert.NotContains(t, snap.GetDetails().GetFields(), tc.key) + assert.NotContains(t, snap.GetDetails().GetFields(), "who") + }) + } +} + +// +// ---- export: the member's id, with the name riding as the suffix ---- +// + +// nameResolver answers with a fixed table, and records what it was asked. +type nameResolver struct { + names map[string]string + asked []string +} + +func (r *nameResolver) ParticipantName(id string) (string, bool) { + r.asked = append(r.asked, id) + name, ok := r.names[id] + return name, ok && name != "" +} + +// The two halves of testParticipantId, for spelling expectations: the space +// the composite embeds and the checksummed identity it folds to (§9). +const ( + testAttribSpaceId = "bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq.30afw2fe3tvff" + testAttribIdentity = "AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA" +) + +// attributionSnapshot is an ordinary object with both attribution details set, +// stored the way real objects store them. +func attributionSnapshot(details map[string]*types.Value) *model.SmartBlockSnapshotBase { + full := map[string]*types.Value{ + "id": str("obj1"), + "name": str("Notes"), + } + for k, v := range details { + full[k] = v + } + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: "obj1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(full), + } +} + +// exportedProperties runs a whole-document export and hands back the +// `properties` object as plain JSON, plus the raw bytes — the public entry +// point, so nothing here can pass by re-implementing the rule. +func exportedProperties(t *testing.T, snap *model.SmartBlockSnapshotBase, opts Options) (map[string]any, string) { + t.Helper() + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (§11 I1)") + var doc struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + return doc.Properties, string(data) +} + +// Attribution is `#` (§3): the folded participant id — +// RESOLVABLE, which the v0.24 name-only spelling was not (API v2 consumers +// need an id to resolve a member, and 76 of 2,478 production participants +// share a display name) — with the member's name as the informative suffix, +// ~57 characters against the 135 the composite was. A plain string, not an +// array: the relation is `maxCount: 1` and 0 of 36,966 corpus values were +// multi-valued. +// +// How these can fail: write the name alone and the id assertions fail; write +// the raw composite and the folded-spelling assertions fail; write the value +// inside a list and the plain-string equality fails; skip the normalizer and +// "Roma Kha" arrives with its space. +func TestAttribution_ExportWritesIdentityAndName(t *testing.T) { + resolver := &nameResolver{names: map[string]string{testParticipantId: "Roma Kha"}} + opts := testOptions() + opts.ResolveParticipants = resolver + opts.SpaceId = testAttribSpaceId + + t.Run("creator is a plain string: folded id, then the name", func(t *testing.T) { + // given + snap := attributionSnapshot(map[string]*types.Value{"creator": strList(testParticipantId)}) + + // when + props, raw := exportedProperties(t, snap, opts) + + // then + assert.Equal(t, testAttribIdentity+"#roma_kha", props["Created by"]) + assert.NotContains(t, raw, testParticipantId, + "the 135-character composite folds; the identity stands in (§9)") + }) + + t.Run("a scalar-stored value reads the same", func(t *testing.T) { + // given real data stores this key both ways + snap := attributionSnapshot(map[string]*types.Value{"creator": str(testParticipantId)}) + + // when + props, _ := exportedProperties(t, snap, opts) + + // then + assert.Equal(t, testAttribIdentity+"#roma_kha", props["Created by"]) + }) + + t.Run("lastModifiedBy is spelled last_modified_by and shaped the same", func(t *testing.T) { + // given + snap := attributionSnapshot(map[string]*types.Value{ + "creator": strList(testParticipantId), + "lastModifiedBy": strList(testParticipantId), + }) + + // when + props, raw := exportedProperties(t, snap, opts) + + // then + assert.Equal(t, testAttribIdentity+"#roma_kha", props["Last modified by"]) + assert.NotContains(t, props, "lastModifiedBy", "the document spells display names (§3)") + assert.NotContains(t, raw, testParticipantId) + }) + + t.Run("without a space id the composite survives whole, still with the name", func(t *testing.T) { + // given the fold is off (§9) — no SpaceId, no fold, either direction + bare := testOptions() + bare.ResolveParticipants = resolver + snap := attributionSnapshot(map[string]*types.Value{"creator": strList(testParticipantId)}) + + // when + props, _ := exportedProperties(t, snap, bare) + + // then + assert.Equal(t, testParticipantId+"#roma_kha", props["Created by"]) + }) +} + +// No resolver, or no name, and the id is written BARE — the id is the +// resolvable half and complete without its caption. Never a dangling `#`, +// and never an omitted property: v0.24's "no name, no property" rule made +// the line unreadable to API consumers precisely when a resolver was +// missing, which is the reversal this change corrects. +// +// How these can fail: keep the old omit-on-no-name rule and every case +// finds the property missing; write "#" with an empty name after it and the +// exact-equality cases fail. +func TestAttribution_BareIdWhenThereIsNoName(t *testing.T) { + for name, opts := range map[string]Options{ + "no participant resolver at all": func() Options { + o := testOptions() + o.SpaceId = testAttribSpaceId + return o + }(), + "a resolver that cannot name this member": func() Options { + o := testOptions() + o.SpaceId = testAttribSpaceId + o.ResolveParticipants = &nameResolver{names: map[string]string{}} + return o + }(), + "a member whose profile name is empty": func() Options { + o := testOptions() + o.SpaceId = testAttribSpaceId + o.ResolveParticipants = &nameResolver{names: map[string]string{testParticipantId: ""}} + return o + }(), + } { + t.Run(name, func(t *testing.T) { + // given + snap := attributionSnapshot(map[string]*types.Value{ + "creator": strList(testParticipantId), + "lastModifiedBy": strList(testParticipantId), + }) + + // when + props, _ := exportedProperties(t, snap, opts) + + // then + assert.Equal(t, testAttribIdentity, props["Created by"], "the bare folded id, nothing else") + assert.Equal(t, testAttribIdentity, props["Last modified by"]) + }) + } + + t.Run("an empty-identity composite is omitted: it addresses nobody", func(t *testing.T) { + // given the real artifact: 9,103 of 37,429 production objects store + // `_participant__` — the composite built from a BLANK + // identity — in lastModifiedBy. 86 characters that resolve to no + // member; the id-shaped analogue of a blank name. + degenerate := "_participant_bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq_30afw2fe3tvff_" + snap := attributionSnapshot(map[string]*types.Value{ + "creator": strList(testParticipantId), + "lastModifiedBy": strList(degenerate), + }) + opts := testOptions() + opts.SpaceId = testAttribSpaceId + + // when + props, raw := exportedProperties(t, snap, opts) + + // then + assert.Equal(t, testAttribIdentity, props["Created by"], "the control: a real id still lands") + assert.NotContains(t, props, "Last modified by") + assert.NotContains(t, raw, degenerate) + }) + + t.Run("an empty stored value is still omitted", func(t *testing.T) { + // given no id at all — a bare id is a complete answer, no id is none + snap := attributionSnapshot(nil) + opts := testOptions() + opts.SpaceId = testAttribSpaceId + + // when + props, _ := exportedProperties(t, snap, opts) + + // then + assert.NotContains(t, props, "Created by") + assert.NotContains(t, props, "Last modified by") + }) +} + +// The control, again, on the export side: `assignee` is `source: details`, +// user-chosen, and its id is the whole of its meaning. It keeps the id (the +// §9 fold applies — the folded spelling IS the id, restated) and the ARRAY +// shape, and it takes no name suffix from the participant resolver: the +// suffix on ordinary references belongs to RefNames + ResolveObjectNames, +// not to the attribution seam. +func TestAttribution_ExportLeavesUserChosenParticipantsAlone(t *testing.T) { + // given + opts := testOptions() + opts.SpaceId = testAttribSpaceId + opts.ResolveParticipants = &nameResolver{names: map[string]string{testParticipantId: "Roma Kha"}} + snap := attributionSnapshot(map[string]*types.Value{ + "creator": strList(testParticipantId), + "assignee": strList(testParticipantId), + }) + + // when + props, _ := exportedProperties(t, snap, opts) + + // then + assert.Equal(t, testAttribIdentity+"#roma_kha", props["Created by"], "a plain string") + assert.Equal(t, []any{testAttribIdentity}, props["Assignee"], + "a user-chosen participant reference keeps its list shape and takes no suffix here") +} + +// The term census reserves what the document SPELLS (§9a). `creator` is on the +// stripped list now, so the census stopped seeing it through the ordinary +// details walk — and a custom relation whose vocabulary spelling collides +// with it would then claim a spelling the attribution line needs, putting +// two properties on one JSON member and losing one. The colliding string +// here is the stored key `creator` itself: verbatim-first reserves it, so +// the custom claimant degrades to its own stored key. +// +// How this can fail: drop the attribution arm of seedTermLedger and the +// custom key takes `creator` while the attribution line spells it too — one +// of the two values gone. +func TestAttribution_CensusReservesTheSpellingItWrites(t *testing.T) { + // given a space that slugs a custom relation onto `creator` + opts := testOptions() + opts.SpaceId = testAttribSpaceId + opts.ResolveParticipants = &nameResolver{names: map[string]string{testParticipantId: "Roma Kha"}} + // the custom key sorts BEFORE `creator`, so it reaches the term ledger + // first and claims the spelling unless the census has reserved it + opts.Keys = slugVocabulary{"aCustomKey": "creator"} + snap := attributionSnapshot(map[string]*types.Value{ + "creator": strList(testParticipantId), + "aCustomKey": str("mine"), + }) + + // when + props, _ := exportedProperties(t, snap, opts) + + // then + assert.Equal(t, testAttribIdentity+"#roma_kha", props["Created by"], + "the attribution key keeps its own spelling") + assert.Equal(t, "mine", props["aCustomKey"], + "the custom key falls back to its stored key, which is always its own address (§3)") +} + +// slugVocabulary spells the stored keys it is given and passes everything else +// through, which is the minimum a KeyVocabulary owes (§3). +type slugVocabulary map[string]string + +func (v slugVocabulary) PropertySlug(key string) string { + if slug, ok := v[key]; ok { + return slug + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (v slugVocabulary) PropertyKey(slug string) (string, bool) { + for key, s := range v { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (v slugVocabulary) TypeSlug(key string) string { return BundledKeyVocabulary{}.TypeSlug(key) } +func (v slugVocabulary) TypeKey(slug string) (string, bool) { + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// The documented cost of the change, pinned so it cannot become a surprise: +// the attribution line does NOT survive a round trip, so an object carrying a +// creator is the one case where Export(S) ≠ Export(Import(Export(S))) for +// something that is not an id (§11). +// +// It is not recoverable and was never data: the value is derived from the tree +// root's signature, and an imported object gets the importing account's own. +// What this pins is that the loss happens ONCE — the second export equals the +// third — so a re-export still diffs cleanly against itself. +func TestAttribution_DoesNotSurviveARoundTrip(t *testing.T) { + // given + opts := testOptions() + opts.SpaceId = testAttribSpaceId + opts.ResolveParticipants = &nameResolver{names: map[string]string{testParticipantId: "Roma Kha"}} + snap := attributionSnapshot(map[string]*types.Value{"creator": strList(testParticipantId)}) + + // when + first, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + _, imported, err := Unmarshal(first, opts) + require.NoError(t, err) + second, err := Marshal(model.SmartBlockType_Page, imported, opts) + require.NoError(t, err) + _, reimported, err := Unmarshal(second, opts) + require.NoError(t, err) + third, err := Marshal(model.SmartBlockType_Page, reimported, opts) + require.NoError(t, err) + + // then + assert.Contains(t, string(first), `"Created by": "`+testAttribIdentity+`#roma_kha"`) + assert.NotContains(t, string(second), `"Created by"`, "import drops it, so the next export has nothing to write") + assert.Equal(t, string(second), string(third), "and everything after the first export is byte-stable") +} diff --git a/pkg/lib/anyblockjson/attributionseam_test.go b/pkg/lib/anyblockjson/attributionseam_test.go new file mode 100644 index 0000000000..cf1e0da182 --- /dev/null +++ b/pkg/lib/anyblockjson/attributionseam_test.go @@ -0,0 +1,90 @@ +package anyblockjson + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// answeringNamer answers every id with one name, including a blank one — the +// shape the exported ParticipantResolver contract permits but the format's own +// rule forbids. The shipped storeresolver never answers blank; a third-party +// implementation may, because the interface only says a resolver that cannot +// answer returns false. +type answeringNamer struct{ name string } + +func (a answeringNamer) ParticipantName(string) (string, bool) { return a.name, true } + +func attributedSnapshot() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), "name": str("Notes"), + "creator": {Kind: &types.Value_ListValue{ListValue: &types.ListValue{ + Values: []*types.Value{str("_participant_a_b_C")}}}}, + }), + ObjectTypes: []string{"ot-page"}, + } +} + +// §3's rule is "a name or nothing after the `#`, never a blank": a dangling +// `#` costs bytes, says less than its absence, and reads to a model as a +// name that exists and is empty. Enforcing it inside the shipped resolver is +// not enough — the seam every resolver passes through has to hold it +// (refNameLabel), or one third-party implementation puts a dangling `#` on +// every object in an export. The id half is unaffected either way: it is +// the resolvable content and is written bare. +// +// This can only fail if the seam stops filtering: it drives the real Marshal +// with a resolver that answers, so a rule enforced only in storeresolver would +// not save it. +func TestExport_AResolverThatAnswersBlankWritesABareId(t *testing.T) { + for name, answer := range map[string]string{ + "empty": "", + "a single space": " ", + "only whitespace": " \t\n ", + } { + t.Run(name, func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, attributedSnapshot(), + Options{ResolveParticipants: answeringNamer{name: answer}}) + require.NoError(t, err) + assert.Contains(t, string(data), `"Created by": "_participant_a_b_C"`, + "the bare id: resolvable, and blank-name-proof") + assert.NotContains(t, string(data), "#", "a blank name is not a name — no dangling separator") + require.NoError(t, Validate(data)) + }) + } + + // the control: a real name still lands as the suffix, so the rule above + // cannot pass by dropping the suffix machinery altogether + t.Run("a real name still lands", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, attributedSnapshot(), + Options{ResolveParticipants: answeringNamer{name: "Roman"}}) + require.NoError(t, err) + assert.Contains(t, string(data), `"Created by": "_participant_a_b_C#roman"`) + }) +} + +// MarshalPropertyValue and UnmarshalPropertyValue are twins: whatever one +// writes, the other reads back. Attribution breaks that on purpose — the +// value is derived from the tree on every rebuild, and no write path could +// honour what a document carries — so the read half must drop it rather +// than hand it back as though it were settable. +func TestFragment_TheValueTwinsDisagreeOnAttribution(t *testing.T) { + for _, key := range []string{"creator", "lastModifiedBy"} { + assert.Nil(t, UnmarshalPropertyValue(key, "_participant_a_b_C#roman", Options{}), + "%s is dropped by whole-document import, so the value door drops it too", key) + } + + // the control: an ordinary property still round-trips through the twins + got := UnmarshalPropertyValue("assignee", "_participant_a_b_C", Options{}) + require.NotNil(t, got, "a user-chosen participant property is untouched") + assert.Equal(t, "_participant_a_b_C", + got.GetListValue().GetValues()[0].GetStringValue(), + "and keeps its full id, resolvable") +} diff --git a/pkg/lib/anyblockjson/authoredname_test.go b/pkg/lib/anyblockjson/authoredname_test.go new file mode 100644 index 0000000000..65e232979b --- /dev/null +++ b/pkg/lib/anyblockjson/authoredname_test.go @@ -0,0 +1,108 @@ +package anyblockjson + +// authoredname_test.go — a property-definition entry that states only a +// NAME is spelled by that name, verbatim. +// +// This was the last derived identifier in a format that has none. The entry +// used to run the api-slug derivation over the name, which transliterates +// and then truncates: "Cooking Time" arrived as `cooking_time` — no longer +// the name a resolver holds — and "☕" arrived as the empty key, which the +// seam then refused. Each of the names below is now its own address. + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTypePropertyNameIsTheSpelling(t *testing.T) { + // every one of these was mangled by the derivation the format no longer + // runs; the third column is what it used to become + names := []struct{ name, wasDerivedAs string }{ + {"Cooking Time", "cooking_time"}, + {"Due Date", "due_date"}, + {"Тоггл", "toggl"}, + {"作業内容", "zuo_ye_nei_rong"}, + {"C++", "c"}, + {"50% done", "50_done"}, + {"Дата выполнения", "data_vypolneniia"}, + {"#", ""}, + {"☕", ""}, + } + + for _, n := range names { + t.Run(n.name, func(t *testing.T) { + // given — an entry that identifies itself by name alone + tp := TypeProperty{Name: n.name} + + // when + term, isInternalKey := tp.authoredKey() + + // then + assert.Equal(t, n.name, term, "the name IS the spelling") + assert.False(t, isInternalKey, "a name derives a spelling, never a stored key") + assert.NotEqual(t, n.wasDerivedAs, term, + "the api-slug derivation is gone; %q used to arrive as %q", n.name, n.wasDerivedAs) + }) + } +} + +// The whole seam, not just the helper: a type document whose definition +// states only a name resolves that name as the property key, and the +// document round-trips through it. +func TestTypePropertyNameOnlyEntryResolvesThroughTheSeam(t *testing.T) { + for _, name := range []string{"Cooking Time", "作業内容", "C++", "50% done"} { + t.Run(name, func(t *testing.T) { + // given + defs, err := json.Marshal([]map[string]any{{"name": name, "format": "text"}}) + require.NoError(t, err) + doc := []byte(fmt.Sprintf(`{"version":2,"kind":"object_type","id":"o1","internal_key":"recipe",`+ + `"type_settings":{"layout":"basic","property_definitions":%s},`+ + `"properties":{"Name":"Recipe"}}`, defs)) + require.NoError(t, Validate(doc), "I1/I2: the document is valid") + + // when + _, snap, err := Unmarshal(doc, Options{GenerateId: seqIds("g")}) + + // then — the name lands in the type's recommended list as the + // key itself, with no resolver to map it to an id + require.NoError(t, err) + var got []string + for _, v := range snap.Details.Fields["recommendedRelations"].GetListValue().GetValues() { + got = append(got, v.GetStringValue()) + } + assert.Equal(t, []string{name}, got, + "the entry's own name is the key the list carries") + }) + } +} + +// A name that cannot BE a spelling derives none, and the seam says so at the +// entry's own pointer rather than truncating it into something else. +func TestTypePropertyUnwritableNameIsRefusedAtItsSlot(t *testing.T) { + long := "" + for i := 0; i <= maxPropertyKeyLen; i++ { + long += "x" + } + for _, tc := range []struct{ name, why string }{ + {"", "a nameless entry identifies nothing"}, + {long, "a name longer than a spelling may be"}, + {"a\nb", "a name carrying a control character"}, + } { + t.Run(tc.why, func(t *testing.T) { + defs, err := json.Marshal([]map[string]any{{"name": tc.name, "format": "text"}}) + require.NoError(t, err) + doc := []byte(fmt.Sprintf(`{"version":2,"kind":"object_type","id":"o1","internal_key":"recipe",`+ + `"type_settings":{"layout":"basic","property_definitions":%s},`+ + `"properties":{"Name":"Recipe"}}`, defs)) + + _, _, err = Unmarshal(doc, Options{GenerateId: seqIds("g")}) + require.Error(t, err, tc.why) + assert.Contains(t, err.Error(), typePropertyDefinitionsPath+"/0/"+memberProperty, + "the refusal points at the entry that carries the name") + }) + } +} diff --git a/pkg/lib/anyblockjson/authoring.go b/pkg/lib/anyblockjson/authoring.go new file mode 100644 index 0000000000..555eebd41e --- /dev/null +++ b/pkg/lib/anyblockjson/authoring.go @@ -0,0 +1,267 @@ +package anyblockjson + +// authoring.go implements the authoring subset (§2g): three schemas under +// schema/authoring/ that narrow the full grammar to exactly what an author — +// increasingly an LLM agent — composes when generating a use case from +// nothing. The subset is the same format at the same version: every document +// valid under an authoring schema is valid under the corresponding full +// schema, an invariant the tests enforce rather than claim. + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "sort" + "strconv" + "sync" + + "github.com/santhosh-tekuri/jsonschema/v6" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" +) + +//go:embed schema/authoring/object.schema.json +var authoringSchemaJSON []byte + +//go:embed schema/authoring/index.schema.json +var authoringIndexSchemaJSON []byte + +//go:embed schema/authoring/properties.schema.json +var authoringPropertiesSchemaJSON []byte + +// AuthoringSchemaJSON returns the embedded authoring object schema (§2g). +// Callers must not mutate the returned slice; discovery surfaces serve it +// verbatim, the way SchemaJSON is served. +func AuthoringSchemaJSON() []byte { return authoringSchemaJSON } + +// AuthoringIndexSchemaJSON returns the embedded authoring index schema (§2g). +func AuthoringIndexSchemaJSON() []byte { return authoringIndexSchemaJSON } + +// AuthoringPropertiesSchemaJSON returns the embedded authoring property +// dictionary schema (§2g). +func AuthoringPropertiesSchemaJSON() []byte { return authoringPropertiesSchemaJSON } + +// The published authoring schema locations (§2g): the full schemas' +// directory plus an authoring/ segment, so the version travels with +// FormatVersion exactly as the full URLs do — and the trailing file names +// stay object|index|properties.schema.json, which is what DocumentKind +// dispatches on, so an authored document declaring one routes to the same +// reader an exported one does. +var ( + AuthoringSchemaURL = schemaBaseURL + strconv.Itoa(FormatVersion) + "/authoring/object.schema.json" + AuthoringIndexSchemaURL = schemaBaseURL + strconv.Itoa(FormatVersion) + "/authoring/index.schema.json" + AuthoringPropertiesSchemaURL = schemaBaseURL + strconv.Itoa(FormatVersion) + "/authoring/properties.schema.json" +) + +// The three authoring schemas are self-contained on purpose — no $ref +// crosses a file — so each compiles from its own bytes alone and an agent +// handed one file has the whole grammar for that surface. +var compileAuthoringSchema = sync.OnceValues(func() (*jsonschema.Schema, error) { + return compileStandalone(AuthoringSchemaURL, authoringSchemaJSON) +}) + +var compileAuthoringIndexSchema = sync.OnceValues(func() (*jsonschema.Schema, error) { + return compileStandalone(AuthoringIndexSchemaURL, authoringIndexSchemaJSON) +}) + +var compileAuthoringPropertiesSchema = sync.OnceValues(func() (*jsonschema.Schema, error) { + return compileStandalone(AuthoringPropertiesSchemaURL, authoringPropertiesSchemaJSON) +}) + +func compileStandalone(url string, schemaBytes []byte) (*jsonschema.Schema, error) { + doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaBytes)) + if err != nil { + return nil, fmt.Errorf("decode embedded schema %s: %w", url, err) + } + c := jsonschema.NewCompiler() + if err := c.AddResource(url, doc); err != nil { + return nil, fmt.Errorf("add schema resource %s: %w", url, err) + } + sch, err := c.Compile(url) + if err != nil { + return nil, fmt.Errorf("compile schema %s: %w", url, err) + } + return sch, nil +} + +// ValidateAuthoring checks an object document against the authoring subset +// (§2g): the FULL validation first — schema and semantic rules, whose +// refusals carry the curated §12 wording — and then the authoring schema, so +// a valid document that reaches outside the subset is told so as a subset +// verdict rather than a format one. A nil return therefore means the +// document is valid AnyBlock JSON, not merely subset-shaped. +func ValidateAuthoring(data []byte) error { + if err := Validate(data); err != nil { + return err + } + // the semantic rules run BEFORE the schema: they are stated on the + // resolved property key and say which key was written and why an author + // does not write it, where the schema's literal list can only say that + // some member matched a `not`. The schema still catches everything the + // semantic rules do not. + if err := authoringSemantics(data); err != nil { + return err + } + return validateAuthoringSubset(data, compileAuthoringSchema, + "valid AnyBlock JSON, but outside the authoring subset — the members below are export's, not an author's") +} + +// ValidateAuthoringIndex is ValidateAuthoring for a bundle index (§2c, §2g). +func ValidateAuthoringIndex(data []byte) error { + if _, err := UnmarshalIndex(data); err != nil { + return err + } + return validateAuthoringSubset(data, compileAuthoringIndexSchema, + "a valid bundle index, but outside the authoring subset") +} + +// ValidateAuthoringPropertyDictionary is ValidateAuthoring for a property +// dictionary (§2f, §2g). +func ValidateAuthoringPropertyDictionary(data []byte) error { + if _, err := UnmarshalPropertyDictionary(data); err != nil { + return err + } + return validateAuthoringSubset(data, compileAuthoringPropertiesSchema, + "a valid property dictionary, but outside the authoring subset") +} + +// validateAuthoringSubset runs one authoring schema over a document the full +// reader has already accepted. The preamble issue names the verdict's +// nature: everything below it is the SUBSET refusing a member or a value, +// never the format — the full validation already passed. +func validateAuthoringSubset(data []byte, compile func() (*jsonschema.Schema, error), preamble string) error { + raw, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + if err != nil { + return &ValidationError{Issues: []Issue{{Message: fmt.Sprintf("invalid JSON: %v", err)}}} + } + sch, err := compile() + if err != nil { + return fmt.Errorf("embedded authoring schema: %w", err) + } + if err := sch.Validate(raw); err != nil { + issues := append([]Issue{{Message: preamble}}, schemaIssues(err, keySlotReport{})...) + return &ValidationError{Issues: issues} + } + return nil +} + +// authoringSemantics is the authoring subset's rules that cannot be written +// as a JSON Schema keyword, and the reason is one thing: **the format +// addresses a property by its display NAME, and resolution is case- and +// separator-insensitive.** "Name", "name" and "NAME" are one key; so are +// "Creation date", "created_date" and "createdDate". JSON Schema's +// `required` and `not/enum` are literal member matches, so a rule written +// there holds for exactly one spelling of a key the codec accepts in many — +// which is not a narrower rule, it is a rule with holes in it. +// +// Two rules used to live in the schema and were losing: +// +// - a type document must name itself. Written as +// `properties: {required: ["name"]}`, it REFUSED the canonical +// `{"Name": "Habit"}` and accepted only the retired lowercase spelling. +// Swapping the literal would have moved the hole, not closed it. +// - the subset refuses the app's own derived keys in `properties`. The +// schema's literal list still bans the pre-raw-name spellings, so the +// nine keys the FULL format does not also refuse — attribution, +// timestamps, revision, internal flags, featured properties, archived — +// lost their authoring refusal the moment their canonical spelling +// became a display name. They are dropped at import instead, silently. +// +// Both are enforced here on the RESOLVED key, through the same chain +// Validate's own admission loop uses, so every spelling of a key gets one +// verdict. The schema keeps its literal list as documentation and as a fast +// front door — an agent reads the schema, not this file — and +// TestAuthoringDeniedKeysMatchTheSchema pins the two together so the list +// cannot rot again. +func authoringSemantics(data []byte) error { + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + return &ValidationError{Issues: []Issue{{Message: fmt.Sprintf("invalid JSON: %v", err)}}} + } + var issues []Issue + add := func(path, format string, args ...any) { + issues = append(issues, Issue{Path: path, Message: fmt.Sprintf(format, args...)}) + } + + props, _ := doc["properties"].(map[string]any) + legend, _ := doc[memberPropertyInternalKeys].(map[string]any) + resolve := func(term string) string { + if v, ok := legend[term]; ok { + if key, isStr := v.(string); isStr && key != "" { + return key + } + } + key, _ := BundledKeyVocabulary{}.PropertyKey(term) + return key + } + + namesItself := false + terms := make([]string, 0, len(props)) + for term := range props { + terms = append(terms, term) + } + sort.Strings(terms) + for _, term := range terms { + key := resolve(term) + path := "/properties/" + escapeJSONPointer(term) + if key == bundle.RelationKeyName.String() { + namesItself = true + } + if reason, denied := authoringDeniedPropertyKeys[key]; denied { + add(path, "%q is %s — the app derives it, so an author does not write it. "+ + "Every spelling of that key is refused here, not just this one", key, reason) + continue + } + // the subset narrows a name-over-number key to the NAME. The full + // format also accepts the stored number, because export writes what + // a space stores; an author has no number to carry over and a bare + // integer is unreadable, so the subset takes the name only. Keyed + // off the resolved key for the reason everything here is: the + // canonical spelling of `layoutAlign` is "Layout align", and a rule + // written against the member name `layout_align` stopped covering it. + if vocab, named := namedEnumProperty(key); named { + if _, isStr := props[term].(string); !isStr { + add(path, "%q takes a %s NAME here, one of %v — the raw stored number is "+ + "the full format's pass-through, which the subset removes", key, vocab.what, vocab.names()) + } + } + } + + // a type document must name itself: the type's display name is the one + // thing nothing else in the document can supply, and a nameless type + // arrives in a space as an untitled row + if isTypeKind(doc) && !namesItself { + add("/properties", "a type document states its own display name here — "+ + "any spelling that resolves to the name property (\"Name\") will do") + } + + if len(issues) == 0 { + return nil + } + preamble := Issue{Message: "valid AnyBlock JSON, but outside the authoring subset — " + + "the rules below are stated on the RESOLVED property key, so they hold for every spelling of it"} + return &ValidationError{Issues: append([]Issue{preamble}, issues...)} +} + +// authoringDeniedPropertyKeys are the stored keys an author never writes in +// `properties` and the FULL format does not already refuse. Everything else +// the authoring schema's literal list names is refused by Validate before +// this pass runs — the deny rule (import refuses exactly what export +// strips) covers ids, icons, covers, spaceId, uniqueKey, snippet, +// backlinks, links, mentions, origin, importType, restrictions and the +// rest. These nine are the remainder: import DROPS them rather than +// refusing them, so without a rule here an author's value disappears +// without a word. +var authoringDeniedPropertyKeys = map[string]string{ + "creator": "attribution the app stamps from the acting identity", + "lastModifiedBy": "attribution the app stamps from the acting identity", + "createdDate": "a timestamp the app stamps", + "lastModifiedDate": "a timestamp the app stamps", + "addedDate": "a timestamp the app stamps", + "revision": "the bundled revision the app records", + "internalFlags": "the app's own creation-flow state", + "featuredRelations": "a per-object featured list no UI sets: the layout syncer owns it, " + + "and a type's featured properties belong in that type's recommended lists", + "isArchived": "the app's bin membership, moved by archiving an object rather than by writing a property", +} diff --git a/pkg/lib/anyblockjson/authoring_test.go b/pkg/lib/anyblockjson/authoring_test.go new file mode 100644 index 0000000000..a24d5ddd53 --- /dev/null +++ b/pkg/lib/anyblockjson/authoring_test.go @@ -0,0 +1,783 @@ +package anyblockjson + +// authoring_test.go pins the authoring subset (§2g). The load-bearing claim +// is in the name: SUBSET. Every document the authoring schemas accept must be +// accepted by the full schemas and the full reader — that is what keeps the +// small surface honest, and it is a TEST here, not a claim: the worked +// example, a structural fixture battery, and a sweep that builds one document +// per enum value the authoring schemas state, each pushed through the full +// Validate and the real codec. + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// authoringOnly runs ONLY the authoring schema — not the full validation the +// public ValidateAuthoring folds in — so the subset invariant can be stated +// as two independent verdicts: the subset accepts, and the full side must +// then accept too. +func authoringOnly(data []byte) error { + return validateAuthoringSubset(data, compileAuthoringSchema, "outside the authoring subset") +} + +func authoringIndexOnly(data []byte) error { + return validateAuthoringSubset(data, compileAuthoringIndexSchema, "outside the authoring subset") +} + +func authoringPropertiesOnly(data []byte) error { + return validateAuthoringSubset(data, compileAuthoringPropertiesSchema, "outside the authoring subset") +} + +// requireSubsetObject is the invariant, per document: in the subset, then +// full-valid, then importable by the real codec. +func requireSubsetObject(t *testing.T, doc string) { + t.Helper() + data := []byte(doc) + require.NoError(t, authoringOnly(data), "the fixture must be inside the authoring subset:\n%s", doc) + require.NoError(t, Validate(data), "subset invariant: an authoring-valid document must be full-valid:\n%s", doc) + _, _, err := Unmarshal(data, Options{}) + require.NoError(t, err, "and the real codec must import it:\n%s", doc) +} + +// --- the worked example ------------------------------------------------- + +// The habit_tracker bundle is the minimal worked example §2g points authors +// at: an index, one type, a property dictionary, a welcome page and two +// objects. It must stay valid against the authoring schemas AND the real +// codec, warning-free, and internally coherent — every id a file names is an +// id a file declares. +// +// How this can fail: a schema change that outgrows the example, or an edit +// to the example that reaches outside the subset. Either way the example is +// the first document an authoring agent imitates, so it rots loudest. +func TestAuthoringExample_HabitTracker(t *testing.T) { + root := filepath.Join("testdata", "authoring", "habit_tracker") + + readFile := func(rel string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, rel)) + require.NoError(t, err) + return data + } + + objectFiles := []string{ + filepath.Join("types", "habit.json"), + filepath.Join("objects", "start.json"), + filepath.Join("objects", "morning-run.json"), + filepath.Join("objects", "weekly-review.json"), + } + + declaredIds := map[string]bool{} + declaredTypeKeys := map[string]bool{} + referencedIds := map[string]string{} // id -> where it was referenced + objectTypeTerms := map[string]string{} + + deepLink := regexp.MustCompile(`objectId=([A-Za-z0-9_-]+)`) + + for _, rel := range objectFiles { + t.Run(rel, func(t *testing.T) { + data := readFile(rel) + + // the subset, stated as its own verdict, then the invariant + require.NoError(t, authoringOnly(data)) + var warnings []Issue + require.NoError(t, ValidateWarn(data, func(i Issue) { warnings = append(warnings, i) })) + assert.Empty(t, warnings, "the worked example must be warning-free") + require.NoError(t, ValidateAuthoring(data)) + + // the real codec + _, snapshot, err := Unmarshal(data, Options{}) + require.NoError(t, err) + require.NotNil(t, snapshot) + + var doc struct { + Kind string `json:"kind"` + Id string `json:"id"` + Type string `json:"type"` + InternalKey string `json:"internal_key"` + Blocks []struct { + Type string `json:"type"` + ObjectId string `json:"object_id"` + Text string `json:"text"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + require.NotEmpty(t, doc.Id, "every example document declares its bundle-local id") + declaredIds[doc.Id] = true + if doc.Kind == "object_type" { + declaredTypeKeys[doc.InternalKey] = true + } else { + objectTypeTerms[doc.Type] = rel + } + for _, b := range doc.Blocks { + if b.Type == "link" { + referencedIds[b.ObjectId] = rel + " (link block)" + } + for _, m := range deepLink.FindAllStringSubmatch(b.Text, -1) { + referencedIds[m[1]] = rel + " (inline object link)" + } + } + }) + } + + t.Run("index.json", func(t *testing.T) { + data := readFile("index.json") + require.NoError(t, authoringIndexOnly(data)) + require.NoError(t, ValidateAuthoringIndex(data)) + idx, err := UnmarshalIndex(data) + require.NoError(t, err) + require.NotEmpty(t, idx.EntryPoint()) + referencedIds[idx.EntryPoint()] = "index.json (entrypoint)" + for _, w := range idx.Widgets { + if !strings.HasPrefix(w.Target, "_") { + referencedIds[w.Target] = "index.json (widget)" + } + } + + // the example exercises the sidebar the format can actually express + // (§2c): a view widget with a limit, a card-styled link, a reserved + // listing — and the archive builder must take all of it, since the + // sidebar an author writes here IS the widget snapshot the importer + // installs + require.Len(t, idx.Widgets, 4) + assert.Equal(t, "view", idx.Widgets[1].Layout) + assert.Equal(t, "card", idx.Widgets[2].CardStyle) + assert.True(t, IsReservedWidgetTarget(idx.Widgets[3].Target)) + snap, err := WidgetsSnapshot(idx) + require.NoError(t, err) + require.NotNil(t, snap) + assert.Len(t, snap.Blocks, 1+2*len(idx.Widgets)) + }) + + t.Run("properties.json", func(t *testing.T) { + data := readFile("properties.json") + require.NoError(t, authoringPropertiesOnly(data)) + require.NoError(t, ValidateAuthoringPropertyDictionary(data)) + var warnings []Issue + dict, err := UnmarshalPropertyDictionaryWarn(data, func(i Issue) { warnings = append(warnings, i) }) + require.NoError(t, err) + assert.Empty(t, warnings, "the worked example must be warning-free") + require.NotEmpty(t, dict.Properties) + }) + + t.Run("the bundle is coherent", func(t *testing.T) { + for id, where := range referencedIds { + assert.True(t, declaredIds[id], + "%s names %q, but no document in the bundle declares that id", where, id) + } + for term, rel := range objectTypeTerms { + // a BUILT-IN type is not declared by the bundle — it ships with + // every reader. The welcome page names one by its canonical + // spelling, "Page"; the stored key `page` resolves to the same + // type through the same ladder, so the skip asks the ladder + // rather than naming one spelling of one type. + if _, bundled := (BundledKeyVocabulary{}).TypeKey(term); bundled { + continue + } + assert.True(t, declaredTypeKeys[term], + "%s has type %q, which no type document in the bundle declares", rel, term) + } + }) +} + +// --- structural fixtures ------------------------------------------------ + +// One fixture per structure the authoring grammar can express, each held to +// the subset invariant. The battery is what catches a subset break that is +// not an enum value: a member combination the authoring schema admits and +// the full schema's per-type closing refuses. +func TestAuthoringSubset_StructuralFixtures(t *testing.T) { + fixtures := map[string]string{ + "minimal document": `{"version": 2}`, + "a full page envelope": `{"version": 2, "id": "page-a", "type": "page", + "icon": {"format": "emoji", "emoji": "🌱"}, + "cover": {"format": "gradient", "gradient": "pinkOrange"}, + "properties": {"name": "A", "description": "a page", "is_favorite": true, + "done": false, "custom_note": null, "tags_of_mine": ["x", "y"]}}`, + "nested blocks": `{"version": 2, "blocks": [ + {"type": "heading_1", "text": "H"}, + {"type": "toggle", "text": "open me"}, + {"indent": 1, "type": "bulleted_list_item", "text": "one"}, + {"indent": 2, "type": "paragraph", "text": "deeper"}, + {"indent": 1, "type": "numbered_list_item", "text": "two"}, + {"type": "quote", "text": "said"}, + {"type": "code", "language": "go", "text": "fmt.Println(1)"}]}`, + "columns": `{"version": 2, "blocks": [ + {"type": "row"}, + {"indent": 1, "type": "column"}, + {"indent": 2, "type": "paragraph", "text": "left"}, + {"indent": 1, "type": "column"}, + {"indent": 2, "type": "paragraph", "text": "right"}]}`, + "a table with empty and padded cells": `{"version": 2, "blocks": [ + {"type": "table", + "columns": [{}, {}, {}], + "rows": [ + {"is_header": true, "cells": ["Name", "Status", "Note"]}, + {"cells": ["Export", null, "spec"]}, + {"cells": ["Short row"]}]}]}`, + "an inline set on a page": `{"version": 2, "id": "page-b", "blocks": [ + {"type": "dataview", "object_id": "coll-shelf", "is_collection": true, + "properties": [{"property": "name", "format": "text"}], + "views": [{"name": "Shelf"}]}]}`, + "a collection": `{"version": 2, "id": "coll-shelf", "type": "collection", + "items": ["page-a", "page-b"], + "blocks": [{"type": "dataview", "is_collection": true, + "views": [{"type": "list", "name": "All"}]}]}`, + "a template": `{"version": 2, "kind": "template", "id": "tpl-habit", + "type": "template", "template_for": "habit", + "properties": {"name": "New habit"}, + "blocks": [{"type": "paragraph", "text": "Why this habit matters:"}]}`, + "a type with the whole settings surface": `{"version": 2, "kind": "object_type", + "id": "type-r", "internal_key": "review", + "icon": {"format": "icon", "name": "book", "color": "teal"}, + "properties": {"name": "Review", "description": "One review."}, + "type_settings": { + "layout": "todo", + "plural_name": "Reviews", + "default_template": "tpl-habit", + "default_view": "kanban", + "property_definitions": [ + {"property": "verdict", "name": "Verdict", "format": "select", + "options": ["Ship", {"name": "Hold", "color": "red"}], "section": "featured"}, + {"name": "Reviewed on", "format": "date", "include_time": true}, + {"property": "owner", "name": "Owner", "format": "objects", + "object_types": ["participant"], "section": "hidden"}]}}`, + "filters, groups and sorts": `{"version": 2, "blocks": [ + {"type": "dataview", + "properties": [{"property": "stage", "format": "select"}, {"property": "when", "format": "date"}], + "views": [{ + "name": "Overdue", + "filters": [ + {"operator": "and", "filters": [ + {"property": "when", "condition": "not_empty"}, + {"property": "when", "condition": "less", "date_preset": "today"}]}, + {"operator": "or", "filters": [ + {"property": "stage", "condition": "in", "value": ["Open"]}, + {"property": "stage", "condition": "empty"}]}], + "sorts": [{"property": "when", "direction": "asc", "empty_placement": "end"}], + "columns": [ + {"property": "name"}, + {"property": "when"}, + {"property": "stage", "hidden": true}, + {"property": "name", "aggregation": "count"}]}]}]}`, + "inline markup": `{"version": 2, "blocks": [ + {"type": "paragraph", "text": "Ship the **new export** by Q3 — see [the plan](https://example.com/plan), or ask in [the space](anytype://object?objectId=page-a). A literal \\*star\\*."}, + {"type": "callout", "icon": {"format": "emoji", "emoji": "💡"}, "text": "Escaping: \\ stays prose."}, + {"type": "checkbox", "checked": true, "text": "done ~~and dusted~~"}]}`, + "embeds and bookmarks": `{"version": 2, "blocks": [ + {"type": "embed", "processor": "mermaid", "text": "graph TD; A-->B"}, + {"type": "embed", "processor": "latex", "text": "e^{i\\pi}+1=0"}, + {"type": "bookmark", "url": "https://example.com"}, + {"type": "divider", "style": "dots"}, + {"type": "table_of_contents"}]}`, + } + + for name, doc := range fixtures { + t.Run(name, func(t *testing.T) { + requireSubsetObject(t, doc) + }) + } +} + +// --- the enum sweep ----------------------------------------------------- + +// schemaAt walks a decoded schema by member names, failing loudly when the +// path is stale — a reshuffled authoring schema must break the sweep, not +// silently shrink it. +func schemaAt(t *testing.T, node any, path ...string) any { + t.Helper() + for _, step := range path { + m, ok := node.(map[string]any) + require.True(t, ok, "schema path %v: not an object at %q", path, step) + node, ok = m[step] + require.True(t, ok, "schema path %v: no member %q", path, step) + } + return node +} + +func schemaEnum(t *testing.T, node any, path ...string) []string { + t.Helper() + raw, ok := schemaAt(t, node, path...).([]any) + require.True(t, ok, "schema path %v: not an enum", path) + out := make([]string, 0, len(raw)) + for _, v := range raw { + s, ok := v.(string) + require.True(t, ok, "schema path %v: non-string enum value %v", path, v) + out = append(out, s) + } + require.NotEmpty(t, out) + return out +} + +// blockConditional finds the authoring block's if/then branch that declares +// the given member, so the sweep does not depend on the allOf's order. +func blockConditional(t *testing.T, schema map[string]any, member string) map[string]any { + t.Helper() + conds, ok := schemaAt(t, schema, "$defs", "block", "allOf").([]any) + require.True(t, ok) + for _, c := range conds { + then, ok := c.(map[string]any)["then"].(map[string]any) + if !ok { + continue + } + props, ok := then["properties"].(map[string]any) + if !ok { + continue + } + if _, ok := props[member]; ok { + return props + } + } + require.Failf(t, "no block conditional", "no authoring block branch declares %q", member) + return nil +} + +// filterBranch finds the filterNode branch that declares the given member — +// "condition" lands on the leaf, "operator" on the group. +func filterBranch(t *testing.T, schema map[string]any, member string) map[string]any { + t.Helper() + branches, ok := schemaAt(t, schema, "$defs", "filterNode", "oneOf").([]any) + require.True(t, ok) + for _, b := range branches { + props, ok := b.(map[string]any)["properties"].(map[string]any) + if !ok { + continue + } + if _, ok := props[member]; ok { + return props + } + } + require.Failf(t, "no filter branch", "no filterNode branch declares %q", member) + return nil +} + +// Every enum value the authoring OBJECT schema states is exercised in a +// document and held to the subset invariant. This is the typo trap: an +// authoring enum value the full schema does not know produces documents the +// subset accepts and the format refuses, which is exactly the break the +// invariant forbids — and a hand-kept list of values would rot, so the sweep +// reads the schema itself. +func TestAuthoringSubset_EveryObjectEnumValueIsFullValid(t *testing.T) { + var schema map[string]any + require.NoError(t, json.Unmarshal(authoringSchemaJSON, &schema)) + + typeDoc := func(settings string) string { + return `{"version": 2, "kind": "object_type", "internal_key": "t1", + "properties": {"name": "T"}, "type_settings": {` + settings + `}}` + } + dataviewDoc := func(view string) string { + return `{"version": 2, "blocks": [{"type": "dataview", "views": [{` + view + `}]}]}` + } + + sweep := func(name string, values []string, build func(v string) string) { + t.Run(name, func(t *testing.T) { + for _, v := range values { + requireSubsetObject(t, build(v)) + } + }) + } + + sweep("kind", schemaEnum(t, schema, "properties", "kind", "enum"), func(v string) string { + switch v { + case "object_type": + return typeDoc(`"layout": "basic"`) + case "template": + return `{"version": 2, "kind": "template", "type": "template", "template_for": "t1"}` + default: + return `{"version": 2, "kind": "` + v + `"}` + } + }) + + sweep("block type", schemaEnum(t, schema, "$defs", "block", "properties", "type", "enum"), func(v string) string { + blocks := map[string]string{ + "checkbox": `{"type": "checkbox", "checked": true, "text": "x"}`, + "callout": `{"type": "callout", "icon": {"format": "emoji", "emoji": "💡"}, "text": "x"}`, + "code": `{"type": "code", "language": "go", "text": "x"}`, + "bookmark": `{"type": "bookmark", "url": "https://example.com"}`, + "link": `{"type": "link", "object_id": "page-two", "card_style": "inline"}`, + "divider": `{"type": "divider"}`, + "row": `{"type": "row"}, {"indent": 1, "type": "column"}, {"indent": 2, "type": "paragraph", "text": "x"}`, + "column": `{"type": "row"}, {"indent": 1, "type": "column"}, {"indent": 2, "type": "paragraph", "text": "x"}`, + "table": `{"type": "table", "columns": [{}, {}], "rows": [{"cells": ["a", null]}]}`, + "embed": `{"type": "embed", "processor": "mermaid", "text": "graph TD; A-->B"}`, + "table_of_contents": `{"type": "table_of_contents"}`, + "dataview": `{"type": "dataview", "views": [{"name": "v"}]}`, + } + b, ok := blocks[v] + if !ok { + b = `{"type": "` + v + `", "text": "x"}` + } + return `{"version": 2, "blocks": [` + b + `]}` + }) + + sweep("type_settings.layout", schemaEnum(t, schema, "properties", "type_settings", "properties", "layout", "enum"), func(v string) string { + return typeDoc(`"layout": "` + v + `"`) + }) + sweep("type_settings.default_view", schemaEnum(t, schema, "properties", "type_settings", "properties", "default_view", "enum"), func(v string) string { + return typeDoc(`"default_view": "` + v + `"`) + }) + sweep("property format", schemaEnum(t, schema, "$defs", "propertyFormat", "enum"), func(v string) string { + return typeDoc(`"property_definitions": [{"property": "p1", "name": "P", "format": "` + v + `"}]`) + }) + sweep("section", schemaEnum(t, schema, "$defs", "propertyDefinition", "properties", "section", "enum"), func(v string) string { + return typeDoc(`"property_definitions": [{"property": "p1", "section": "` + v + `"}]`) + }) + sweep("layout_align", schemaEnum(t, schema, "$defs", "blockAlign", "enum"), func(v string) string { + return `{"version": 2, "properties": {"layout_align": "` + v + `"}}` + }) + sweep("palette colour on icons", schemaEnum(t, schema, "$defs", "paletteColor", "enum"), func(v string) string { + return `{"version": 2, "icon": {"format": "icon", "name": "book", "color": "` + v + `"}}` + }) + sweep("palette colour on options", schemaEnum(t, schema, "$defs", "paletteColor", "enum"), func(v string) string { + return typeDoc(`"property_definitions": [{"property": "p1", "format": "select", + "options": [{"name": "O", "color": "` + v + `"}]}]`) + }) + sweep("icon format", schemaEnum(t, schema, "$defs", "icon", "properties", "format", "enum"), func(v string) string { + icons := map[string]string{ + "emoji": `{"format": "emoji", "emoji": "🌱"}`, + "icon": `{"format": "icon", "name": "book"}`, + "color": `{"format": "color", "color": "teal"}`, + } + icon, ok := icons[v] + require.True(t, ok, "no builder for icon format %q", v) + return `{"version": 2, "icon": ` + icon + `}` + }) + sweep("cover format", schemaEnum(t, schema, "$defs", "cover", "properties", "format", "enum"), func(v string) string { + covers := map[string]string{ + "color": `{"format": "color", "color": "black"}`, + "gradient": `{"format": "gradient", "gradient": "sky"}`, + } + cover, ok := covers[v] + require.True(t, ok, "no builder for cover format %q", v) + return `{"version": 2, "cover": ` + cover + `}` + }) + + embed := blockConditional(t, schema, "processor") + sweep("embed processor", schemaEnum(t, embed, "processor", "enum"), func(v string) string { + return `{"version": 2, "blocks": [{"type": "embed", "processor": "` + v + `", "text": "x"}]}` + }) + link := blockConditional(t, schema, "card_style") + sweep("link card_style", schemaEnum(t, link, "card_style", "enum"), func(v string) string { + return `{"version": 2, "blocks": [{"type": "link", "object_id": "page-two", "card_style": "` + v + `"}]}` + }) + divider := blockConditional(t, schema, "style") + sweep("divider style", schemaEnum(t, divider, "style", "enum"), func(v string) string { + return `{"version": 2, "blocks": [{"type": "divider", "style": "` + v + `"}]}` + }) + + sweep("view type", schemaEnum(t, schema, "$defs", "view", "properties", "type", "enum"), func(v string) string { + return dataviewDoc(`"type": "` + v + `", "name": "v"`) + }) + sweep("view card_size", schemaEnum(t, schema, "$defs", "view", "properties", "card_size", "enum"), func(v string) string { + return dataviewDoc(`"type": "gallery", "card_size": "` + v + `"`) + }) + sweep("sort direction", schemaEnum(t, schema, "$defs", "sort", "properties", "direction", "enum"), func(v string) string { + return dataviewDoc(`"sorts": [{"property": "p1", "direction": "` + v + `"}]`) + }) + sweep("sort empty_placement", schemaEnum(t, schema, "$defs", "sort", "properties", "empty_placement", "enum"), func(v string) string { + return dataviewDoc(`"sorts": [{"property": "p1", "empty_placement": "` + v + `"}]`) + }) + sweep("column aggregation", schemaEnum(t, schema, "$defs", "viewColumn", "properties", "aggregation", "enum"), func(v string) string { + return dataviewDoc(`"columns": [{"property": "p1", "aggregation": "` + v + `"}]`) + }) + + leaf := filterBranch(t, schema, "condition") + sweep("filter condition", schemaEnum(t, leaf, "condition", "enum"), func(v string) string { + value := `, "value": ["x"]` + switch v { + case "empty", "not_empty": + value = "" + case "greater", "less", "greater_or_equal", "less_or_equal": + value = `, "value": 5` + } + return dataviewDoc(`"filters": [{"property": "p1", "condition": "` + v + `"` + value + `}]`) + }) + sweep("filter date_preset", schemaEnum(t, leaf, "date_preset", "enum"), func(v string) string { + return dataviewDoc(`"filters": [{"property": "p1", "condition": "greater_or_equal", "date_preset": "` + v + `"}]`) + }) + group := filterBranch(t, schema, "operator") + sweep("filter group operator", schemaEnum(t, group, "operator", "enum"), func(v string) string { + return dataviewDoc(`"filters": [{"operator": "` + v + `", "filters": [{"property": "p1", "condition": "not_empty"}]}]`) + }) +} + +// The index and dictionary sweeps, same invariant against their own full +// readers. +func TestAuthoringSubset_IndexAndDictionaryEnumValues(t *testing.T) { + requireSubsetIndex := func(t *testing.T, doc string) { + t.Helper() + data := []byte(doc) + require.NoError(t, authoringIndexOnly(data), "must be inside the subset:\n%s", doc) + _, err := UnmarshalIndex(data) + require.NoError(t, err, "subset invariant: the full index reader must accept:\n%s", doc) + } + requireSubsetDictionary := func(t *testing.T, doc string) { + t.Helper() + data := []byte(doc) + require.NoError(t, authoringPropertiesOnly(data), "must be inside the subset:\n%s", doc) + _, err := UnmarshalPropertyDictionary(data) + require.NoError(t, err, "subset invariant: the full dictionary reader must accept:\n%s", doc) + } + + var indexSchema map[string]any + require.NoError(t, json.Unmarshal(authoringIndexSchemaJSON, &indexSchema)) + + t.Run("widget layout", func(t *testing.T) { + for _, v := range schemaEnum(t, indexSchema, "$defs", "widget", "properties", "layout", "enum") { + requireSubsetIndex(t, `{"version": 2, "name": "X", "entrypoint": "page-a", + "widgets": [{"target": "page-a", "layout": "`+v+`", "limit": 6}]}`) + } + }) + t.Run("reserved widget targets", func(t *testing.T) { + branches, ok := schemaAt(t, indexSchema, "$defs", "widget", "properties", "target", "anyOf").([]any) + require.True(t, ok) + var reserved []string + for _, b := range branches { + if e, ok := b.(map[string]any)["enum"]; ok { + for _, v := range e.([]any) { + reserved = append(reserved, v.(string)) + } + } + } + require.NotEmpty(t, reserved, "the target anyOf must state the reserved listings") + for _, v := range reserved { + requireSubsetIndex(t, `{"version": 2, "name": "X", "entrypoint": "page-a", + "widgets": [{"target": "`+v+`"}]}`) + } + }) + + var propsSchema map[string]any + require.NoError(t, json.Unmarshal(authoringPropertiesSchemaJSON, &propsSchema)) + t.Run("dictionary format", func(t *testing.T) { + for _, v := range schemaEnum(t, propsSchema, "$defs", "property", "properties", "format", "enum") { + requireSubsetDictionary(t, `{"version": 2, "properties": [{"property": "p1", "format": "`+v+`"}]}`) + } + }) + t.Run("installed and a name-identified entry", func(t *testing.T) { + requireSubsetDictionary(t, `{"version": 2, "installed": ["due_date", "tag"], + "properties": [{"name": "Cooking Time", "format": "number"}, + {"property": "owner", "format": "objects", "object_types": ["participant"], + "description": "who runs it"}, + {"property": "when", "format": "date", "include_time": true}]}`) + }) +} + +// --- what the subset refuses -------------------------------------------- + +// Each document here is VALID under the full schema and reader — that is +// asserted, so this list cannot drift into restating format errors — and +// refused by the authoring subset, because the member it carries is one only +// a live space can write honestly: ids, legends, provenance, output-only +// state, non-authorable kinds and variants. +func TestAuthoringSubset_RefusesBackupOnlySurfaces(t *testing.T) { + cases := map[string]string{ + "a block id": `{"version": 2, "blocks": [{"id": "b1", "type": "paragraph", "text": "x"}]}`, + "the store escape hatch": `{"version": 2, "store": {"k": 1}}`, + "the root escape hatch": `{"version": 2, "root": {"background_color": "grey"}}`, + "the property legend": `{"version": 2, "property_internal_keys": {"prio": "6a32d4856761631534b22f85"}}`, + "the type legend": `{"version": 2, "type_internal_keys": {"task": "task"}}`, + "the option legend": `{"version": 2, "properties": {"prio": ["High"]}, "option_ids": {"prio": {"High": "bafyreiopt1"}}}`, + "attribution in properties": `{"version": 2, "properties": {"creator": "A6eK73Jm#roma"}}`, + "a non-authorable kind": `{"version": 2, "kind": "participant"}`, + "an icon by file": `{"version": 2, "icon": {"format": "file", "file": "bafyreicfd"}}`, + "an image cover": `{"version": 2, "cover": {"format": "image", "file": "bafyreigejp", "y": -0.25}}`, + "internal_key on a page": `{"version": 2, "internal_key": "x"}`, + "a counting date preset": `{"version": 2, "blocks": [{"type": "dataview", "views": [{"filters": [{"property": "p", "condition": "less", "date_preset": "number_of_days_ago", "value": 7}]}]}]}`, + "a view id": `{"version": 2, "blocks": [{"type": "dataview", "views": [{"id": "v1", "name": "v"}]}]}`, + "block alignment": `{"version": 2, "blocks": [{"type": "paragraph", "text": "x", "align": "center"}]}`, + "the heading_4 input alias": `{"version": 2, "blocks": [{"type": "heading_4", "text": "x"}]}`, + "the equation input alias": `{"version": 2, "blocks": [{"type": "equation", "text": "E=mc^2"}]}`, + "a widget block": `{"version": 2, "blocks": [{"type": "widget", "layout": "tree"}]}`, + "the legacy group container": `{"version": 2, "blocks": [{"type": "group"}]}`, + "a file block": `{"version": 2, "blocks": [{"type": "image", "object_id": "bafyimg"}]}`, + "a custom sort order": `{"version": 2, "blocks": [{"type": "dataview", "views": [{"sorts": [{"property": "p", "direction": "custom", "custom_order": ["b", "a"]}]}]}]}`, + "dataview output-only state": `{"version": 2, "blocks": [{"type": "dataview", "source": ["bafysrc"], "views": [{"name": "v"}]}]}`, + "include_time off a date property": `{"version": 2, "kind": "object_type", "internal_key": "t1", + "properties": {"name": "T"}, + "type_settings": {"property_definitions": [{"property": "p1", "format": "text", "include_time": true}]}}`, + } + for name, doc := range cases { + t.Run(name, func(t *testing.T) { + data := []byte(doc) + require.NoError(t, Validate(data), + "the case must be FULL-valid, or it is a format error rather than a subset boundary:\n%s", doc) + err := ValidateAuthoring(data) + require.Error(t, err, "the authoring subset must refuse:\n%s", doc) + assert.Contains(t, err.Error(), "outside the authoring subset", + "the refusal names itself a subset verdict, not a format one") + }) + } + + t.Run("member-format coupling the format refuses at semantics", func(t *testing.T) { + // The subset invariant's first failure, found by probing rather than + // by a sweep: `options` off select/multi_select and `object_types` + // off objects/files are §12 ERRORS the authoring schema originally + // admitted — an authoring-valid document the format refused. The + // coupling is schema-expressible, so the subset now refuses both at + // generation time, and this pins that the two sides agree. + for name, doc := range map[string]string{ + "options off select": `{"version": 2, "kind": "object_type", "internal_key": "t1", + "properties": {"name": "T"}, + "type_settings": {"property_definitions": [{"property": "p1", "format": "date", "options": ["A"]}]}}`, + "object_types off objects/files": `{"version": 2, "kind": "object_type", "internal_key": "t1", + "properties": {"name": "T"}, + "type_settings": {"property_definitions": [{"property": "p1", "format": "number", "object_types": ["task"]}]}}`, + } { + t.Run(name, func(t *testing.T) { + data := []byte(doc) + require.Error(t, Validate(data), "validation refuses the pairing") + require.Error(t, authoringOnly(data), "and the subset must refuse it at the schema, or it admits what the format rejects") + }) + } + }) + + t.Run("the shape that used to mean a template is refused at the schema", func(t *testing.T) { + // {"type": "template"} with no kind meant a template before `kind` + // existed. The full reader refused it until the freeze; the version + // gate answers for every pre-freeze document now (§15 #9), so at + // version 2 the full reader reads it as an ordinary page. The + // authoring schema keeps refusing it by its own conditional, which is + // a subset's privilege and the right verdict for an AUTHOR: nobody + // writing a bundle from nothing means "a page whose type is the + // template type", and the mistake is caught before the format ever + // has to guess. + require.NoError(t, Validate([]byte(`{"version": 2, "type": "template"}`)), + "the full reader has no special case here any more") + assert.Error(t, authoringOnly([]byte(`{"version": 2, "type": "template"}`))) + }) + + t.Run("index surfaces", func(t *testing.T) { + // `_all_objects` used to be the example here — a listing the importer + // dropped, so the subset refused it. The importer knows the whole + // inventory since GO-7383 and the listing is authorable now; what the + // subset still refuses on the index is the machine-written state the + // widget-object lift carries: the auto-widget ledger and the + // auto-added flag, which only a live client can write honestly. + for name, doc := range map[string]string{ + "the manifest": `{"version": 2, "name": "X", "entrypoint": "p1", + "manifest": {"properties": "properties.json"}}`, + "the auto-widget ledger": `{"version": 2, "name": "X", "entrypoint": "p1", + "auto_widget_targets": ["_bin"]}`, + "an auto-added widget": `{"version": 2, "name": "X", "entrypoint": "p1", + "widgets": [{"target": "p1", "auto_added": true}]}`, + } { + t.Run(name, func(t *testing.T) { + data := []byte(doc) + _, err := UnmarshalIndex(data) + require.NoError(t, err, "must be full-valid:\n%s", doc) + require.Error(t, ValidateAuthoringIndex(data)) + }) + } + }) + + t.Run("a dictionary entry identified only by internal_key", func(t *testing.T) { + doc := []byte(`{"version": 2, "properties": [{"internal_key": "6a32d4856761631534b22f85", "format": "number"}]}`) + _, err := UnmarshalPropertyDictionary(doc) + require.NoError(t, err, "must be full-valid") + require.Error(t, ValidateAuthoringPropertyDictionary(doc), + "an author states a spelling or a name; a stored id is the app's to mint") + }) + + t.Run("dictionary member-format coupling", func(t *testing.T) { + // the full dictionary reader TOLERATES these pairings (unlike a type + // entry, where §12 refuses them), so on this surface the coupling is + // an ordinary subset narrowing: full-valid, subset-refused + for name, doc := range map[string]string{ + "options off select": `{"version": 2, "properties": [{"property": "p1", "format": "date", "options": ["A"]}]}`, + "object_types off objects": `{"version": 2, "properties": [{"property": "p1", "format": "number", "object_types": ["task"]}]}`, + "include_time off date": `{"version": 2, "properties": [{"property": "p1", "format": "text", "include_time": true}]}`, + } { + t.Run(name, func(t *testing.T) { + data := []byte(doc) + _, err := UnmarshalPropertyDictionary(data) + require.NoError(t, err, "must be full-valid:\n%s", doc) + require.Error(t, ValidateAuthoringPropertyDictionary(data)) + }) + } + }) +} + +// --- schema hygiene ----------------------------------------------------- + +// The three authoring schemas carry their published identity and none of the +// export machinery: no x-output-only member survives into a surface whose +// whole point is that nothing in it is output-only. +func TestAuthoringSchemas_IdentityAndHygiene(t *testing.T) { + for name, tc := range map[string]struct { + bytes []byte + url string + }{ + "object": {authoringSchemaJSON, AuthoringSchemaURL}, + "index": {authoringIndexSchemaJSON, AuthoringIndexSchemaURL}, + "properties": {authoringPropertiesSchemaJSON, AuthoringPropertiesSchemaURL}, + } { + t.Run(name, func(t *testing.T) { + var doc struct { + Id string `json:"$id"` + Properties struct { + Version struct { + Const *int `json:"const"` + } `json:"version"` + } `json:"properties"` + } + require.NoError(t, json.Unmarshal(tc.bytes, &doc)) + assert.Equal(t, tc.url, doc.Id, "$id must be the published URL FormatVersion derives") + // the version const is the other copy the compiler cannot keep + // honest, and TestVersionIdentity covers only the three full + // schemas: an authoring schema left behind at a bump would refuse + // every document the format then writes + require.NotNil(t, doc.Properties.Version.Const, "schema must pin the version") + assert.Equal(t, FormatVersion, *doc.Properties.Version.Const) + assert.NotContains(t, string(tc.bytes), "x-output-only", + "an authoring schema has no output-only members by construction") + + // and the URL still dispatches to the right grammar (§2g): the + // trailing file name is what DocumentKind matches on + kind, decided := documentKindOf([]byte(`{"$schema": "` + tc.url + `"}`)) + require.True(t, decided) + switch name { + case "object": + assert.Equal(t, KindObject, kind) + case "index": + assert.Equal(t, KindIndex, kind) + case "properties": + assert.Equal(t, KindPropertyDictionary, kind) + } + }) + } +} + +// The authoring schema reserves the same TEN listing words §1 bans as +// bundle-local document ids — it held the pre-v0.45 six for a while, so +// `ValidateAuthoring` accepted ids `chat`, `bin`, `allObjects` and +// `recentOpen`, and `chat`/`bin` are the two most common listing widgets: an +// authored bundle really would collide there first. +// +// How this can fail: regenerate or hand-edit the object authoring schema's +// $defs/documentId from the shorter list and the four newer words validate +// again, while the index authoring schema beside it still refuses them — +// the same id legal in one file and reserved in the other. +func TestAuthoring_AllTenReservedIdsAreRefused(t *testing.T) { + for _, id := range []string{ + "favorite", "recent", "recentOpen", "set", "collection", + "allObjects", "chat", "bin", "widgets", "graph", + } { + t.Run(id, func(t *testing.T) { + doc := `{"version": 2, "id": "` + id + `", "blocks": [{"type": "paragraph", "text": "x"}]}` + require.Error(t, ValidateAuthoring([]byte(doc)), + "a reserved listing word must not be a bundle-local id") + }) + } + t.Run("an ordinary id stays legal", func(t *testing.T) { + doc := `{"version": 2, "id": "chat-notes", "blocks": [{"type": "paragraph", "text": "x"}]}` + require.NoError(t, ValidateAuthoring([]byte(doc))) + }) +} diff --git a/pkg/lib/anyblockjson/authoringsemantics_test.go b/pkg/lib/anyblockjson/authoringsemantics_test.go new file mode 100644 index 0000000000..107e6e49fe --- /dev/null +++ b/pkg/lib/anyblockjson/authoringsemantics_test.go @@ -0,0 +1,161 @@ +package anyblockjson + +// authoringsemantics_test.go — the two authoring-subset rules that had to +// leave the schema, because JSON Schema matches a member name literally and +// the format resolves a property key case- and separator-insensitively. +// +// A literal rule over a key the codec spells many ways is not a narrower +// rule; it is a rule with holes. Both rules below had one: the canonical +// `{"Name": "Habit"}` was REFUSED as a type document's name while the +// retired lowercase spelling was accepted, and nine derived keys lost their +// authoring refusal entirely the moment their canonical spelling became a +// display name. + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A type document must name itself, and every spelling that RESOLVES to the +// name property satisfies it. The rule used to be `required: ["name"]` in +// the authoring schema, which accepted exactly one of these four and +// refused the canonical one. +func TestAuthoringTypeDocumentNamesItself(t *testing.T) { + typeDoc := func(props string) []byte { + return []byte(`{"version":2,"kind":"object_type","id":"habit","internal_key":"habit",` + + `"type_settings":{"layout":"basic"},"properties":{` + props + `}}`) + } + + t.Run("every spelling of the name property satisfies the rule", func(t *testing.T) { + for _, spelling := range []string{"Name", "name", "NAME"} { + doc := typeDoc(`"` + spelling + `":"Habit"`) + require.NoError(t, Validate(doc), "the full format accepts it: %s", doc) + assert.NoError(t, ValidateAuthoring(doc), + "%q resolves to the name property, so the type names itself", spelling) + } + }) + + t.Run("a type document with no name at all is still refused", func(t *testing.T) { + doc := typeDoc(`"Description":"a habit"`) + require.NoError(t, Validate(doc), "nameless is valid AnyBlock JSON — the subset is what refuses it") + + err := ValidateAuthoring(doc) + require.Error(t, err) + assert.Contains(t, err.Error(), "/properties") + assert.Contains(t, err.Error(), "states its own display name") + }) + + t.Run("the rule is a type document's alone", func(t *testing.T) { + doc := []byte(`{"version":2,"id":"page1","properties":{"Description":"no title"}}`) + assert.NoError(t, ValidateAuthoring(doc), + "an ordinary object may be untitled; only a type must name itself") + }) +} + +// The nine keys the FULL format does not refuse — it DROPS them at import — +// are refused by the authoring subset under every spelling, not only the +// pre-raw-name one the schema's literal list happens to name. +func TestAuthoringDeniedKeysAreRefusedUnderEverySpelling(t *testing.T) { + for key, why := range authoringDeniedPropertyKeys { + name := BundledKeyVocabulary{}.PropertySlug(key) + require.NotEqual(t, key, name, "%s has a display name to be spelled by", key) + + for _, spelling := range []string{key, name, strings.ToLower(name)} { + t.Run(key+" as "+spelling, func(t *testing.T) { + b, err := json.Marshal(map[string]any{ + "version": 2, "id": "o1", + "properties": map[string]any{spelling: "whatever"}, + }) + require.NoError(t, err) + + err = ValidateAuthoring(b) + require.Error(t, err, "an author does not write %q (%s)", key, why) + assert.Contains(t, err.Error(), key, + "the refusal names the resolved key, not just the spelling written") + }) + } + } +} + +// The Go set is the authority and the schema's literal list is the +// documentation; this pins them together, in both directions, so neither +// can rot the way the list already did once. +func TestAuthoringDeniedKeysMatchTheSchema(t *testing.T) { + var schema struct { + Defs struct { + PropertyMap struct { + PropertyNames struct { + Not struct { + Enum []string `json:"enum"` + } `json:"not"` + } `json:"propertyNames"` + } `json:"propertyMap"` + } `json:"$defs"` + } + require.NoError(t, json.Unmarshal(AuthoringSchemaJSON(), &schema)) + listed := map[string]bool{} + for _, e := range schema.Defs.PropertyMap.PropertyNames.Not.Enum { + listed[e] = true + } + require.NotEmpty(t, listed) + + t.Run("every authoring-denied key is listed under both its spellings", func(t *testing.T) { + for key := range authoringDeniedPropertyKeys { + assert.True(t, listed[key], "the schema list is missing the stored key %q", key) + name := BundledKeyVocabulary{}.PropertySlug(key) + assert.True(t, listed[name], + "the schema list is missing %q, which is how the format spells %q today", name, key) + } + }) + + t.Run("no listed spelling has quietly gone toothless", func(t *testing.T) { + // A listed spelling that RESOLVES to a stored key is a rule about + // that key, and something has to enforce it for every OTHER + // spelling too: either the full format's deny rule (import refuses + // exactly what export strips) or the Go set above. An entry backed + // by neither states a rule nothing enforces — which is precisely + // what happened when the canonical spelling of these keys became a + // display name and the list kept banning only the old one. + // + // Entries that resolve to NOTHING are skipped, and they are the + // bulk of the list: a denied key's fold class deliberately answers + // nothing, so `icon_emoji` is an ordinary custom property name in + // the full format. Banning it here is the subset catching an author + // who meant the envelope icon, and only the subset can. + for _, spelling := range schema.Defs.PropertyMap.PropertyNames.Not.Enum { + key, resolves := BundledKeyVocabulary{}.PropertyKey(spelling) + if !resolves { + continue + } + if _, denied := authoringDeniedPropertyKeys[key]; denied { + continue + } + doc := []byte(fmt.Sprintf(`{"version":2,"id":"o1","properties":{%q:"x"}}`, key)) + assert.Error(t, Validate(doc), + "the schema bans %q, which resolves to %q — but nothing refuses that key "+ + "under its own spelling, so the ban covers one spelling of many", spelling, key) + } + }) +} + +// The subset narrows a name-over-number key to the NAME, and that rule was +// keyed to the retired member name too. +func TestAuthoringNamedEnumTakesTheNameUnderItsCanonicalSpelling(t *testing.T) { + for _, spelling := range []string{"Layout align", "layout_align", "layoutAlign"} { + t.Run(spelling, func(t *testing.T) { + named := []byte(fmt.Sprintf(`{"version":2,"id":"o1","properties":{%q:"center"}}`, spelling)) + assert.NoError(t, ValidateAuthoring(named), "the name is what an author writes") + + bare := []byte(fmt.Sprintf(`{"version":2,"id":"o1","properties":{%q:1}}`, spelling)) + require.NoError(t, Validate(bare), "the full format passes the stored number through") + err := ValidateAuthoring(bare) + require.Error(t, err, "the subset removes the stored-value pass-through") + assert.Contains(t, err.Error(), "layoutAlign") + }) + } +} diff --git a/pkg/lib/anyblockjson/blockvocab.go b/pkg/lib/anyblockjson/blockvocab.go new file mode 100644 index 0000000000..73c1f46b70 --- /dev/null +++ b/pkg/lib/anyblockjson/blockvocab.go @@ -0,0 +1,180 @@ +package anyblockjson + +// blockvocab.go exports the §5 block-type vocabulary the way viewvocab.go +// exports §6.2's: as the single list the API layer consumes, so a surface +// that publishes the types to a generator cannot drift from what the codec +// reads. Here the derivation is literal — the names are read out of the +// embedded JSON Schema's own enum, so there is no second list to keep in +// step at all. + +import ( + "encoding/json" + "fmt" + "slices" +) + +// blockTypeNames is the §5 inventory, read out of the embedded schema at +// startup. It is a must-decode for the same reason regexp.MustCompile is: the +// input is a build-time embedded asset, and a schema that does not carry its +// own block-type inventory is not one this package can read documents with. +var blockTypeNames = mustSchemaBlockTypes() + +func mustSchemaBlockTypes() []string { + var doc struct { + Defs struct { + BlockCore struct { + Properties struct { + Type struct { + Enum []string `json:"enum"` + } `json:"type"` + } `json:"properties"` + } `json:"blockCore"` + } `json:"$defs"` + } + if err := json.Unmarshal(schemaJSON, &doc); err != nil { + panic(fmt.Sprintf("anyblockjson: decode embedded schema: %v", err)) + } + names := doc.Defs.BlockCore.Properties.Type.Enum + if len(names) == 0 { + panic("anyblockjson: the embedded schema publishes no block-type enum") + } + return names +} + +// structuralBlockTypes are the §7 structural blocks: derivable from +// properties, dropped on export, and — at indent 0 — absorbed or dropped on +// import (topLevelBlocks reads this map, so which types are structural is +// stated once). They are part of the format's vocabulary but not of a body a +// caller can author: whatever is written into one, the block does not survive +// the round trip. +var structuralBlockTypes = map[string]bool{ + "title": true, "description": true, "featured_properties": true, +} + +// transparentBlockTypes are the §7a transparent containers: types that carry +// containment and nothing else, so export writes their children in their +// place and import lifts them back out. They are part of the format's +// vocabulary — Validate must keep accepting what Unmarshal accepts, so +// `group` stays in BlockTypeNames and in the schema's `blockCore.type` enum +// — but no export produces one and nothing a caller writes into one +// survives, so they are not part of an authorable body either. +// +// This is deliberately NOT merged with structuralBlockTypes, which reads +// nearly the same and means the opposite: topLevelBlocks drops a structural +// block TOGETHER WITH ITS SUBTREE, where a container's whole point is that +// its subtree stays and only it goes. +var transparentBlockTypes = map[string]bool{ + "group": true, +} + +// StructuralBlockType reports whether typ is a §7 structural block. Exported +// for the API v2 surfaces that publish an authorable vocabulary. +func StructuralBlockType(typ string) bool { + return structuralBlockTypes[typ] +} + +// TransparentBlockType reports whether typ is a §7a transparent container — +// a type a document may carry that resolves to no block of its own. Exported +// for the same API v2 surfaces as StructuralBlockType: a generator shown +// this type would write a block that vanishes on the next read. +func TransparentBlockType(typ string) bool { + return transparentBlockTypes[typ] +} + +// BlockTypeNames lists every §5 block type the format reads, in schema order +// (input aliases — heading4/header4 for heading3, equation for embed — +// included: they are values a document may legitimately carry). +func BlockTypeNames() []string { + return slices.Clone(blockTypeNames) +} + +// AuthorableBlockTypeNames is the vocabulary a document BODY can be written +// in: BlockTypeNames minus the §7 structural types, which import absorbs into +// properties or drops, and minus the §7a transparent containers, which import +// lifts away. Order follows BlockTypeNames. +func AuthorableBlockTypeNames() []string { + out := make([]string, 0, len(blockTypeNames)) + for _, name := range blockTypeNames { + if !structuralBlockTypes[name] && !transparentBlockTypes[name] { + out = append(out, name) + } + } + return out +} + +// blockPropertyNames is the §5 block ATTRIBUTE inventory, read out of the same +// embedded schema — the shared core plus every conditional per-type branch. +// Must-decode for the same reason blockTypeNames is. +var blockPropertyNames = mustSchemaBlockProperties() + +func mustSchemaBlockProperties() map[string]bool { + var doc struct { + Defs map[string]json.RawMessage `json:"$defs"` + } + if err := json.Unmarshal(schemaJSON, &doc); err != nil { + panic(fmt.Sprintf("anyblockjson: decode embedded schema: %v", err)) + } + names := map[string]bool{} + var collect func(raw json.RawMessage) + collect = func(raw json.RawMessage) { + var node map[string]json.RawMessage + if err := json.Unmarshal(raw, &node); err != nil { + // an array of subschemas (allOf/anyOf/oneOf) — recurse into each + var list []json.RawMessage + if json.Unmarshal(raw, &list) == nil { + for _, item := range list { + collect(item) + } + } + return + } + if props, ok := node["properties"]; ok { + var fields map[string]json.RawMessage + if json.Unmarshal(props, &fields) == nil { + for name := range fields { + names[name] = true + } + } + } + // only the composition keywords are descended: a property's own value + // is a schema for that property, not another property inventory + for _, keyword := range []string{"allOf", "anyOf", "oneOf", "if", "then", "else"} { + if sub, ok := node[keyword]; ok { + collect(sub) + } + } + } + for _, def := range []string{"blockCore", "block", "cellBlock"} { + if raw, ok := doc.Defs[def]; ok { + collect(raw) + } + } + if len(names) == 0 { + panic("anyblockjson: the embedded schema publishes no block properties") + } + return names +} + +// BlockPropertyNames lists every ATTRIBUTE name the format's block schema +// knows — the shared core plus the per-type conditional branches — sorted. +// +// Exported for the API v2 surfaces that re-publish a subset of the block shape +// (the PATCH op schemas): those defs are additionalProperties:false, so a name +// they publish that the format does not know is a field no document can ever +// carry, and a constrained decoder shown it emits a block the codec rejects. +// Block attribute names are NOT key slots (§3) — this is the +// build-enforced form of that exclusion. +func BlockPropertyNames() []string { + out := make([]string, 0, len(blockPropertyNames)) + for name := range blockPropertyNames { + out = append(out, name) + } + slices.Sort(out) + return out +} + +// KnownBlockProperty reports whether the format's block schema knows this +// attribute name. +func KnownBlockProperty(name string) bool { + return blockPropertyNames[name] +} diff --git a/pkg/lib/anyblockjson/blockvocab_test.go b/pkg/lib/anyblockjson/blockvocab_test.go new file mode 100644 index 0000000000..677996db04 --- /dev/null +++ b/pkg/lib/anyblockjson/blockvocab_test.go @@ -0,0 +1,117 @@ +package anyblockjson + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestBlockVocabularyIsTheSchemaEnum pins the exported §5 vocabulary to the +// embedded schema's own enum — the list is READ from it, so this asserts the +// reading, not a hand-kept copy. A type added to the schema appears here with +// no code change; a type that stops being in the schema disappears from every +// surface that publishes the vocabulary at the same moment. +func TestBlockVocabularyIsTheSchemaEnum(t *testing.T) { + // given + var doc map[string]any + require.NoError(t, json.Unmarshal(SchemaJSON(), &doc)) + defs := doc["$defs"].(map[string]any) + blockCore := defs["blockCore"].(map[string]any) + props := blockCore["properties"].(map[string]any) + typeProp := props["type"].(map[string]any) + var want []string + for _, v := range typeProp["enum"].([]any) { + want = append(want, v.(string)) + } + + // when + got := BlockTypeNames() + + // then + require.NotEmpty(t, want) + assert.Equal(t, want, got, "the exported vocabulary is the schema's own enum, in schema order") + assert.Contains(t, got, "checkbox", "the type a checkbox item needs is in the vocabulary") +} + +// TestAuthorableVocabularyDropsTheStructuralTypes: §7 structural blocks are +// part of the format's vocabulary but not of a body a caller can author — +// import absorbs title/description into properties and drops +// featuredProperties, so a payload naming one produces no block. A published +// enum must not offer a value that cannot survive. +func TestAuthorableVocabularyDropsTheStructuralTypes(t *testing.T) { + // when + authorable := AuthorableBlockTypeNames() + + // then + assert.Len(t, authorable, len(BlockTypeNames())-len(structuralBlockTypes)-len(transparentBlockTypes)) + for _, typ := range []string{"title", "description", "featured_properties"} { + assert.True(t, StructuralBlockType(typ), "%s is structural (§7)", typ) + assert.NotContains(t, authorable, typ) + assert.Contains(t, BlockTypeNames(), typ, "it is still part of the format's vocabulary") + } + assert.False(t, StructuralBlockType("paragraph")) + assert.Subset(t, BlockTypeNames(), authorable) +} + +// TestAuthorableVocabularyDropsTheTransparentContainers is the §7a half of +// the same rule, and the reason the two sets stay apart: a structural block +// is dropped WITH its subtree, a container is dropped and its subtree stays. +// Both are readable, neither is authorable. +func TestAuthorableVocabularyDropsTheTransparentContainers(t *testing.T) { + // when + authorable := AuthorableBlockTypeNames() + + // then + for typ := range transparentBlockTypes { + assert.True(t, TransparentBlockType(typ), "%s is a transparent container (§7a)", typ) + assert.False(t, StructuralBlockType(typ), "%s is not structural — its subtree survives", typ) + assert.NotContains(t, authorable, typ) + assert.Contains(t, BlockTypeNames(), typ, + "it stays readable: Validate must keep accepting what Unmarshal accepts (I2)") + } + assert.False(t, TransparentBlockType("paragraph")) + assert.False(t, TransparentBlockType("row"), "a row is author-created and grammar-bearing (§5)") +} + +// TestStructuralTypesAreDroppedOnImport is the behaviour the exclusion above +// claims: the importer reads the same map, so every structural name really +// does produce no block. +func TestStructuralTypesAreDroppedOnImport(t *testing.T) { + // the baseline: the same document without the structural block + _, plain, err := Unmarshal([]byte(`{"version":2,"type":"page","blocks":[{"type":"paragraph","text":"body"}]}`), Options{}) + require.NoError(t, err) + + for typ := range structuralBlockTypes { + t.Run(typ, func(t *testing.T) { + // given — featuredProperties carries no text of its own (§5) + structural := fmt.Sprintf(`{"type":%q,"text":"structural"}`, typ) + if typ == "featured_properties" { + structural = fmt.Sprintf(`{"type":%q}`, typ) + } + doc := fmt.Sprintf(`{"version":2,"type":"page","blocks":[%s,{"type":"paragraph","text":"body"}]}`, structural) + + // when + _, snapshot, err := Unmarshal([]byte(doc), Options{}) + + // then + require.NoError(t, err) + assert.Len(t, snapshot.Blocks, len(plain.Blocks), "a structural block must not survive as a block") + for _, b := range snapshot.Blocks { + if text := b.GetText(); text != nil { + assert.NotEqual(t, "structural", text.Text) + } + } + }) + } +} + +// TestBlockVocabularyCopiesOut: callers get their own slice — the package +// list is read at startup and shared by every surface that publishes it. +func TestBlockVocabularyCopiesOut(t *testing.T) { + got := BlockTypeNames() + got[0] = "clobbered" + assert.NotEqual(t, "clobbered", BlockTypeNames()[0]) +} diff --git a/pkg/lib/anyblockjson/buildrecommended_typelegend_test.go b/pkg/lib/anyblockjson/buildrecommended_typelegend_test.go new file mode 100644 index 0000000000..d09c73e5d6 --- /dev/null +++ b/pkg/lib/anyblockjson/buildrecommended_typelegend_test.go @@ -0,0 +1,71 @@ +package anyblockjson + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The missing half of TestBuildRecommendedLists_HonoursTheLegendItIsHandedOver: +// object_types is a TYPE key slot and it runs through the SAME legend, but no +// test names it, so opts.legendTypeKey can be reverted to opts.typeKey and the +// whole package stays green while a type's property targets re-point. +type recTypeVocab struct { + BundledKeyVocabulary + slugs map[string]string +} + +func (v recTypeVocab) TypeSlug(k string) string { + if s, ok := v.slugs[k]; ok { + return s + } + return BundledKeyVocabulary{}.TypeSlug(k) +} +func (v recTypeVocab) TypeKey(s string) (string, bool) { + for k, sl := range v.slugs { + if sl == s { + return k, true + } + } + return BundledKeyVocabulary{}.TypeKey(s) +} + +type capturingResolver struct{ seen *[]string } + +func (r capturingResolver) PropertyById(string) (PropertyDefinition, bool) { + return PropertyDefinition{}, false +} +func (r capturingResolver) PropertyId(def PropertyDefinition) (string, bool) { + *r.seen = append(*r.seen, def.ObjectTypes...) + return "id-" + string(def.Key), true +} + +func TestBuildRecommendedLists_ObjectTypesHonourTheLegendToo(t *testing.T) { + const liveType = "69bbfc78877a91b1d12d1a7c" + props := []TypeProperty{{Property: "who", Section: "featured", + Format: "objects", ObjectTypes: []string{"initiative"}}} + + // the decoy: this reader binds the spelling `initiative` to another type + base := Options{Keys: recTypeVocab{slugs: map[string]string{"decoyType": "initiative"}}} + + t.Run("without the legend the reader's own answer wins", func(t *testing.T) { + var seen []string + o := base + o.ResolveProperties = capturingResolver{seen: &seen} + _, err := BuildRecommendedLists(props, o) + require.NoError(t, err) + assert.Equal(t, []string{"decoyType"}, seen) + }) + + t.Run("with it the document's own statement is chain step 1", func(t *testing.T) { + var seen []string + o := base + o.ResolveProperties = capturingResolver{seen: &seen} + o.Legend = Legend{TypeKeys: map[string]string{"initiative": liveType}} + _, err := BuildRecommendedLists(props, o) + require.NoError(t, err) + assert.Equal(t, []string{liveType}, seen, + "the property's targets name the types the document meant") + }) +} diff --git a/pkg/lib/anyblockjson/bundledname.go b/pkg/lib/anyblockjson/bundledname.go new file mode 100644 index 0000000000..fbd6f8894a --- /dev/null +++ b/pkg/lib/anyblockjson/bundledname.go @@ -0,0 +1,359 @@ +package anyblockjson + +// bundledname.go — the wire spellings of BUNDLED property and type keys: +// their display names, from the tables that ship with every reader. +// +// The format spells every property and type key by the entity's display +// name, NFC-normalized and otherwise verbatim — one uniform rule for +// bundled and space-minted keys alike. For bundled keys the name comes from +// `pkg/lib/bundle`'s relations.json/types.json, so `createdDate` spells +// "Creation date" and the Page type spells "Page", offline, with no store. +// This file is the bundled half of that rule, both directions, and it +// replaces two mechanisms at once: +// +// - the derived api-slug table (`bundle.ApiSlug`, i.e. strcase.ToSnake): +// the slug survives on the API surface, whose key convention is a +// separate decision, but it is no longer a document spelling — 0 of 194 +// bundled relation names are byte-equal to their derived slug, and the +// name is the surface users and models actually see; +// - the retired alias table (alias.go, deleted): its sixteen respellings +// existed because the stored key said "relation" where the format says +// "property", and the display names never did — the relation TYPE is +// named "Property" in the bundle, `relationFormat` is named "Format" — +// so the names carry the rename with no table behind it. The sixteen +// alias spellings themselves (`featured_properties`, …) are cut rather +// than kept as an accept-only layer: the format is pre-freeze, no +// back-compat is owed, and every existing bundle re-exports under the +// new spelling either way. +// +// Legacy derived-slug spellings keep resolving WITHOUT a compatibility +// table, through the fold layer: ToSnake only inserts `_` and lowercases, +// and the fold strips `_`, `-` and case, so fold(ToSnake(key)) == fold(key) +// by construction and every pre-change spelling (`created_date`) lands in +// its stored key's fold class. The fold below answers for both the key's +// class and the name's class, so `creation_date` — the guess shaped like +// the name — forgives too. +// +// **A name that does not uniquely invert is not a spelling.** Nine hidden +// transient relations share the name "Underlying file id"; all nine sit in +// the stripped set (transientProperties), so no document ever spells them — +// but the rule does not lean on that: a key whose name the reverse table +// cannot bind spells its stored key verbatim, which is always its own +// address, so an ambiguous bundled name can never be emitted at all. The +// guard tests in bundledname_test.go keep the wire-reachable population +// clean so this fallback only ever covers invisible machinery. + +import ( + "sort" + "strings" + "unicode" + + "golang.org/x/text/unicode/norm" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" +) + +// FoldKeyTerm is the format's forgiving fold over key terms — the class two +// spellings must share for the near-miss layer to bridge them. Wider than +// the api surface's bundle.FoldApiKey on purpose: a display name separates +// its words with SPACES where a stored key uses case and a derived slug +// uses `_`, so a fold that kept spaces would put "Due Date 2" and +// `due_date_2` in different classes and the legacy-continuity proof above +// would not hold. NFC first (a hand-edited document can arrive decomposed), +// then lowercase, then drop `_`, `-`, every whitespace rune, and the +// default-ignorable code points (variation selectors, ZWJ, word joiners — +// two production names carry an invisible variation selector, and an +// invisible near-miss is the least visible near-miss there is). +// +// Exact match always wins before the fold is consulted, and two keys +// folding together is an ambiguity the caller must refuse to resolve by +// guess — the same contract bundle.FoldApiKey states. +// +// NFC runs TWICE, and the second pass is not belt and braces. Dropping a +// separator can put two runes next to each other that were not neighbours +// before, and a composable pair only composes when it is adjacent: +// "A_" + a combining acute folded to `a` + U+0301, while the precomposed +// "Á" folded to U+00E1 — two spellings a reader would call the same word, +// in different fold classes, and the fold was not even idempotent on its +// own output. Normalizing after the map puts every result in one form. +func FoldKeyTerm(s string) string { + return norm.NFC.String(strings.Map(func(r rune) rune { + switch { + case r == '_' || r == '-': + return -1 + case unicode.IsSpace(r): + return -1 + case unicode.Is(unicode.Variation_Selector, r) || + unicode.Is(unicode.Other_Default_Ignorable_Code_Point, r) || + unicode.Is(unicode.Cf, r): + return -1 + } + return r + }, strings.ToLower(norm.NFC.String(s)))) +} + +// The bundled name tables, built once. Forward holds only names the reverse +// uniquely binds; reverse is exact NFC name → stored key; fold holds every +// entry's key class and name class, multi-valued where classes collide. +var ( + bundledPropertyNameByKey map[string]string + bundledPropertyKeyByName map[string]string + bundledPropertyKeysByFold map[string][]string + bundledTypeNameByKey map[string]string + bundledTypeKeyByName map[string]string + bundledTypeKeysByFold map[string][]string +) + +func init() { + relKeys := make([]string, 0, 256) + for _, k := range bundle.ListRelationsKeys() { + relKeys = append(relKeys, string(k)) + } + sort.Strings(relKeys) + relName := func(key string) string { + rel, err := bundle.PickRelation(domain.RelationKey(key)) + if err != nil { + return "" + } + return rel.Name + } + // A DENIED key's fold class must answer nothing. The exact name stays in + // the reverse table — "Format" in a `properties` map is refused WITH the + // envelope repair named, and a reference slot naming the key spells it — + // but the fold is the forgiving layer, and forgiveness toward a key + // import refuses would re-break what the lift settled: `format` and + // `include_time` are legal CUSTOM property names on any document (the + // phantom-member warning is the guard there), and a fold that pulled + // them onto the lifted stored keys would turn that warning into a + // refusal. + bundledPropertyNameByKey, bundledPropertyKeyByName, bundledPropertyKeysByFold = + buildNameTables(relKeys, relName, func(key string) bool { + _, denied := deniedPropertyKey(key) + return denied + }) + + typeKeys := make([]string, 0, 64) + for _, k := range bundle.ListTypesKeys() { + typeKeys = append(typeKeys, string(k)) + } + sort.Strings(typeKeys) + typeName := func(key string) string { + t, err := bundle.GetType(domain.TypeKey(key)) + if err != nil { + return "" + } + return t.Name + } + bundledTypeNameByKey, bundledTypeKeyByName, bundledTypeKeysByFold = + buildNameTables(typeKeys, typeName, func(string) bool { return false }) +} + +// buildNameTables derives one namespace's three tables from the shipped +// bundle. Keys arrive SORTED so that where the input is ambiguous the +// outcome is still deterministic — though ambiguity never picks a winner: +// +// - a name two keys share binds NEITHER in the reverse table and spells +// neither in the forward one (both fall back to their stored keys); +// - a name that byte-equals a DIFFERENT entry's stored key is refused the +// same way — the stored key resolves verbatim-first at every reader, so +// a spelling equal to it could never invert to anyone else; +// - a name that is not a writable key (empty, over the bound, control +// characters) has no wire form and the key spells itself. +// +// The fold table is built over every entry regardless: an ambiguous fold +// class simply holds several candidates, which the forgiving layer already +// treats as "refuse, never guess". +func buildNameTables(keys []string, nameOf func(string) string, foldExcluded func(string) bool) ( + nameByKey, keyByName map[string]string, foldTable map[string][]string) { + stored := make(map[string]bool, len(keys)) + for _, k := range keys { + stored[k] = true + } + claim := map[string][]string{} + for _, k := range keys { + name := norm.NFC.String(nameOf(k)) + if name == "" || name == k || !isWritablePropertyKey(name) { + continue + } + if stored[name] { + continue // another entry's stored key outranks any name, verbatim-first + } + claim[name] = append(claim[name], k) + } + nameByKey = make(map[string]string, len(claim)) + keyByName = make(map[string]string, len(claim)) + for name, holders := range claim { + if len(holders) != 1 { + continue // shared name: nobody spells it, nobody answers to it + } + nameByKey[holders[0]] = name + keyByName[name] = holders[0] + } + foldTable = map[string][]string{} + addFoldClass := func(class, key string) { + for _, existing := range foldTable[class] { + if existing == key { + return + } + } + foldTable[class] = append(foldTable[class], key) + } + for _, k := range keys { + if foldExcluded(k) { + continue + } + addFoldClass(FoldKeyTerm(k), k) + if name := norm.NFC.String(nameOf(k)); name != "" { + addFoldClass(FoldKeyTerm(name), k) + } + } + return nameByKey, keyByName, foldTable +} + +// bundledPropertySpelling is the wire spelling of a BUNDLED relation key: +// its display name where the name uniquely inverts, the stored key itself +// otherwise (always its own address). The caller has already established +// the key is bundled — a non-bundled key must never reach the bundled +// table (dictionaryKeySpelling's bson-id rule). +func bundledPropertySpelling(key string) string { + if name, ok := bundledPropertyNameByKey[key]; ok { + return name + } + return key +} + +// BundledPropertyKeyByName inverts bundledPropertySpelling EXACTLY: an NFC +// display name names its key, and nothing else answers. It is the exact +// layer alone, deliberately: a stored key resolves verbatim at chain step 2 +// without this table, and near-misses (the legacy derived slugs included) +// belong to the fold layer, which must see EVERY candidate — a space's own +// fold claimants included — before it may answer. Exported because +// storeresolver's exact candidate layer runs this same table, and folding +// inside it would let a bundled near-miss win while a space-minted twin in +// the same fold class went unseen. +func BundledPropertyKeyByName(spelling string) (string, bool) { + key, ok := bundledPropertyKeyByName[spelling] + return key, ok +} + +func bundledPropertyKeyBySpelling(spelling string) (string, bool) { + return BundledPropertyKeyByName(spelling) +} + +// bundledTypeSpelling / bundledTypeKeyBySpelling are the type namespace's +// halves of the same two rules. +func bundledTypeSpelling(key string) string { + if name, ok := bundledTypeNameByKey[key]; ok { + return name + } + return key +} + +func bundledTypeKeyBySpelling(spelling string) (string, bool) { + return BundledTypeKeyByName(spelling) +} + +// BundledTypeKeyByName is BundledPropertyKeyByName on the type namespace. +func BundledTypeKeyByName(spelling string) (string, bool) { + key, ok := bundledTypeKeyByName[spelling] + return key, ok +} + +// BundledPropertyKeysByFold is the bundled arm of the fold layer (§3 chain +// step 4 — the near-miss forgiveness): every bundled relation key whose +// stored key OR display name folds to the input's class. Zero matches: not +// bundled; one: the forgiveness; two or more: an ambiguity the caller must +// refuse rather than resolve. Exported because storeresolver's chain runs +// the same bundled arm the package-only reader does, and the two must not +// disagree about which fold class a key answers to. +func BundledPropertyKeysByFold(term string) []string { + return bundledPropertyKeysByFold[FoldKeyTerm(term)] +} + +// BundledTypeKeysByFold is BundledPropertyKeysByFold on the type namespace. +func BundledTypeKeysByFold(term string) []string { + return bundledTypeKeysByFold[FoldKeyTerm(term)] +} + +// BundledPropertyNameExtendedBy reports the bundled relation display name +// that `term` extends with trailing text, when one does — the copy-boundary +// hazard's bundled half: a writer gluing an annotation onto a name it was +// copying ("Creation date (text)") produces a term no table answers, and +// the one useful fact about it is which live name it started as. The +// LONGEST extended name wins so the report names the fullest match; +// equal-length ties break lexicographically for determinism. +func BundledPropertyNameExtendedBy(term string) (string, bool) { + return nameExtendedBy(term, bundledPropertyKeyByName) +} + +// BundledTypeNameExtendedBy is the type namespace's half. +func BundledTypeNameExtendedBy(term string) (string, bool) { + return nameExtendedBy(term, bundledTypeKeyByName) +} + +func nameExtendedBy(term string, names map[string]string) (string, bool) { + var best string + for name := range names { + if !KeyTermExtendsName(term, name) { + continue + } + if len(name) > len(best) || (len(name) == len(best) && name < best) { + best = name + } + } + return best, best != "" +} + +// KeyTermExtendsName reports whether term is name plus trailing text that +// begins at a word boundary. The boundary check is what separates a glued +// annotation ("Tag (text)") from a longer name that happens to share a +// prefix ("Tagline" is not "Tag" with something glued on): the first rune +// past the name must not be a letter or digit. Exported because the +// space-backed vocabulary applies the same rule to its own live names, and +// the two diagnostics must not disagree about what counts as glue. +func KeyTermExtendsName(term, name string) bool { + if name == "" || len(term) <= len(name) || !strings.HasPrefix(term, name) { + return false + } + for _, r := range term[len(name):] { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + } + return false +} + +// DisambiguatedKeySpelling is the shared rung (b) of the collision ladder — +// the spelling a claimant takes when its plain name is unusable (contested +// inside one document, or equal to another live stored key) and its own +// stored key is not a readable substitute. It answers "" in exactly the +// cases where rung (a) or (c) applies instead: +// +// - a stored key that is NOT a minted 24-hex bson id is readable, and the +// honest disambiguation is the key itself, verbatim (rung a); +// - a suffixed form that would not be a writable key — a name already at +// the length bound has no room — is refused rather than truncated, and +// the stored key is written regardless (rung c). +// +// Otherwise the answer is ` ()`, tail6 = the stored key's last +// six hex: deterministic, immutable while the key lives, and visibly +// synthetic. Exported because the exporter's per-document term ledger and +// storeresolver's space vocabulary both run this ladder, and a claimant +// must take the same spelling whichever seam degrades it. +func DisambiguatedKeySpelling(name, key string) string { + if name == "" || !isBsonShapedKey(key) { + return "" + } + suffixed := name + " (" + key[len(key)-6:] + ")" + if !isWritablePropertyKey(suffixed) { + return "" + } + return suffixed +} + +// isBsonShapedKey reports the one stored-key shape the ladder calls +// unreadable: the 24-char lowercase-hex bson id every editor-minted +// relation and type carries. Everything else — a bundled camelCase key, an +// API-minted readable key — is its own honest spelling. +func isBsonShapedKey(key string) bool { + return isHexLower(key, 24) +} diff --git a/pkg/lib/anyblockjson/bundledname_test.go b/pkg/lib/anyblockjson/bundledname_test.go new file mode 100644 index 0000000000..50eedeefec --- /dev/null +++ b/pkg/lib/anyblockjson/bundledname_test.go @@ -0,0 +1,356 @@ +package anyblockjson + +// bundledname_test.go — bundled keys spell their display names +// (bundledname.go), the stored keys stay their own verbatim addresses, the +// legacy derived slugs keep resolving through the fold, and the shipped name +// table stays clean enough to carry the whole scheme: the CI guards at the +// bottom are the condition under which a bundled entry may be added or +// renamed. + +import ( + "strings" + "testing" + "unicode" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The headline of the uniform rule: a bundled key writes its display name — +// spaces and all — with NO legend entry, because the name table ships with +// every reader. `audioGenre` spelling "Audio genre" is also the rename that +// made this possible: its old name "Genre" collided with the genre +// relation's, and two wire spellings reading "Genre" could not both invert. +// +// How this can fail: fall back to the derived slug and `audio_genre` comes +// back; make recordPropertyKey ask a table that does not bind names and +// every bundled key starts paying a legend line for a spelling every reader +// already knows. +func TestBundledNames_KeysSpellTheirDisplayNames(t *testing.T) { + // given + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), + "audioGenre": str("ambient"), + }), + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"Audio genre"`) + assert.NotContains(t, string(data), `"audio_genre"`, + "the derived slug must not survive anywhere in the document") + assert.NotContains(t, string(data), `"property_internal_keys"`, + "a bundled name is a shipped-table fact and owes no legend entry") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + + _, back, err := Unmarshal(data, testOptions()) + require.NoError(t, err) + assert.Equal(t, str("ambient"), back.Details.Fields["audioGenre"], + "the name inverts onto the stored key") +} + +// The stored key itself still resolves VERBATIM — §3 chain step 2 is +// untouched by the name layer — and the pre-change derived slug keeps +// resolving through the FOLD, with no compatibility table: ToSnake only +// inserts `_` and lowercases, and the fold strips `_`, `-` and case, so +// every old slug sits in its stored key's fold class. +// +// How this can fail: drop the fold step from BundledKeyVocabulary and every +// document written before the re-spell mints phantom keys in a package-only +// reader; let the name table answer near-misses and an ambiguous class +// resolves by luck. +func TestBundledNames_StoredKeyVerbatimAndLegacySlugThroughTheFold(t *testing.T) { + t.Run("the stored key is its own address", func(t *testing.T) { + doc := `{"version":2,"id":"o1","properties":{"audioGenre":"jazz"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, str("jazz"), snap.Details.Fields["audioGenre"]) + }) + + t.Run("the legacy derived slug folds onto its key", func(t *testing.T) { + doc := `{"version":2,"id":"o1","properties":{"audio_genre":"jazz"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, str("jazz"), snap.Details.Fields["audioGenre"], + "fold(ToSnake(key)) == fold(key): the continuity proof, exercised") + assert.Nil(t, snap.Details.Fields["audio_genre"]) + }) + + t.Run("the name-shaped guess folds too", func(t *testing.T) { + // the §6.3 consolation: "Creation date" is the canonical spelling, + // and `creation_date` — the guess shaped like the name — lands in + // the same fold class + doc := `{"version":2,"id":"o1","properties":{"creation_date":1700000000}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + require.NotNil(t, snap.Details.Fields["createdDate"]) + }) + + t.Run("the v0.38 alias spellings come back through the fold", func(t *testing.T) { + // the alias TABLE is gone, and its spellings resolve anyway: the + // bundled name says "Property option color", and the fold strips + // case and `_`, so `property_option_color` lands in that name's + // class. Nothing had to be kept for back-compat — renaming the + // eleven bundled names that still said "relation" is what restored + // them, and it restored the derived-slug form of each new name at + // the same time. + doc := `{"version":2,"id":"o1","properties":{"property_option_color":"ice"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, str("ice"), snap.Details.Fields["relationOptionColor"]) + assert.Nil(t, snap.Details.Fields["property_option_color"]) + }) + + t.Run("a spelling no bundled name folds onto passes through verbatim", func(t *testing.T) { + // pre-freeze, no back-compat: a term whose fold class is nobody's + // is its own address — chain step 4 + doc := `{"version":2,"id":"o1","properties":{"property_wine_region":"ice"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, str("ice"), snap.Details.Fields["property_wine_region"]) + }) +} + +// The type namespace carries the same rules: a bundled type spells its +// display name with no legend entry — a property document says +// `"type": "Property"`, because the relation TYPE is named "Property" in +// the bundle and the name carries the v0.38 rename with no table behind it +// — and the stored key still names itself verbatim on the way in. +func TestBundledNames_TypeKeysSpellTheirDisplayNames(t *testing.T) { + t.Run("export spells the name, no legend owed", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, typedSnapshot("ot-relation"), Options{}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "Property", doc.Type) + assert.Empty(t, doc.TypeKeys, "a bundled name is a shipped-table fact and owes no legend entry") + + _, snap, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-relation"}, snap.ObjectTypes, + "a package-only reader inverts the name through the shipped table") + }) + + t.Run("the stored type key still names itself verbatim", func(t *testing.T) { + _, snap, err := Unmarshal([]byte(`{"version":2,"id":"o1","type":"relation"}`), + Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-relation"}, snap.ObjectTypes, + "an exact stored key wins before any table (§3 chain step 2)") + }) + + t.Run("the legacy type slug folds onto its key", func(t *testing.T) { + _, snap, err := Unmarshal([]byte(`{"version":2,"id":"o1","type":"object_type"}`), + Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-objectType"}, snap.ObjectTypes) + }) + + t.Run("the Space pair reads exactly, both ways", func(t *testing.T) { + // the other commit-one rename: `space` is "Space settings" and + // spaceView keeps "Space" — two exact names, two exact answers + key, ok := (BundledKeyVocabulary{}).TypeKey("Space settings") + require.True(t, ok) + assert.Equal(t, "space", key) + key, ok = (BundledKeyVocabulary{}).TypeKey("Space") + require.True(t, ok) + assert.Equal(t, "spaceView", key) + }) +} + +// The fold layer refuses an ambiguous class rather than resolving it. The +// one KNOWN ambiguous class in the shipped tables is the Space pair's: +// spaceView's NAME "Space" folds onto the stored key `space`, so a +// near-miss like "SPACE" answers to two keys and the forgiveness declines — +// measured, both keys appear as type-key spellings in 0 of 28,560 corpus +// documents, so the degraded forgiveness costs nothing real. The EXACT +// spellings stay unambiguous either way (the test above). +func TestBundledNames_AnAmbiguousFoldClassIsRefused(t *testing.T) { + assert.Len(t, BundledTypeKeysByFold("SPACE"), 2, + "the class holds both keys — the fixture is real") + _, snap, err := Unmarshal([]byte(`{"version":2,"id":"o1","type":"SPACE"}`), + Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-SPACE"}, snap.ObjectTypes, + "an ambiguous near-miss degrades to the verbatim term, never to a guess") + + // the nine "Underlying file id" transients are the property namespace's + // ambiguous class: shared name, so none of them spells it and the name + // answers to nobody exactly + _, ok := (BundledKeyVocabulary{}).PropertyKey("Underlying file id") + assert.False(t, ok, "a shared name binds nothing") + assert.Equal(t, "fileId", (BundledKeyVocabulary{}).PropertySlug("fileId"), + "a key whose name cannot invert spells its stored key, always its own address") +} + +// The CI rule the uniform spelling stands on (§3): over the WIRE-REACHABLE +// bundled population, names are unique, invertible, writable and clean. A +// new bundled entry that breaks any line here needs a different name — the +// way audioGenre took "Audio genre" — not a looser table. +// +// Wire-reachable means: not one of the stripped internal keys +// (InternalPropertyKeys — those never appear in a document's key slots +// under their own spelling). The nine "Underlying file id" transients are +// the tolerated remainder: they share one name, they are all stripped, and +// the table refuses to spell or bind the shared name at all, so the +// tolerance can never leak into a document. +func TestBundledNames_TheWireReachableTableStaysClean(t *testing.T) { + stripped := InternalPropertyKeys() + + t.Run("properties", func(t *testing.T) { + byName := map[string][]string{} + keys := bundledRelationKeys() + require.NotEmpty(t, keys) + for _, key := range keys { + rel, err := bundle.PickRelation(domain.RelationKey(key)) + require.NoError(t, err) + name := norm.NFC.String(rel.Name) + + if !stripped[key] { + require.NotEqualf(t, "", name, "wire-reachable %q has no name to spell", key) + byName[name] = append(byName[name], key) + got, ok := BundledPropertyKeyByName(name) + require.Truef(t, ok, "the name %q of %q must invert", name, key) + assert.Equalf(t, key, got, "the name %q must invert to its own key", name) + } + if name == "" { + continue + } + assert.Truef(t, isWritablePropertyKey(name), + "bundled name %q (of %q) is not a writable key", name, key) + assert.Equalf(t, name, strings.TrimSpace(name), + "bundled name %q (of %q) carries edge whitespace", name, key) + for _, r := range name { + assert.Falsef(t, unicode.Is(unicode.Variation_Selector, r) || + unicode.Is(unicode.Other_Default_Ignorable_Code_Point, r) || + unicode.Is(unicode.Cf, r), + "bundled name %q (of %q) carries an invisible code point", name, key) + } + for _, other := range keys { + assert.Falsef(t, other != key && name == other, + "the name %q of %q byte-equals the stored key %q — verbatim-first would answer first", + name, key, other) + } + } + for name, owners := range byName { + assert.Lenf(t, owners, 1, + "wire-reachable bundled properties %v share the name %q", owners, name) + } + // and no wire-reachable name sits in another wire-reachable entry's + // fold class: the forgiveness layer must never be pre-degraded for + // the population documents actually spell + byFold := map[string][]string{} + for _, key := range keys { + if stripped[key] { + continue + } + rel, _ := bundle.PickRelation(domain.RelationKey(key)) + for _, class := range []string{FoldKeyTerm(key), FoldKeyTerm(rel.Name)} { + owners := byFold[class] + if len(owners) == 0 || owners[len(owners)-1] != key { + byFold[class] = append(owners, key) + } + } + } + for class, owners := range byFold { + assert.Lenf(t, owners, 1, "wire-reachable bundled properties %v share the fold class %q", + owners, class) + } + }) + + t.Run("types", func(t *testing.T) { + byName := map[string][]string{} + for _, tk := range bundle.ListTypesKeys() { + typ, err := bundle.GetType(tk) + require.NoError(t, err) + name := norm.NFC.String(typ.Name) + require.NotEqualf(t, "", name, "bundled type %q has no name to spell", tk) + byName[name] = append(byName[name], string(tk)) + got, ok := BundledTypeKeyByName(name) + require.Truef(t, ok, "the name %q of type %q must invert", name, tk) + assert.Equal(t, string(tk), got) + assert.True(t, isWritablePropertyKey(name)) + assert.Equal(t, name, strings.TrimSpace(name)) + } + for name, owners := range byName { + assert.Lenf(t, owners, 1, "bundled types %v share the name %q", owners, name) + } + // the fold classes hold exactly ONE tolerated collision — the Space + // pair (pinned above); anything beyond it is a new defect + byFold := map[string]map[string]bool{} + for _, tk := range bundle.ListTypesKeys() { + typ, _ := bundle.GetType(tk) + for _, class := range []string{FoldKeyTerm(string(tk)), FoldKeyTerm(typ.Name)} { + if byFold[class] == nil { + byFold[class] = map[string]bool{} + } + byFold[class][string(tk)] = true + } + } + for class, owners := range byFold { + if class == "space" { + assert.Len(t, owners, 2, "the tolerated Space pair") + continue + } + assert.Lenf(t, owners, 1, "bundled types %v share the fold class %q", owners, class) + } + }) +} + +// Every bundled spelling — names with spaces included — is a writable key +// the whole codec carries: as a `properties` member name, a legend spelling +// and an envelope type term. The old guard asserted bundled slugs were bare +// FILTER-GRAMMAR identifiers; raw names deliberately are not (the +// identifier grammar binds only the compact filter string, which is the API +// request surface's), so the writable-key rule is the right bound now. +func TestBundledNames_EverySpellingIsAWritableKey(t *testing.T) { + for _, key := range bundledRelationKeys() { + spelling := (BundledKeyVocabulary{}).PropertySlug(key) + assert.Truef(t, isWritablePropertyKey(spelling), + "bundled key %q spells %q, which is not a writable key", key, spelling) + } + for _, tk := range bundle.ListTypesKeys() { + spelling := (BundledKeyVocabulary{}).TypeSlug(string(tk)) + assert.Truef(t, isWritablePropertyKey(spelling), + "bundled type %q spells %q, which is not a writable key", tk, spelling) + } +} + +// The fold is IDEMPOTENT, and it has to be: a caller that folds a term it +// already folded — the near-miss layer indexing its own keys, a test +// comparing two classes — must land in the same class, or two spellings a +// reader calls one word sit in two. +// +// The way it failed is worth keeping: dropping a separator makes two runes +// neighbours that were not, and a composable pair only composes when it is +// adjacent. NFC ran BEFORE the strip, so `A_` + a combining acute folded to +// a decomposed `á` while the precomposed `Á` folded to U+00E1. +func TestFoldKeyTerm_NormalizesAfterTheStrip(t *testing.T) { + const ( + combining = "A_́" // "A", "_", COMBINING ACUTE ACCENT + precomposed = "Á" // "Á" + ) + t.Run("a separator between a letter and its accent does not split the class", func(t *testing.T) { + assert.Equal(t, FoldKeyTerm(precomposed), FoldKeyTerm(combining), + "the accent composes onto the letter the strip made it adjacent to") + }) + + t.Run("folding a folded term changes nothing", func(t *testing.T) { + for _, s := range []string{combining, precomposed, "Due Date", "due_date", "Дата выполнения", "作業内容", "C++", "☕"} { + once := FoldKeyTerm(s) + assert.Equal(t, once, FoldKeyTerm(once), "fold is idempotent on %q", s) + } + }) +} diff --git a/pkg/lib/anyblockjson/compactsplit_test.go b/pkg/lib/anyblockjson/compactsplit_test.go new file mode 100644 index 0000000000..73387df300 --- /dev/null +++ b/pkg/lib/anyblockjson/compactsplit_test.go @@ -0,0 +1,265 @@ +package anyblockjson + +// compactsplit_test.go covers what id compaction is after v0.20: ONE half. +// Object-ref compaction and its `refs` legend are deleted (§9a), so the only +// compaction left is CompactBlockLabels — doc-local relabeling that carries +// no legend — and CompactIds is its alias. + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func compactSplitSnapshot() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{ + "id": str("bafyreiselfobjectidxxxxxxx"), + "name": str("Doc"), + }), + Blocks: []*model.Block{ + {Id: "bafyreiselfobjectidxxxxxxx", ChildrenIds: []string{"64b2c1d2e3f4a5b6c7d8e9f0", "featuredRelations"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + textBlock("64b2c1d2e3f4a5b6c7d8e9f0", model.BlockContentText_Paragraph, "ping Roman", + mark(mMention, 5, 10, "bafyreimentiontargetidxxx")), + // a real editor id that is NOT minted-shaped: the old charset rule + // relabeled it ("tions"), the minted rule serves it verbatim — the + // discriminating id the labels-only test pins the rule with + textBlock("featuredRelations", model.BlockContentText_Paragraph, "meaningful id"), + }, + } +} + +type parsedCompactDoc struct { + Blocks []struct { + Id string `json:"id"` + } `json:"blocks"` +} + +func marshalCompactSplit(t *testing.T, opts Options) (parsedCompactDoc, string) { + t.Helper() + data, err := Marshal(model.SmartBlockType_Page, compactSplitSnapshot(), opts) + require.NoError(t, err) + require.NoError(t, Validate(data)) + var doc parsedCompactDoc + require.NoError(t, json.Unmarshal(data, &doc)) + return doc, string(data) +} + +// Every shape leaves object references full and inline. Stated over the +// compaction flag AND over the default, because "object ids compact" was the +// documented behaviour of exactly one of those and this is what replaced it. +func TestExport_ObjectRefsAreNeverCompacted(t *testing.T) { + for name, opts := range map[string]Options{ + "default": {}, + "CompactBlockLabels": {CompactBlockLabels: true}, + "CompactIds": {CompactIds: true}, + "OmitIds": {OmitIds: true}, + } { + t.Run(name, func(t *testing.T) { + // given / when + _, s := marshalCompactSplit(t, opts) + + // then: the mention target is spelled in full where it is used, + // and there is no legend anywhere to look it up in + assert.Contains(t, s, `object_id=\"bafyreimentiontargetidxxx\"`, + "the mention target must be written in full") + assert.NotContains(t, s, `"idxxx"`, "no short label for it may appear") + }) + } +} + +func TestExport_CompactBlockLabelsOnly(t *testing.T) { + // given / when + doc, s := marshalCompactSplit(t, Options{CompactBlockLabels: true}) + + // then: ONLY the minted id relabels — a meaningful editor id serves + // verbatim (the old charset rule relabeled "featuredRelations" to + // "tions"; without this id the assertion could not fail against either + // rule, both relabel a 24-hex id) + require.Len(t, doc.Blocks, 2) + assert.Equal(t, "8e9f0", doc.Blocks[0].Id) + assert.Equal(t, "featuredRelations", doc.Blocks[1].Id) + assert.Contains(t, s, "bafyreimentiontargetidxxx", + "the mention target must survive uncompacted in the document body") +} + +func TestExport_CompactIdsIsAnAliasForBlockLabels(t *testing.T) { + // given / when + viaAlias, aliasBytes := marshalCompactSplit(t, Options{CompactIds: true}) + viaFlag, flagBytes := marshalCompactSplit(t, Options{CompactBlockLabels: true}) + + // then — byte-identical, which is the whole content of "alias" + assert.Equal(t, flagBytes, aliasBytes) + assert.Equal(t, viaFlag, viaAlias) + require.Len(t, viaAlias.Blocks, 2) + assert.Equal(t, "8e9f0", viaAlias.Blocks[0].Id, "minted-only relabeling holds under CompactIds too") +} + +// TestExport_MintedShapeRelabeling pins the relabel rule: only machine- +// minted opaque ids (24-hex bson/API mints, RFC-4122 view UUIDs) relabel; +// anything that could carry meaning — structural constants like "dataview", +// readable seeded/imported ids, short hand-authored ids — keeps its full +// spelling and is reserved so no label can alias it. The fixture carries +// the three id populations the review asked for (minted, hyphenated-tail, +// 5-char) plus the aliasing pair that used to serve two blocks under one id. +func TestExport_MintedShapeRelabeling(t *testing.T) { + // given — id populations: + // minted 24-hex, relabels to its last 5 chars + // readable hyphenated tail, stays full + // constants "dataview", "featuredRelations": dash-free tails that the + // old charset rule relabeled — must stay full now + // short-hex "abcde": 5 lowercase hex chars, a label look-alike + // alias-minted a real minted id ENDING in "abcde" — must not take the + // short block's id as its label + const ( + minted = "64b2c1d2e3f4a5b6c7d8e9f0" + readable = "pages-roadmap-home-1" + constant1 = "dataview" + constant2 = "featuredRelations" + shortHex = "abcde" + aliasMint = "fffffffffffffffffffabcde" + viewUuid = "32726bf3-cd8b-4099-aafb-688e9525ed67" + mintedWant = "8e9f0" + ) + children := []string{minted, readable, constant1, constant2, shortHex, aliasMint} + blocks := []*model.Block{ + {Id: "rootselfid", ChildrenIds: children, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + } + for _, id := range children { + blocks = append(blocks, textBlock(id, model.BlockContentText_Paragraph, "text of "+id)) + } + blocks[3] = &model.Block{Id: constant1, Content: &model.BlockContentOfDataview{ + Dataview: &model.BlockContentDataview{Views: []*model.BlockContentDataviewView{{ + Id: viewUuid, + Type: model.BlockContentDataviewView_Table, + Name: "All", + }}}, + }} + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("rootselfid"), "name": str("Doc")}), + Blocks: blocks, + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{CompactBlockLabels: true}) + require.NoError(t, err) + // the served document must stay schema-valid + require.NoError(t, Validate(data)) + + var doc struct { + Blocks []struct { + Id string `json:"id"` + Views []struct { + Id string `json:"id"` + } `json:"views"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + served := make([]string, 0, len(doc.Blocks)) + seen := map[string]int{} + var viewIds []string + for _, b := range doc.Blocks { + served = append(served, b.Id) + seen[b.Id]++ + for _, v := range b.Views { + viewIds = append(viewIds, v.Id) + seen[v.Id]++ + } + } + + // then — which ids come back relabeled + assert.Contains(t, served, mintedWant, "a minted 24-hex id relabels") + assert.Contains(t, served, readable, "a hyphenated readable id stays full") + assert.Contains(t, served, constant1, "the dataview constant stays full") + assert.Contains(t, served, constant2, "featuredRelations stays full — the old charset rule relabeled it to \"tions\"") + assert.Contains(t, served, shortHex, "a short id serves as itself") + assert.Contains(t, served, aliasMint, + "a minted id whose suffix spells another block's id must stay full, not alias it") + assert.Equal(t, []string{"5ed67"}, viewIds, "a UUID view id relabels") + + // the invariant behind finding 1, pinned independently of the rule that + // produces it: no two blocks/views ever share a served id + for id, n := range seen { + assert.Equalf(t, 1, n, "served id %q appears %d times", id, n) + } +} + +// The `refs` legend is not merely unwritten — a document that CARRIES one is +// refused, and refused at an address the writer can act on. +// +// This is the headline deletion of v0.20, and until now nothing stood on it. +// The exporter no longer emits `refs`, so every export-side assertion passes +// whether or not a reader would accept one; what makes the deletion real is +// the read side refusing the member, and that rests on a single token in the +// schema — the envelope's `additionalProperties: false`. Re-admitting `refs` +// there (as a permissive object, which is how it was spelled) leaves every +// other test in this package green, and the format quietly grows back a +// legend whose values nothing resolves: the labels inside the document would +// then be read as literal object ids, silently re-pointing every reference. +// +// The refusal has to say WHICH member to drop (§12) — an agent regenerating a +// document from a pre-v0.20 memory needs the name, not "the document is bad +// somewhere" — and at the envelope root that name arrives in the MESSAGE, not +// in the path. The root closes with `additionalProperties: false`, which the +// validator reports once for the object and lists the offending names in its +// text; inside a block the same refusal comes from `unevaluatedProperties`, +// which is reported per member and so carries `/blocks/0/bogus` +// (TestValidate_ErrorsDoNotCascade). The assertions below pin what the +// refusal actually says, message included, so a member-addressed root path +// would be a deliberate change and not a silent one. +func TestValidate_TheRefsLegendIsRefused(t *testing.T) { + refused := func(t *testing.T, doc string) []Issue { + t.Helper() + err := Validate([]byte(doc)) + require.Error(t, err, "a document carrying a `refs` legend must not validate") + var ve *ValidationError + require.True(t, errors.As(err, &ve), "got %v", err) + return ve.Issues + } + + t.Run("the legend a pre-v0.20 exporter wrote", func(t *testing.T) { + // the exact shape: short labels in the body, the legend to invert them + got := refused(t, `{"version": 2, "id": "bafyreiselfobjectidxxxxxxx", + "refs": {"idxxx": "bafyreimentiontargetidxxx"}, + "blocks": [{"id": "b1", "type": "paragraph", + "text": "ping Roman"}]}`) + require.Len(t, got, 1, "got: %v", got) + // the refusal addresses the MEMBER, not the envelope holding it: the + // schema's own closed-set verdict carries the object's location and + // named `refs` only inside its text, so a reader was handed an empty + // path for a fault it could point at (§12) + assert.Equal(t, "/refs", got[0].Path) + // and it says what happened. `version` is still 1 across the grammar + // change, so this message is the only place a pre-v0.20 document is + // told why it stopped validating — and the only place the reader is + // warned off the repair the bare verdict suggests + assert.Contains(t, got[0].Message, "written in full", + "the message states the rule that replaced the legend") + assert.Contains(t, got[0].Message, "address nothing", + "and warns that deleting the legend alone strands the labels it inverted") + }) + + t.Run("an empty legend is refused as well", func(t *testing.T) { + // nothing about the refusal may depend on the legend's CONTENTS: a + // schema node admitting `refs` and constraining it would still let + // this through + got := refused(t, `{"version": 2, "refs": {}}`) + require.Len(t, got, 1, "got: %v", got) + assert.Equal(t, "/refs", got[0].Path) + }) + + t.Run("import refuses it too, so no reader takes the labels literally", func(t *testing.T) { + _, _, err := Unmarshal([]byte(`{"version": 2, "refs": {"idxxx": "bafyreitarget"}, + "blocks": [{"type": "paragraph", "text": "x"}]}`), Options{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "refs") + }) +} diff --git a/pkg/lib/anyblockjson/compose/compose.go b/pkg/lib/anyblockjson/compose/compose.go new file mode 100644 index 0000000000..5285ea262c --- /dev/null +++ b/pkg/lib/anyblockjson/compose/compose.go @@ -0,0 +1,556 @@ +// Package compose is the bundle-level composition of an AnyBlock JSON +// export: everything above one document that a bundle must state — +// properties.json, index.json with its manifest, and the omission-and-lift +// of the documents those two files carry INSTEAD of (SPEC.md §2c, §2f). +// +// It exists because composition is a bundle-level act the one-document codec +// deliberately does not own (SPEC.md §13 gives it this named home), and +// because two independent writers need the SAME implementation: the +// production exporter (core/block/export/anyblock) and the cmd tools +// (cmd/anyblockroundtrip's corpus sweep, which is what makes the sweep an +// end-to-end test of production composition rather than of a private copy). +// +// The shape follows the exporter design (EXPORTER_DESIGN.md §1.1/§1.5): +// BuildPlan runs single-threaded over details-level facts before the first +// emit; the Composer's Observe* methods are safe for concurrent emit tasks +// and accumulate only commutative aggregates under one mutex; Finish sorts +// everything it writes and re-reads both files through the package's own +// Unmarshal before handing them back — the bundle-level I1 discipline, so a +// bundle this code writes that the package refuses is found at export time. +package compose + +import ( + "fmt" + "sort" + "strings" + "sync" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/snapshotdiff" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +// Issue is one bundle-level finding — an omitted document whose lift or +// reconstruction does not account for everything it held. The exporter logs +// these (an omission that loses data is a bug here, not a reason to fail a +// user's export); the round-trip harness counts them as failures. +type Issue struct { + Category string + Detail string +} + +// Stats is what Finish can say about the composed bundle, for summaries. +type Stats struct { + DictionaryInstalled int + DictionaryEntries int + ManifestTypes int + ManifestFiles int + OptionDocs int + DictionaryBytes int + IndexBytes int + OmittedDocs int + // OrphanUsedKeys are referenced property keys with no definition + // anywhere — no relation object, not bundled — so the dictionary cannot + // state a format for them (§2f names every property it CAN). + OrphanUsedKeys []string +} + +// Composer accumulates, across one bundle's emit, everything the two +// bundle-level files state: which bundled relations are installed (and which +// of their documents the emit omitted), the definitions the dictionary +// carries, the option vocabularies, the index lift from the omitted +// space-settings and widget documents, and where the manifest finds each +// type and each file blob. +// +// Observe, ObserveWritten and ObserveFileBlob are safe for concurrent use — +// the emit phase runs width-bounded tasks (design §1.5) and everything +// shared here is commutative map/set insertion under one mutex, held for +// microseconds against marshal work measured in milliseconds. Finish is +// called once, after every emit task returned. +type Composer struct { + mu sync.Mutex + + opts anyblockjson.Options + spaceName string + + installed map[string]bool + // entries the space's own documents define: a KEPT bundled-key relation + // document (divergent from the table, or carrying something only a + // document can) contributes its stored definition, so the dictionary + // states the divergence the `installed` list alone would paper over + entries map[string]anyblockjson.PropertyDefinition + + typePaths map[string]string + filePaths map[string]string + optionPaths map[string]string + // optionsByKey is the select vocabulary each property actually has in + // this space, gathered from the option documents so the dictionary can + // state it inline (§2f). Keyed by STORED property key, and held with the + // stored `orderId` so the inline array can be written in the order the + // space actually shows. + optionsByKey map[string][]storedOption + + // used is the referenced-key census the dictionary's used-only rule + // needs (§2f), gathered from each document's marshalled bytes as it is + // observed. From the BYTES, not a re-read: a zip export cannot re-read + // its own entries before Close, so the scan runs before the write — + // which is also what lets the cmd tools and production share it + // (UsedPropertyKeysFromBytes, design §1.1). + used map[string]bool + + written int + omitted int + + // the fields the space's own document and the widget object are the + // sources of (§2c). Lifted as each document is observed and omitted, so + // the index states what the dropped documents held. + index anyblockjson.Index +} + +// NewComposer creates a composer for one bundle. opts is consulted from +// inside the composer's own mutex only, so a store-backed resolver that is +// not safe for concurrent use (storeresolver.Resolvers) is fine HERE — but +// it must then be a dedicated instance, not one an emit worker also uses. +// spaceName is the fallback for a space whose own document states no name. +func NewComposer(opts anyblockjson.Options, spaceName string) *Composer { + return &Composer{ + opts: opts, + spaceName: spaceName, + installed: map[string]bool{}, + entries: map[string]anyblockjson.PropertyDefinition{}, + typePaths: map[string]string{}, + filePaths: map[string]string{}, + optionPaths: map[string]string{}, + optionsByKey: map[string][]storedOption{}, + used: map[string]bool{}, + } +} + +// Observe classifies one snapshot for the composition. For an omitted +// document it also verifies the trip the object takes INSTEAD of a document +// — the index lift, or installed key → the reader's bundled table — through +// the same comparator as every ordinary round trip, so the omission +// predicate and the reconstruction cannot drift apart silently. +// +// The caller emits the document iff omitted is false; issues are reported +// either way (an issue on an omitted document means the lift lost +// something, which the §1.7 contract forbids). +func (c *Composer) Observe(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase) (omitted bool, issues []Issue) { + c.mu.Lock() + defer c.mu.Unlock() + omitted, issues = c.observe(sbType, base) + if omitted { + c.omitted++ + } + return omitted, issues +} + +func (c *Composer) observe(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase) (bool, []Issue) { + if base == nil { + return false, nil + } + // the space's own object: index.json states everything it holds (§2c), + // so the composer lifts those fields and drops the document. The lift + // runs BEFORE the omission is recorded, so a bundle can never drop the + // document without having written what it carried. + if anyblockjson.OmittedSpaceSettings(sbType, base) { + anyblockjson.IndexFromSpaceSettings(&c.index, base) + return true, nil + } + // the deprecated per-space profile object: superseded by `participant`, + // and what survives in a real account is an empty hidden object carrying + // someone else's name, dragged in by an import (§2c) + if anyblockjson.OmittedProfilePage(sbType, base) { + return true, nil + } + // the sidebar's object: index.json states everything it holds (§2c) — + // the wrapper-and-link pairs flat in `widgets`, the auto-widget ledger + // at index level — so the composer lifts those fields and drops the + // document, the space-settings rule again. The lift runs BEFORE the + // omission is recorded, and the snapshot a bundle carries INSTEAD + // (WidgetsSnapshot, the same function cmd/anyblockconvert installs + // from) is verified against the original through the same comparator as + // every ordinary round trip, so the lift and the rebuild cannot drift + // apart silently. A nil snapshot means the index carries no sidebar + // state because the object held none — the predicate is the proof. + if anyblockjson.OmittedWidgetObject(sbType, base) { + anyblockjson.IndexFromWidgetObject(&c.index, base) + rebuilt, err := anyblockjson.WidgetsSnapshot(&c.index) + if err != nil { + return true, []Issue{{Category: "omitted_reconstruction", + Detail: fmt.Sprintf("widget object: %v", err)}} + } + var issues []Issue + if rebuilt != nil { + for _, d := range snapshotdiff.Compare(base, rebuilt, sbType, c.opts) { + issues = append(issues, Issue{Category: "omitted_reconstruction", Detail: d}) + } + } + return true, issues + } + if key, ok := anyblockjson.OmittedBundledRelation(sbType, base, c.opts); ok { + c.installed[key] = true + det, ok := anyblockjson.InstalledRelationDetails(key, c.opts) + if !ok { + return true, []Issue{{Category: "omitted_reconstruction", + Detail: fmt.Sprintf("installed key %q has no bundled reconstruction", key)}} + } + var issues []Issue + got := &model.SmartBlockSnapshotBase{Details: det, ObjectTypes: base.ObjectTypes} + for _, d := range snapshotdiff.Compare(base, got, sbType, c.opts) { + issues = append(issues, Issue{Category: "omitted_reconstruction", Detail: d}) + } + return true, issues + } + if det := base.GetDetails().GetFields(); det != nil && + (sbType == model.SmartBlockType_STRelation || sbType == model.SmartBlockType_BundledRelation) { + key := det["relationKey"].GetStringValue() + if key != "" && bundle.HasRelation(domain.RelationKey(key)) { + // installed but not omittable: the document stays, and the + // dictionary carries its stored definition as the full entry + // the §2f divergence rule requires + c.installed[key] = true + c.entries[key] = storedRelationDefinition(base, c.opts) + } + } + return false, nil +} + +// ObserveWritten records one emitted document: its place for the manifest — +// a type by its STORED key, a file blob binding is ObserveFileBlob's — the +// option vocabulary an option document contributes, and the property keys +// the document's bytes reference (the dictionary's used-only census, §2f). +// path is the document's bundle-relative path, slash-separated, exactly as +// it should appear in the manifest. +func (c *Composer) ObserveWritten(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase, doc []byte, path string) error { + used, err := UsedPropertyKeysFromBytes(doc) + if err != nil { + return fmt.Errorf("scan used property keys: %w", err) + } + c.mu.Lock() + defer c.mu.Unlock() + c.written++ + for key := range used { + c.used[key] = true + } + det := base.GetDetails().GetFields() + if det == nil { + return nil + } + switch sbType { + case model.SmartBlockType_STType, model.SmartBlockType_BundledObjectType: + if key := strings.TrimPrefix(det["uniqueKey"].GetStringValue(), "ot-"); key != "" { + c.typePaths[key] = path + } + case model.SmartBlockType_STRelationOption: + // an option's whole meaning is three details — which property it + // belongs to, its name, and its colour — wrapped in a document whose + // remaining forty lines are derived scaffolding. The dictionary + // states those three inline, so a bundle declares a select vocabulary + // in the same place it declares the property (§2f). + if key := det["relationKey"].GetStringValue(); key != "" { + if name := det["name"].GetStringValue(); name != "" { + c.optionsByKey[key] = append(c.optionsByKey[key], storedOption{ + order: det["orderId"].GetStringValue(), + id: det["id"].GetStringValue(), + def: anyblockjson.OptionDefinition{ + Name: name, + Color: det["relationOptionColor"].GetStringValue(), + // the option's stored key: minted, so derivable from + // nothing, unlike its name, colour, position and api + // key (§2f). Carried by uniqueKey `opt-`. + InternalKey: strings.TrimPrefix( + det["uniqueKey"].GetStringValue(), "opt-"), + }, + }) + } + } + if id := det["id"].GetStringValue(); id != "" { + c.optionPaths[id] = path + } + } + return nil +} + +// ObserveFileBlob records one written blob for the manifest `files` map +// (§2c): the file object's id → the blob's bundle-relative path. +// Called only after the bytes are actually written, so the manifest never +// points at a blob a failed stream left absent. +func (c *Composer) ObserveFileBlob(objectId, path string) { + c.mu.Lock() + defer c.mu.Unlock() + c.filePaths[objectId] = path +} + +// Finish composes the bundle's two files and re-reads both through the +// package's own Unmarshal — the bundle-level twin of the I1 discipline: a +// file this composer writes that the package refuses is a bug here, found +// at export time rather than at restore time. Both byte slices are nil when +// nothing was written (an empty bundle states nothing). +func (c *Composer) Finish() (index, properties []byte, stats Stats, err error) { + c.mu.Lock() + defer c.mu.Unlock() + stats.OmittedDocs = c.omitted + if c.written == 0 { + return nil, nil, stats, nil + } + + // the dictionary names every property the documents actually reference + // (§2f, used-only): the space's own definitions first (divergent + // installed copies, space-minted relation documents keep their files but + // the dictionary still answers for every USED key), then the resolver, + // then the bundled table. A key none of them can define — an orphan + // detail no relation object describes — is reported, not invented. + entries := map[string]anyblockjson.PropertyDefinition{} + for key, def := range c.entries { + if c.used[key] { + entries[key] = def + } else if _, installedToo := c.installed[key]; installedToo { + // a divergent installed copy is an entry whether or not + // anything uses it: `installed` would otherwise restore the + // table's shape over the divergence + entries[key] = def + } + } + var orphans []string + for key := range c.used { + if _, have := entries[key]; have { + continue + } + if def, ok := resolvedDefinition(key, c.opts); ok { + entries[key] = def + continue + } + if rel, relErr := bundle.GetRelation(domain.RelationKey(key)); relErr == nil { + entries[key] = anyblockjson.PropertyDefinition{ + Key: domain.RelationKey(key), Name: rel.Name, Format: rel.Format, + ObjectTypes: bundledTargetKeys(rel.ObjectTypes), + } + continue + } + orphans = append(orphans, key) + } + sort.Strings(orphans) + + // the select vocabulary travels with the property that owns it. A + // property whose options a space minted has no entry otherwise — it is an + // ordinary installed bundled key — and its vocabulary would exist only in + // the option documents, where an author generating a bundle has to know + // to look for it. + for key, stored := range c.optionsByKey { + if !c.used[key] { + continue // §2f is used-only: an unused property's vocabulary buys a reader nothing + } + // in the order the SPACE shows them, which the stored `orderId` + // carries: `status` really reads To Do → In Progress → Done, and + // sorting by name turned that workflow into Done → In Progress → + // To Do on 42 of the 61 vocabularies that state an order. + // + // An option with no orderId sorts AFTER the ordered ones, by name, + // and that is not a compromise — it is the app's own model. Ordering + // is a newer feature than options: 229 of 312 vocabularies state no + // order at all and 21 state one for only some members, and the app's + // own placement query (objectcreator/relation_option.go) filters + // `orderId NotEmpty`, so an option without one is not in the app's + // ordering either. There is no order to lose; name is what makes the + // canonical form deterministic. + sort.SliceStable(stored, func(i, j int) bool { + a, b := stored[i], stored[j] + if (a.order == "") != (b.order == "") { + return a.order != "" + } + if a.order != b.order { + return a.order < b.order + } + if a.def.Name != b.def.Name { + return a.def.Name < b.def.Name + } + // the total-order tie-break (see storedOption.id): without it a + // name shared by two options left the pair in insertion order, + // which the concurrent emit does not fix + return a.id < b.id + }) + opts := make([]anyblockjson.OptionDefinition, 0, len(stored)) + for _, so := range stored { + opts = append(opts, so.def) + } + def, have := entries[key] + if !have { + if resolved, ok := resolvedDefinition(key, c.opts); ok { + def = resolved + } else if rel, relErr := bundle.GetRelation(domain.RelationKey(key)); relErr == nil { + def = anyblockjson.PropertyDefinition{ + Key: domain.RelationKey(key), Name: rel.Name, Format: rel.Format, + ObjectTypes: bundledTargetKeys(rel.ObjectTypes), + } + } else { + continue // nothing can say what this property is; §2f reports it as an orphan + } + } + def.Options = opts + entries[key] = def + } + + dict := &anyblockjson.PropertyDictionary{} + for key := range c.installed { + dict.Installed = append(dict.Installed, key) + } + for _, key := range sortedEntryKeys(entries) { + dict.Properties = append(dict.Properties, entries[key]) + } + dictData, err := anyblockjson.MarshalPropertyDictionary(dict) + if err != nil { + return nil, nil, stats, fmt.Errorf("marshal property dictionary: %w", err) + } + if _, err := anyblockjson.UnmarshalPropertyDictionary(dictData); err != nil { + return nil, nil, stats, fmt.Errorf("re-read property dictionary: %w", err) + } + + // start from what the space's own document was lifted into (§2c) rather + // than copying its fields across by hand: the hand-written version listed + // three, and silently dropped the space ICON the moment the lift learned + // to carry one. Whatever IndexFromSpaceSettings writes now travels + // without this function being told about it. + idx := c.index + // the caller's name is the fallback for a space whose document has none + if idx.Name == "" { + idx.Name = c.spaceName + } + idx.Manifest = &anyblockjson.Manifest{ + Types: copyNonEmpty(c.typePaths), + Properties: anyblockjson.PropertiesFileName, + Files: copyNonEmpty(c.filePaths), + } + idxData, err := anyblockjson.MarshalIndex(&idx) + if err != nil { + return nil, nil, stats, fmt.Errorf("marshal index: %w", err) + } + if _, err := anyblockjson.UnmarshalIndex(idxData); err != nil { + return nil, nil, stats, fmt.Errorf("re-read index: %w", err) + } + + stats.DictionaryInstalled = len(dict.Installed) + stats.DictionaryEntries = len(dict.Properties) + stats.ManifestTypes = len(c.typePaths) + stats.ManifestFiles = len(c.filePaths) + stats.OptionDocs = len(c.optionPaths) + stats.DictionaryBytes = len(dictData) + stats.IndexBytes = len(idxData) + stats.OrphanUsedKeys = orphans + return idxData, dictData, stats, nil +} + +// storedOption is one option document's contribution to the inline +// vocabulary: the definition the dictionary states, plus the stored `orderId` +// that decides where it sits. The orderId itself never reaches a document — +// it is a lexid, which is exactly the spelling this format keeps out of an +// author's way; the ARRAY POSITION is what carries the order. +type storedOption struct { + order string + // id is the option document's own id — the total-order tie-break. Two + // options of one property may legitimately share a name (and even a + // colour), and (order, name) alone is then not a total order: the tie + // fell back to insertion order, which under the concurrent emit is + // scheduling order, and the corpus sweep caught two exports of one + // space disagreeing about which colour sat at which position. The id is + // the one member that cannot tie. + id string + def anyblockjson.OptionDefinition +} + +// storedRelationDefinition reads the definition a kept relation document +// states, off its stored details — the §2f full entry for a divergent +// installed copy. Members mirror what the document itself would carry. +func storedRelationDefinition(base *model.SmartBlockSnapshotBase, opts anyblockjson.Options) anyblockjson.PropertyDefinition { + det := base.GetDetails().GetFields() + def := anyblockjson.PropertyDefinition{ + Key: domain.RelationKey(det["relationKey"].GetStringValue()), + Name: det["name"].GetStringValue(), + Format: model.RelationFormat(int32(det["relationFormat"].GetNumberValue())), + Description: det["description"].GetStringValue(), + MaxCount: int64(det["relationMaxCount"].GetNumberValue()), + Readonly: det["relationReadonlyValue"].GetBoolValue(), + } + if v := det["relationFormatIncludeTime"]; v != nil { + if _, isBool := v.GetKind().(*types.Value_BoolValue); isBool { + b := v.GetBoolValue() + def.IncludeTime = &b + } + } + if v := det["relationDefaultValue"]; v != nil { + if _, isNull := v.GetKind().(*types.Value_NullValue); !isNull { + def.DefaultValue = pbtypes.ValueToInterface(v) + } + } + if v := det["relationFormatObjectTypes"]; v != nil { + tr, _ := opts.ResolveProperties.(anyblockjson.TypeResolver) + for _, entry := range pbtypes.GetStringListValue(v) { + if key, err := bundle.TypeKeyFromUrl(entry); err == nil { + def.ObjectTypes = append(def.ObjectTypes, string(key)) + continue + } + if tr != nil { + if key, ok := tr.TypeKeyById(entry); ok && key != "" { + def.ObjectTypes = append(def.ObjectTypes, key) + continue + } + } + def.ObjectTypes = append(def.ObjectTypes, entry) + } + } + return def +} + +// resolvedDefinition asks the space's resolver for a used key's definition — +// the storeresolver path a live export runs on. +func resolvedDefinition(key string, opts anyblockjson.Options) (anyblockjson.PropertyDefinition, bool) { + r := opts.ResolveProperties + if r == nil { + return anyblockjson.PropertyDefinition{}, false + } + if id, ok := r.PropertyId(anyblockjson.PropertyDefinition{Key: domain.RelationKey(key)}); ok { + if def, ok := r.PropertyById(id); ok { + return def, true + } + } + return anyblockjson.PropertyDefinition{}, false +} + +// bundledTargetKeys turns the bundled table's target urls into type keys. +func bundledTargetKeys(urls []string) []string { + var out []string + for _, u := range urls { + if k, err := bundle.TypeKeyFromUrl(u); err == nil { + out = append(out, string(k)) + } + } + return out +} + +// sortedEntryKeys lists a map's keys in order — the canonical entry order. +func sortedEntryKeys(m map[string]anyblockjson.PropertyDefinition) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// copyNonEmpty snapshots a path map, nil when there is nothing to state — +// the §4 omit-empty canon for the manifest's tables. +func copyNonEmpty(m map[string]string) map[string]string { + if len(m) == 0 { + return nil + } + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} diff --git a/pkg/lib/anyblockjson/compose/compose_test.go b/pkg/lib/anyblockjson/compose/compose_test.go new file mode 100644 index 0000000000..23e3d9e4ef --- /dev/null +++ b/pkg/lib/anyblockjson/compose/compose_test.go @@ -0,0 +1,262 @@ +package compose + +// compose_test.go pins the composer against the §2c/§2f composition it +// re-homes from the roundtrip harness's spaceComposer: the lift-before-omit +// discipline, the used-only dictionary, the manifest's three tables, and the +// bundle-level I1 re-read. The corpus sweep exercises the same code end to +// end over 38k real documents; these tests pin the mechanism on a space +// small enough to read. + +import ( + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func strVal(s string) *types.Value { + return &types.Value{Kind: &types.Value_StringValue{StringValue: s}} +} +func numVal(n float64) *types.Value { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: n}} +} +func boolVal(b bool) *types.Value { return &types.Value{Kind: &types.Value_BoolValue{BoolValue: b}} } + +func detFields(det map[string]*types.Value) *types.Struct { + return &types.Struct{Fields: det} +} + +// testSpaceSnapshot is a space document index.json fully states — the +// omission's happy case (§2c). +func testSpaceSnapshot() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "bafyreispace", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: detFields(map[string]*types.Value{ + "id": strVal("bafyreispace"), "name": strVal("Corpus"), + "homepage": strVal("bafyreihome"), "layout": numVal(9), "resolvedLayout": numVal(10), + "isHidden": boolVal(true), + }), + } +} + +// testInstalledCopy is a field-identical installed copy of a bundled +// relation — the omit-into-`installed` case (§2f), install provenance and +// all. +func testInstalledCopy(t *testing.T, key string) *model.SmartBlockSnapshotBase { + t.Helper() + det, ok := anyblockjson.InstalledRelationDetails(key, anyblockjson.Options{}) + require.True(t, ok) + det.Fields["createdDate"] = numVal(1700000000) + det.Fields["origin"] = numVal(2) + det.Fields["sourceObject"] = strVal("_br" + key) + det.Fields["layout"] = numVal(float64(model.ObjectType_relation)) + return &model.SmartBlockSnapshotBase{Details: det} +} + +// One small space, end to end: the two omitted documents lift into the +// bundle files, the written ones feed the manifest, the option document's +// vocabulary lands inline on the property that owns it, and both files +// re-read through the package's own Unmarshal (I1 at bundle scope). +// +// How this can fail: record the omission before the lift (the space's name +// vanishes with its document); build the dictionary from ALL keys instead +// of used ones (§2f's used-only rule breaks); key the manifest by the +// document spelling instead of the stored key; or skip the re-read and ship +// a bundle the package itself refuses — found at restore time instead of +// here. +func TestComposer_ComposesTheBundleFiles(t *testing.T) { + // given + c := NewComposer(anyblockjson.Options{}, "Fallback name") + + omitted, issues := c.Observe(model.SmartBlockType_Workspace, testSpaceSnapshot()) + require.True(t, omitted, "index.json states everything the space document holds") + require.Empty(t, issues) + + omitted, issues = c.Observe(model.SmartBlockType_STRelation, testInstalledCopy(t, "dueDate")) + require.True(t, omitted, "a field-identical installed copy travels as its key") + require.Empty(t, issues) + + typeSnap := &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{ + "id": strVal("bafytask"), "uniqueKey": strVal("ot-task"), + })} + omitted, _ = c.Observe(model.SmartBlockType_STType, typeSnap) + require.False(t, omitted) + require.NoError(t, c.ObserveWritten(model.SmartBlockType_STType, typeSnap, + []byte(`{"version":2}`), "types/bafytask.anyblock.json")) + + optSnap := &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{ + "id": strVal("bafyurgent"), "relationKey": strVal("tag"), + "name": strVal("urgent"), "relationOptionColor": strVal("red"), + "uniqueKey": strVal("opt-abcd1234"), + })} + omitted, _ = c.Observe(model.SmartBlockType_STRelationOption, optSnap) + require.False(t, omitted) + require.NoError(t, c.ObserveWritten(model.SmartBlockType_STRelationOption, optSnap, + []byte(`{"version":2}`), "options/bafyurgent.anyblock.json")) + + pageSnap := &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{ + "id": strVal("bafypage"), + })} + pageDoc := []byte(`{"version":2,"properties":{"due_date":"2026-01-01","tag":["urgent"]}}`) + omitted, _ = c.Observe(model.SmartBlockType_Page, pageSnap) + require.False(t, omitted) + require.NoError(t, c.ObserveWritten(model.SmartBlockType_Page, pageSnap, + pageDoc, "objects/bafypage.anyblock.json")) + + c.ObserveFileBlob("bafyfile", "files/bafyfile.png") + + // when + indexData, dictData, stats, err := c.Finish() + require.NoError(t, err) + + // then — the index carries the lift and the manifest's three tables + idx, err := anyblockjson.UnmarshalIndex(indexData) + require.NoError(t, err) + assert.Equal(t, "Corpus", idx.Name, "the space document's own name wins over the fallback") + assert.Equal(t, "bafyreihome", idx.Homepage) + require.NotNil(t, idx.Manifest) + assert.Equal(t, map[string]string{"task": "types/bafytask.anyblock.json"}, idx.Manifest.Types) + assert.Equal(t, map[string]string{"bafyfile": "files/bafyfile.png"}, idx.Manifest.Files) + assert.Equal(t, anyblockjson.PropertiesFileName, idx.Manifest.Properties) + + // the dictionary: the installed key, and one entry per USED key — with + // the minted vocabulary inline on the property that owns it + dict, err := anyblockjson.UnmarshalPropertyDictionary(dictData) + require.NoError(t, err) + assert.Equal(t, []string{"dueDate"}, dict.Installed) + byKey := map[string]anyblockjson.PropertyDefinition{} + for _, def := range dict.Properties { + byKey[string(def.Key)] = def + } + require.Contains(t, byKey, "tag") + require.Len(t, byKey["tag"].Options, 1) + assert.Equal(t, "urgent", byKey["tag"].Options[0].Name) + assert.Equal(t, "red", byKey["tag"].Options[0].Color) + assert.Equal(t, "abcd1234", byKey["tag"].Options[0].InternalKey) + + assert.Equal(t, 1, stats.DictionaryInstalled) + assert.Equal(t, 1, stats.ManifestTypes) + assert.Equal(t, 1, stats.ManifestFiles) + assert.Equal(t, 1, stats.OptionDocs) + assert.Equal(t, 2, stats.OmittedDocs) + assert.Empty(t, stats.OrphanUsedKeys) +} + +// The emit phase is concurrent and unordered; the composer's aggregates are +// commutative and Finish sorts everything it writes — so observation ORDER +// must never reach the bytes. This is the §1.5 determinism claim, proved on +// the aggregate rather than asserted in a comment. +// +// How this can fail: accumulate anything order-sensitive (first-writer-wins +// naming, an append the finish does not sort) and the reversed run produces +// different bytes. +func TestComposer_ObservationOrderNeverReachesTheBytes(t *testing.T) { + type obs struct { + sbType model.SmartBlockType + base *model.SmartBlockSnapshotBase + doc []byte + path string + } + build := func(t *testing.T) []obs { + return []obs{ + {model.SmartBlockType_Workspace, testSpaceSnapshot(), nil, ""}, + {model.SmartBlockType_STRelation, testInstalledCopy(t, "dueDate"), nil, ""}, + {model.SmartBlockType_STRelation, testInstalledCopy(t, "assignee"), nil, ""}, + {model.SmartBlockType_STType, &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{ + "id": strVal("bafytask"), "uniqueKey": strVal("ot-task"), + })}, []byte(`{"version":2}`), "types/bafytask.anyblock.json"}, + {model.SmartBlockType_Page, &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{ + "id": strVal("bafypage"), + })}, []byte(`{"version":2,"properties":{"due_date":"2026-01-01"}}`), "objects/bafypage.anyblock.json"}, + } + } + run := func(t *testing.T, seq []obs) (string, string) { + c := NewComposer(anyblockjson.Options{}, "Corpus") + for _, o := range seq { + omitted, _ := c.Observe(o.sbType, o.base) + if !omitted && o.doc != nil { + require.NoError(t, c.ObserveWritten(o.sbType, o.base, o.doc, o.path)) + } + } + c.ObserveFileBlob("bafyfile", "files/bafyfile.png") + index, dict, _, err := c.Finish() + require.NoError(t, err) + return string(index), string(dict) + } + + fwd := build(t) + rev := build(t) + for i, j := 0, len(rev)-1; i < j; i, j = i+1, j-1 { + rev[i], rev[j] = rev[j], rev[i] + } + + i1, d1 := run(t, fwd) + i2, d2 := run(t, rev) + assert.Equal(t, i1, i2, "index bytes must not depend on observation order") + assert.Equal(t, d1, d2, "dictionary bytes must not depend on observation order") +} + +// An empty composition states nothing: no written document, no bundle +// files — the harness's own rule for a space whose dump produced nothing. +func TestComposer_NothingWrittenNothingStated(t *testing.T) { + c := NewComposer(anyblockjson.Options{}, "Corpus") + index, dict, stats, err := c.Finish() + require.NoError(t, err) + assert.Nil(t, index) + assert.Nil(t, dict) + assert.Zero(t, stats.DictionaryEntries) +} + +// Two options of one property may share a NAME — real accounts hold such +// pairs — and (order, name) alone is then not a total order: the tie used +// to fall back to insertion order, which under the concurrent emit is +// scheduling order. The corpus sweep caught it as two exports of one space +// disagreeing about which colour sat at which vocabulary position. The +// option document's own id is the tie-break, because it is the one member +// that cannot tie. +// +// How this can fail: drop the id from the sort key (the reversed run puts +// the twins in arrival order and the bytes differ); or dedupe by name +// instead of ordering (one of two real options silently vanishes from the +// vocabulary). +func TestComposer_SameNamedOptionsHaveATotalOrder(t *testing.T) { + optSnap := func(id, color string) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{ + "id": strVal(id), "relationKey": strVal("tag"), + "name": strVal("urgent"), "relationOptionColor": strVal(color), + })} + } + run := func(t *testing.T, reversed bool) string { + c := NewComposer(anyblockjson.Options{}, "Corpus") + twins := []*model.SmartBlockSnapshotBase{optSnap("bafyaaa", "teal"), optSnap("bafyzzz", "purple")} + if reversed { + twins[0], twins[1] = twins[1], twins[0] + } + for _, snap := range twins { + omitted, _ := c.Observe(model.SmartBlockType_STRelationOption, snap) + require.False(t, omitted) + require.NoError(t, c.ObserveWritten(model.SmartBlockType_STRelationOption, snap, + []byte(`{"version":2}`), "options/"+snap.Details.Fields["id"].GetStringValue()+".anyblock.json")) + } + pageSnap := &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{"id": strVal("bafypage")})} + require.NoError(t, c.ObserveWritten(model.SmartBlockType_Page, pageSnap, + []byte(`{"version":2,"properties":{"tag":["urgent"]}}`), "objects/bafypage.anyblock.json")) + _, dict, _, err := c.Finish() + require.NoError(t, err) + return string(dict) + } + + fwd := run(t, false) + rev := run(t, true) + assert.Equal(t, fwd, rev, "vocabulary bytes must not depend on observation order") + assert.Contains(t, fwd, "teal") + assert.Contains(t, fwd, "purple", "both real options stay; ordering, not deduping") + assert.Less(t, strings.Index(fwd, "teal"), strings.Index(fwd, "purple"), + "the id tie-break is ascending: bafyaaa's colour sits first") +} diff --git a/pkg/lib/anyblockjson/compose/plan.go b/pkg/lib/anyblockjson/compose/plan.go new file mode 100644 index 0000000000..6cf1f12cf9 --- /dev/null +++ b/pkg/lib/anyblockjson/compose/plan.go @@ -0,0 +1,229 @@ +package compose + +// plan.go — the plan phase of the exporter pipeline (EXPORTER_DESIGN.md +// §1.1): classify every collected document into its kind directory and fix +// every path, single-threaded, from DETAILS-LEVEL facts only, before the +// first emit task starts. Under the settled id naming (design §1.3) a path +// is a pure per-document function of the id — no collision machinery, no +// global set, nothing ordering-sensitive left for the concurrent emit phase +// to disagree about. That purity is the whole determinism argument: same +// space state ⇒ same names, with nothing to prove about scheduling. + +import ( + "fmt" + "strings" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// DocExtension is the document filename suffix, on every kind (design +// §1.3, SPEC §15 #1 settled): the id verbatim plus this. The double +// extension is the entire is-this-a-document test — a FAT bundle carries +// blobs that are themselves .json files, so bare .json cannot be one. +const DocExtension = ".anyblock.json" + +// The kind directories of the bundle layout (design §1.2, settled Q1): +// format vocabulary, snake_case, one word each — never the store's +// `relations`/`relationsOptions` spellings, since the format promised the +// word "relation" appears nowhere a reader looks first. +const ( + DirObjects = "objects" + DirTypes = "types" + DirTemplates = "templates" + DirProperties = "properties" + DirOptions = "options" + DirParticipants = "participants" + DirFiles = "files" +) + +// DocMeta is what the plan reads about one collected document — details +// only, never content (the invariant that keeps plan O(collected details) +// and free of object loads, design §1.1). +type DocMeta struct { + Id string + SbType model.SmartBlockType + // FileExt and FileMime are a file object's stored `fileExt` / + // `fileMimeType` details, raw — the blob path inputs. Ignored for every + // other kind. Raw because the corpus measured `fileExt` dirty as a path + // component (431 empty, 12 literally "json", dozens non-alphanumeric); + // sanitation is BlobExtension's job, not the caller's. + FileExt string + FileMime string +} + +// Plan is the deterministic path table: id → document path, and for file +// objects id → blob path, all bundle-relative and slash-separated. A name +// planned for a document the emit then omits simply goes unused — +// determinism is unaffected, since omission is itself a deterministic +// function of state. +type Plan struct { + docPaths map[string]string + blobPaths map[string]string +} + +// BuildPlan fixes every path before the first emit task starts, for the +// given space. It refuses an id that cannot be a filename stem — empty, +// path separators, a dot-only component — because such an id would escape +// the bundle root; the corpus's two id populations (lowercase-base32 CIDs, +// base58 participant identities) can never trip it, so a refusal here +// means the store handed us something that is not an object id. +// +// The filename stem is the ENVELOPE id, which for a participant document +// is not the store id: Marshal folds `_participant__` +// to the bare identity (§9), and a file named by the composite would break +// the pure reference→path function §1.3 exists for — a reference carries +// the FOLDED id — besides claiming a `_`-prefixed name in the platform's +// reserved namespace (§1). The fold declines (foreign space, non-identity +// tail) exactly when Marshal's does, so stem and envelope cannot disagree. +// The Plan stays keyed by the STORE id, which is what the emit loop holds. +func BuildPlan(spaceId string, docs []DocMeta) (*Plan, error) { + p := &Plan{ + docPaths: make(map[string]string, len(docs)), + blobPaths: map[string]string{}, + } + for _, d := range docs { + stem := anyblockjson.FoldParticipantId(spaceId, d.Id) + if err := checkIdSafe(stem); err != nil { + return nil, fmt.Errorf("plan document paths: %w", err) + } + dir := KindDirectory(d.SbType) + p.docPaths[d.Id] = dir + "/" + stem + DocExtension + if dir == DirFiles { + // the blob sits beside its document: same directory, same stem + // (the id), real sanitized extension — so the two halves of a + // file sort adjacent in any listing. The manifest `files` map is + // what BINDS them (§2c); adjacency is this exporter's layout. + blobPath := dir + "/" + stem + "." + BlobExtension(d.FileExt, d.FileMime) + // a blob may never wear the document extension. The extension + // class admits no dot, so only a stem ENDING ".anyblock" plus a + // literal "json" extension could produce one — no real id + // population contains such a stem, but the invariant is checked + // rather than left as a property of today's ids. + if strings.HasSuffix(blobPath, DocExtension) { + return nil, fmt.Errorf("plan document paths: blob path %q for %s would wear the document extension", blobPath, d.Id) + } + p.blobPaths[d.Id] = blobPath + } + } + return p, nil +} + +// DocPath is the planned bundle-relative document path for id. +func (p *Plan) DocPath(id string) (string, bool) { + path, ok := p.docPaths[id] + return path, ok +} + +// BlobPath is the planned bundle-relative blob path for a file object id. +func (p *Plan) BlobPath(id string) (string, bool) { + path, ok := p.blobPaths[id] + return path, ok +} + +// KindDirectory maps a document's smartblock type onto its kind directory +// (design §1.2). Everything without a dedicated home — pages, the rare +// fail-closed widget or workspace document an omission predicate refuses — +// lands flat in objects/. +func KindDirectory(sbType model.SmartBlockType) string { + switch sbType { + case model.SmartBlockType_STType, model.SmartBlockType_BundledObjectType: + return DirTypes + case model.SmartBlockType_Template, model.SmartBlockType_BundledTemplate: + return DirTemplates + case model.SmartBlockType_STRelation, model.SmartBlockType_BundledRelation: + return DirProperties + case model.SmartBlockType_STRelationOption: + return DirOptions + case model.SmartBlockType_Participant: + return DirParticipants + case model.SmartBlockType_File, model.SmartBlockType_FileObject: + return DirFiles + default: + return DirObjects + } +} + +// blobExtensionByMime is the fallback for a file object whose stored +// `fileExt` is unusable: the conventional extension for the commonest +// stored mime types. A fixed table rather than the platform's mime +// registry, deliberately — mime.ExtensionsByType reads OS files and its +// answer varies by machine, which would make the same space export +// different bytes on different hosts. +var blobExtensionByMime = map[string]string{ + "image/jpeg": "jpg", + "image/png": "png", + "image/gif": "gif", + "image/webp": "webp", + "image/svg+xml": "svg", + "image/tiff": "tiff", + "image/bmp": "bmp", + "image/heic": "heic", + "application/pdf": "pdf", + "text/plain": "txt", + "text/csv": "csv", + "text/markdown": "md", + "application/json": "json", + "application/zip": "zip", + "video/mp4": "mp4", + "video/quicktime": "mov", + "audio/mpeg": "mp3", + "audio/mp4": "m4a", + "audio/wav": "wav", + "audio/x-wav": "wav", + "application/x-tar": "tar", + "application/gzip": "gz", + "application/msword": "doc", + // the two icon spellings live spaces actually hold (172 + 30 measured) + "image/x-icon": "ico", + "image/vnd.microsoft.icon": "ico", +} + +// BlobExtension sanitizes a file object's stored extension into a safe path +// component (design §1.4 — cosmetic only, since the manifest map binds and +// `file_mime_type` travels in the document): `fileExt` lowercased and +// restricted to [a-z0-9]{1,10}; failing that, the conventional extension +// for `fileMimeType` — with any media-type parameters stripped first, since +// the store holds parameterised values like `text/plain; charset=utf-8` +// (150 corpus objects) that would otherwise miss the table; failing that, +// "bin". BuildPlan additionally asserts the result never wears the +// document extension. +func BlobExtension(fileExt, fileMime string) string { + ext := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(fileExt), ".")) + if isCleanExt(ext) { + return ext + } + mime := strings.ToLower(strings.TrimSpace(fileMime)) + if i := strings.IndexByte(mime, ';'); i >= 0 { + mime = strings.TrimSpace(mime[:i]) + } + if byMime, ok := blobExtensionByMime[mime]; ok { + return byMime + } + return "bin" +} + +func isCleanExt(ext string) bool { + if len(ext) < 1 || len(ext) > 10 { + return false + } + for _, r := range ext { + if (r < 'a' || r > 'z') && (r < '0' || r > '9') { + return false + } + } + return true +} + +// checkIdSafe refuses an id that cannot serve as a filename stem inside the +// bundle root. Not a slugging step — ids are written verbatim (design §1.3) +// — just the containment guarantee. +func checkIdSafe(id string) error { + switch { + case id == "" || id == "." || id == "..": + return fmt.Errorf("id %q cannot name a file", id) + case strings.ContainsAny(id, "/\\\x00"): + return fmt.Errorf("id %q contains a path separator", id) + } + return nil +} diff --git a/pkg/lib/anyblockjson/compose/plan_test.go b/pkg/lib/anyblockjson/compose/plan_test.go new file mode 100644 index 0000000000..456fae0c2f --- /dev/null +++ b/pkg/lib/anyblockjson/compose/plan_test.go @@ -0,0 +1,157 @@ +package compose + +// plan_test.go pins the plan phase's whole value: a path is a pure +// per-document function of the id, fixed before the first emit task, with +// no collision machinery for the concurrent phase to disagree about +// (EXPORTER_DESIGN.md §1.1, §1.3). + +import ( + "strings" + "testing" + + "github.com/anyproto/anytype-heart/core/domain" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// Every kind lands in its design-§1.2 directory, named by its id verbatim +// plus the settled double extension; a file object additionally gets a blob +// path with the same stem, so document and blob sort adjacent. +// +// How this can fail: reintroduce the legacy store vocabulary (`relations`, +// `relationsOptions`) and the bundle's first impression breaks the format's +// own naming promise; derive the name from anything but the id and id→path +// stops being a pure function of a reference. +func TestBuildPlan_PathsAreAPureFunctionOfTheId(t *testing.T) { + // given + plan, err := BuildPlan("space1", []DocMeta{ + {Id: "bafypage", SbType: model.SmartBlockType_Page}, + {Id: "bafytype", SbType: model.SmartBlockType_STType}, + {Id: "bafytmpl", SbType: model.SmartBlockType_Template}, + {Id: "bafyrel", SbType: model.SmartBlockType_STRelation}, + {Id: "bafyopt", SbType: model.SmartBlockType_STRelationOption}, + {Id: "AAjEparticipant", SbType: model.SmartBlockType_Participant}, + {Id: "bafyfile", SbType: model.SmartBlockType_FileObject, FileExt: "png", FileMime: "image/png"}, + // the rare fail-closed widget document has no dedicated home + {Id: "bafywidget", SbType: model.SmartBlockType_Widget}, + }) + require.NoError(t, err) + + // then + want := map[string]string{ + "bafypage": "objects/bafypage.anyblock.json", + "bafytype": "types/bafytype.anyblock.json", + "bafytmpl": "templates/bafytmpl.anyblock.json", + "bafyrel": "properties/bafyrel.anyblock.json", + "bafyopt": "options/bafyopt.anyblock.json", + "AAjEparticipant": "participants/AAjEparticipant.anyblock.json", + "bafyfile": "files/bafyfile.anyblock.json", + "bafywidget": "objects/bafywidget.anyblock.json", + } + for id, path := range want { + got, ok := plan.DocPath(id) + require.True(t, ok, id) + assert.Equal(t, path, got) + } + blob, ok := plan.BlobPath("bafyfile") + require.True(t, ok) + assert.Equal(t, "files/bafyfile.png", blob, "same stem as the document, real extension") + _, ok = plan.BlobPath("bafypage") + assert.False(t, ok, "only file objects plan a blob") +} + +// The stored `fileExt` is dirty as a path component — measured on the +// corpus: 431 empty, 9 longer than 10 chars, dozens non-alphanumeric +// (`0-rc01`, `9-alpha`), 12 literally "json". The three-step rule: the +// extension when clean, the mime's conventional extension when not, `bin` +// when neither — and `anyblock.json` as a computed suffix is impossible by +// construction, so a blob can never impersonate a document. +// +// How this can fail: pass the extension through raw (a `9-alpha` blob name +// carries shrapnel and an empty one ends in a bare dot); consult the OS +// mime registry instead of the fixed table (the same space exports +// different bytes on different machines). +func TestBlobExtension_SanitizesTheMeasuredDirt(t *testing.T) { + cases := []struct{ ext, mime, want string }{ + {"png", "", "png"}, + {".PNG", "", "png"}, + {"", "image/jpeg", "jpg"}, + {"", "application/octet-stream", "bin"}, + {"0-rc01", "application/zip", "zip"}, + {"9-alpha", "", "bin"}, + {"json", "", "json"}, // a JSON blob keeps its extension; the double doc extension is the discriminator + {"averylongextension", "application/pdf", "pdf"}, + {"", "", "bin"}, + // parameterised media types are stripped at the ';' — the store + // holds `text/plain; charset=utf-8` on 150 corpus objects + {"", "text/plain; charset=utf-8", "txt"}, + // both icon spellings live spaces hold (172 + 30 measured) + {"", "image/x-icon", "ico"}, + {"", "image/vnd.microsoft.icon", "ico"}, + } + for _, c := range cases { + got := BlobExtension(c.ext, c.mime) + assert.Equal(t, c.want, got, "ext=%q mime=%q", c.ext, c.mime) + assert.False(t, strings.HasSuffix("x."+got, DocExtension), "a blob may never look like a document") + } +} + +// An id that cannot be a filename stem is refused up front — the +// containment guarantee. The corpus's two id populations can never trip +// this; a refusal means the store handed us something that is not an id. +func TestBuildPlan_RefusesAPathHostileId(t *testing.T) { + for _, id := range []string{"", ".", "..", "a/b", `a\b`} { + _, err := BuildPlan("space1", []DocMeta{{Id: id, SbType: model.SmartBlockType_Page}}) + assert.Error(t, err, "id %q", id) + } +} + +// A participant document's filename is its ENVELOPE id — the §9 fold of the +// store composite to the bare identity — never the store id: a reference +// carries the folded id, so only the folded stem keeps id→path a pure +// function of the reference, and the composite would claim a `_`-prefixed +// name in the platform's reserved namespace (§1). The foreign-space +// composite stays unfolded, exactly as Marshal keeps it as the envelope id +// — the plan and the envelope decline together. +// +// This was the native sweep's first real-data catch: the plan named the +// file by the store id while the document inside declared the folded one, +// and the very first exported space flagged its own member's document as +// stem_id_mismatch. +func TestBuildPlan_ParticipantStemIsTheFoldedIdentity(t *testing.T) { + // a real, checksum-valid identity (the fold classifies by decoding it) + const identity = "AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA" + const spaceId = "bafyreispace.lav62qbhdcf9" + own := domain.NewParticipantId(spaceId, identity) + foreign := domain.NewParticipantId("bafyreiother.zzz", identity) + + plan, err := BuildPlan(spaceId, []DocMeta{ + {Id: own, SbType: model.SmartBlockType_Participant}, + {Id: foreign, SbType: model.SmartBlockType_Participant}, + }) + require.NoError(t, err) + + got, ok := plan.DocPath(own) + require.True(t, ok, "the plan stays keyed by the STORE id the emit loop holds") + assert.Equal(t, "participants/"+identity+".anyblock.json", got) + + got, ok = plan.DocPath(foreign) + require.True(t, ok) + assert.Equal(t, "participants/"+foreign+".anyblock.json", got, + "a foreign-space composite stays unfolded, like its envelope id") +} + +// The blob-suffix invariant is CHECKED, not left as a property of today's +// id populations: a stem ending ".anyblock" plus a literal "json" +// extension is the one combination that would dress a blob as a document, +// and BuildPlan refuses it instead of writing it. +func TestBuildPlan_RefusesABlobWearingTheDocumentExtension(t *testing.T) { + _, err := BuildPlan("space1", []DocMeta{ + {Id: "evil.anyblock", SbType: model.SmartBlockType_FileObject, FileExt: "json"}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "document extension") +} diff --git a/pkg/lib/anyblockjson/compose/usedkeys.go b/pkg/lib/anyblockjson/compose/usedkeys.go new file mode 100644 index 0000000000..0aaf54ce24 --- /dev/null +++ b/pkg/lib/anyblockjson/compose/usedkeys.go @@ -0,0 +1,87 @@ +package compose + +// usedkeys.go — the used-property-key census, at byte level. The dictionary +// is used-only (§2f): it names every property the bundle's documents +// actually reference, so somebody has to read every emitted document and say +// which keys those are. The cmd tools did this by re-reading written files +// (cmd/internal/anyblockbatch.UsedPropertyKeys); production cannot — a zip +// export has no read path to its own entries before Close — so the scan runs +// on the marshalled bytes BEFORE the write, and this byte-level form is the +// single implementation both sides share (design §1.1). + +import ( + "encoding/json" + "fmt" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" +) + +// usedKeysDoc is the slice of a document the census reads: the legend, the +// properties object, and the §2a property definitions (which moved off the +// document root into `type_settings.property_definitions` — a +// scanner still reading the root would silently see no declarations). +type usedKeysDoc struct { + PropertyKeys map[string]string `json:"property_internal_keys"` + Properties map[string]json.RawMessage `json:"properties"` + TypeSettings *struct { + PropertyDefinitions []usedKeysDef `json:"property_definitions"` + } `json:"type_settings"` +} + +// usedKeysDef is one §2e entry's identity: its `property` spelling, else +// its `internal_key`. A name-only entry states no identity and is skipped — +// the codec derives the spelling from the name at import. +type usedKeysDef struct { + Property string `json:"property"` + InternalKey string `json:"internal_key"` +} + +// UsedPropertyKeysFromBytes reports every STORED property key one document +// references — its contribution to the population the dictionary's +// `properties` list names (§2f, used-only). Two slots count as a reference, +// resolved through the same chain every scan runs (the document's own +// `property_internal_keys` legend, then the bundled table, then verbatim): +// a `properties` member, and a +// `type_settings.property_definitions[].property`. A dataview's column list +// is deliberately NOT one — it is a per-view cache carrying its own inline +// format (§6.2), so a key that appears there and nowhere else gives a +// reader nothing to look up. +func UsedPropertyKeysFromBytes(doc []byte) (map[string]bool, error) { + var d usedKeysDoc + if err := json.Unmarshal(doc, &d); err != nil { + return nil, fmt.Errorf("parse document: %w", err) + } + out := map[string]bool{} + for k := range d.Properties { + // id and type are envelope facts, skipped on the SPELLING the way + // the codec skips them (importer.build) + if k == "id" || k == "type" { + continue + } + out[resolveUsedTerm(d.PropertyKeys, k)] = true + } + if d.TypeSettings != nil { + for _, def := range d.TypeSettings.PropertyDefinitions { + switch { + case def.Property != "": + out[resolveUsedTerm(d.PropertyKeys, def.Property)] = true + case def.InternalKey != "": + // a stated internal key IS the stored key and skips the + // ladder — a stored id is its own address (§2e) + out[def.InternalKey] = true + } + } + } + return out, nil +} + +// resolveUsedTerm binds one property term to the stored key it names, +// running the §3 chain: the document's own legend, then the bundled derived +// table, then verbatim (BundledKeyVocabulary's pass-through IS chain step 4). +func resolveUsedTerm(legend map[string]string, term string) string { + if key, ok := legend[term]; ok && key != "" { + return key + } + key, _ := anyblockjson.BundledKeyVocabulary{}.PropertyKey(term) + return key +} diff --git a/pkg/lib/anyblockjson/compose/usedkeys_test.go b/pkg/lib/anyblockjson/compose/usedkeys_test.go new file mode 100644 index 0000000000..8fa74cbd30 --- /dev/null +++ b/pkg/lib/anyblockjson/compose/usedkeys_test.go @@ -0,0 +1,56 @@ +package compose + +// usedkeys_test.go pins the byte-level used-key census against the chain +// the codec itself runs (§3): legend first, bundled table second, verbatim +// last. It is the shared implementation behind +// cmd/internal/anyblockbatch.UsedPropertyKeys, whose own drift-pin test +// (TestLintResolvesPropertyTermsLikeTheCodec) covers the codec agreement; +// this one covers the slots. + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// How this can fail: read `recommended*` off the document root instead of +// `type_settings.property_definitions` (post-v0.32 declarations vanish from +// the census and the dictionary silently shrinks); stop skipping id/type +// (two envelope facts join every dictionary); resolve an `internal_key` +// through the slug ladder (a stored id that happens to fold onto a bundled +// key gets rewritten). +func TestUsedPropertyKeysFromBytes(t *testing.T) { + doc := []byte(`{ + "version": 2, + "id": "bafyx", + "type": "task", + "property_internal_keys": {"aroma": "6a32d4856761631534b22f85"}, + "properties": { + "aroma": "smoky", + "due_date": "2026-01-01", + "custom_verbatim": 3 + }, + "type_settings": { + "property_definitions": [ + {"property": "due_date"}, + {"internal_key": "64f2d485676163153aaaaaaa", "name": "Team"}, + {"name": "name-only entries state no identity"} + ] + } + }`) + + used, err := UsedPropertyKeysFromBytes(doc) + require.NoError(t, err) + + assert.True(t, used["6a32d4856761631534b22f85"], "the legend resolves the spelling (chain step 1)") + assert.True(t, used["dueDate"], "the bundled table resolves the slug (chain step 2)") + assert.True(t, used["custom_verbatim"], "an unresolvable spelling passes through verbatim (chain step 4)") + assert.True(t, used["64f2d485676163153aaaaaaa"], "a stated internal_key is its own address") + assert.False(t, used["id"], "envelope facts are not property references") + assert.False(t, used["type"], "envelope facts are not property references") + assert.Len(t, used, 4) + + _, err = UsedPropertyKeysFromBytes([]byte("not json")) + assert.Error(t, err) +} diff --git a/pkg/lib/anyblockjson/dataview.go b/pkg/lib/anyblockjson/dataview.go new file mode 100644 index 0000000000..72a9bc03f1 --- /dev/null +++ b/pkg/lib/anyblockjson/dataview.go @@ -0,0 +1,665 @@ +package anyblockjson + +// dataview.go maps Content.Dataview to the §6.2 JSON form and back: cleaned +// names, lowerCamel enums, defaults omitted, filter trees with implicit +// top-level AND, and select values as option names. + +import ( + "encoding/json" + "fmt" + "sort" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// dvFormat resolves a property key's format from the dataview's live +// relationLinks first, then bundle/resolver (§6.2). +func (e *exporter) dvFormat(dv *model.BlockContentDataview, key string) (model.RelationFormat, bool) { + for _, rl := range dv.RelationLinks { + if rl != nil && rl.Key == key { + return rl.Format, true + } + } + return e.resolveFormat(key) +} + +func (e *exporter) dataviewToJSON(m *omap, dv *model.BlockContentDataview) error { + m.set("type", "dataview") + // a singular reference slot: a target the space does not hold is + // written as the sentinel, never as if it existed (§9) + m.setNonEmpty("object_id", e.singularObjectRef("/blocks", "dataview object_id", dv.TargetObjectId)) + m.setNonEmpty("is_collection", dv.IsCollection) + m.setNonEmpty("source", stringsToAny(dv.Source)) + + var props []any + for _, rl := range dv.RelationLinks { + if rl == nil || rl.Key == "" { + continue + } + // an unwritable stored key has no spelling at any slot (§3); + // slotPropertySlug warned and the entry is dropped like a nameless one + slug := e.slotPropertySlug(rl.Key, "a dataview `properties` entry") + if slug == "" { + continue + } + pm := &omap{} + pm.set(memberProperty, slug) + pm.setNonEmpty("format", formatName(rl.Format)) + props = append(props, pm) + } + m.setNonEmpty("properties", props) + + var views []any + for _, v := range dv.Views { + if v == nil { + continue + } + vm, err := e.viewToJSON(v, dv) + if err != nil { + return err + } + views = append(views, vm) + } + m.setNonEmpty("views", views) + // activeView and the deprecated relations field are dropped (§6.2) + return nil +} + +func (e *exporter) viewToJSON(v *model.BlockContentDataviewView, dv *model.BlockContentDataview) (*omap, error) { + vm := &omap{} + e.recordEmitted(v.Id) + if !e.opts.OmitIds { + vm.setNonEmpty("id", e.localId(v.Id)) + } + if v.Type != model.BlockContentDataviewView_Table { + // an out-of-range view type is omitted rather than emitted as an + // empty string, which the schema would reject; it therefore reads + // back as table. The only value this can be today is the client's + // experimental Timeline (ViewType.Timeline = 6, gated behind + // config.experimental and absent from the proto enum), so the loss + // is confined to a view the protocol cannot describe anyway. + vm.setNonEmpty("type", viewTypeNames.name(v.Type)) + } + vm.setNonEmpty("name", v.Name) + // the three singular key slots share the reference-slot drop rule: an + // unwritable stored key is omitted with a warning (slotPropertySlug) + vm.setNonEmpty("group_by", e.slotPropertySlug(v.GroupRelationKey, "a view's `group_by`")) + vm.setNonEmpty("cover_property", e.slotPropertySlug(v.CoverRelationKey, "a view's `cover_property`")) + vm.setNonEmpty("end_property", e.slotPropertySlug(v.EndRelationKey, "a view's `end_property`")) + vm.setNonEmpty("hide_icon", v.HideIcon) + if v.CardSize != model.BlockContentDataviewView_Small { + vm.setNonEmpty("card_size", cardSizeNames.name(v.CardSize)) + } + vm.setNonEmpty("cover_fit", v.CoverFit) + vm.setNonEmpty("colored_groups", v.GroupBackgroundColors) + vm.setNonEmpty("page_size", v.PageLimit) + vm.setNonEmpty("default_template_id", v.DefaultTemplateId) + vm.setNonEmpty("default_type_id", v.DefaultObjectTypeId) + vm.setNonEmpty("wrap_content", v.WrapContent) + if v.ListSize != model.BlockContentDataviewView_Compact { + vm.setNonEmpty("list_size", listSizeNames.name(v.ListSize)) + } + vm.setNonEmpty("alternate_rows", v.AlternateRows) + + var sorts []any + for _, s := range v.Sorts { + // a sort without a property key is junk and would fail the schema + if s == nil || s.RelationKey == "" { + continue + } + if sm := e.sortToJSON(s, dv); sm != nil { + sorts = append(sorts, sm) + } + } + vm.setNonEmpty("sorts", sorts) + + var filters []any + for _, f := range v.Filters { + if f == nil { + continue + } + if fm := e.filterToJSON(f, dv); fm != nil { + filters = append(filters, fm) + } + } + vm.setNonEmpty("filters", filters) + + var columns []any + for _, r := range v.Relations { + if r == nil || r.Key == "" { + continue + } + if cm := e.viewColumnToJSON(r); cm != nil { + columns = append(columns, cm) + } + } + vm.setNonEmpty("columns", columns) + + if !e.opts.OmitIds { + vm.setNonEmpty("groups", e.viewGroupsToJSON(v.Id, dv)) + vm.setNonEmpty("object_orders", e.objectOrdersToJSON(v.Id, dv)) + } + return vm, nil +} + +// viewGroupsToJSON emits the kanban group display order: array order, the +// proto's per-group index derived from it (§6.2). +func (e *exporter) viewGroupsToJSON(viewId string, dv *model.BlockContentDataview) []any { + var out []any + for _, groupOrder := range dv.GroupOrders { + if groupOrder == nil || groupOrder.ViewId != viewId { + continue + } + groups := make([]*model.BlockContentDataviewViewGroup, 0, len(groupOrder.ViewGroups)) + for _, g := range groupOrder.ViewGroups { + if g != nil { + groups = append(groups, g) + } + } + sort.SliceStable(groups, func(i, j int) bool { return groups[i].Index < groups[j].Index }) + for _, g := range groups { + gm := &omap{} + gm.setNonEmpty("id", g.GroupId) + gm.setNonEmpty("hidden", g.Hidden) + gm.setNonEmpty("background_color", g.BackgroundColor) + out = append(out, gm) + } + } + return out +} + +func (e *exporter) objectOrdersToJSON(viewId string, dv *model.BlockContentDataview) []any { + var out []any + for _, oo := range dv.ObjectOrders { + if oo == nil || oo.ViewId != viewId { + continue + } + om := &omap{} + om.setNonEmpty("group_id", oo.GroupId) + var ids []any + for _, id := range oo.ObjectIds { + if id != "" { + ids = append(ids, e.objectRef(id)) + } + } + om.setNonEmpty("object_ids", ids) + out = append(out, om) + } + return out +} + +func (e *exporter) sortToJSON(s *model.BlockContentDataviewSort, dv *model.BlockContentDataview) *omap { + // an unwritable stored key drops the sort, warned, like the nameless one + // the caller already skips (§3) + slug := e.slotPropertySlug(s.RelationKey, "a sort") + if slug == "" { + return nil + } + sm := &omap{} + sm.setNonEmpty(memberProperty, slug) + if s.Type != model.BlockContentDataviewSort_Asc { + sm.setNonEmpty("direction", sortDirectionNames.name(s.Type)) + } + if len(s.CustomOrder) > 0 { + order := make([]any, 0, len(s.CustomOrder)) + for _, cv := range s.CustomOrder { + order = append(order, e.dvValueToJSON(dv, s.RelationKey, cv)) + } + sm.set("custom_order", order) + } + if s.EmptyPlacement != model.BlockContentDataviewSort_NotSpecified { + sm.setNonEmpty("empty_placement", emptyPlacementNames.name(s.EmptyPlacement)) + } + sm.setNonEmpty("include_time", s.IncludeTime) + sm.setNonEmpty("no_collate", s.NoCollate) + if !e.opts.OmitIds { + sm.setNonEmpty("id", s.Id) + } + // the cached per-node format is dropped; import rehydrates it (§6.2) + return sm +} + +func (e *exporter) filterToJSON(f *model.BlockContentDataviewFilter, dv *model.BlockContentDataview) *omap { + fm := &omap{} + if len(f.NestedFilters) > 0 { + // a proto node with nested filters maps to a group; leaf fields drop + op := "and" + if f.Operator == model.BlockContentDataviewFilter_Or { + op = "or" + } + var nested []any + for _, nf := range f.NestedFilters { + if nf == nil { + continue + } + if nm := e.filterToJSON(nf, dv); nm != nil { + nested = append(nested, nm) + } + } + if len(nested) == 0 { + return nil // a group with no live children is a no-op + } + fm.set("operator", op) + fm.set("filters", nested) + return fm + } + // a leaf filter has to name the property it filters on (§6) — the rule + // the sort and the column beside it have carried all along. Without it a + // filter whose stored relation key is empty was emitted as a nameless + // node: it filtered on nothing, the schema accepted it, import stored the + // empty key, and the next export wrote the same node again. Dropping it + // is what the sort loop above already does with the same input. + if f.RelationKey == "" { + e.warn("", "a filter names no property and is dropped; a filter has to name "+ + "the property it filters on") + return nil + } + // an unwritable stored key drops the filter the same way — warned by + // slotPropertySlug — instead of being emitted verbatim (§3) + slug := e.slotPropertySlug(f.RelationKey, "a filter") + if slug == "" { + return nil + } + fm.setNonEmpty(memberProperty, slug) + if f.Condition != model.BlockContentDataviewFilter_None { + fm.setNonEmpty("condition", conditionNames.name(f.Condition)) + } + switch f.Condition { + case model.BlockContentDataviewFilter_Empty, + model.BlockContentDataviewFilter_NotEmpty, + model.BlockContentDataviewFilter_Exists: + // value is dropped on presence-only conditions (§11) + default: + // the day-count presets take their operand from value (§6.2), so a + // zero count is meaningful data rather than an absent field: it must + // survive the usual empty-elision, or the document silently stops + // saying which day it means + if countingPreset(f.QuickOption) { + fm.set("value", e.dayCountOperand(f)) + } else if f.Value != nil { + // a filter's value is DATA, and a falsy one is the most ordinary + // data there is: `done = false` is how every task view spells + // "not finished yet", and `priority = 0` how a number view + // spells "unset". Eliding them leaves `done equal ` — a + // different query, in a document that still validates, and one + // the round trip cannot notice because both generations lose it + // identically. Measured over 38,061 production objects: 151 of + // the 1,494 filters carrying a value carry a falsy one, 122 of + // those on `done`, across 70 documents. + // + // This is the same trap the counting preset above states and + // fixes; the fix was never generalized to the branch beside it. + if v := e.dvValueToJSON(dv, f.RelationKey, f.Value); v != nil { + fm.set("value", v) + } + } + } + if f.QuickOption != model.BlockContentDataviewFilter_ExactDate { + fm.setNonEmpty("date_preset", datePresetNames.name(f.QuickOption)) + } + fm.setNonEmpty("include_time", f.IncludeTime) + fm.setNonEmpty("nested_property", f.RelationProperty) + if !e.opts.OmitIds { + fm.setNonEmpty("id", f.Id) + } + // a contentless leaf (at most an id) is a no-op node: drop it + if len(fm.keys) == 0 || (len(fm.keys) == 1 && fm.keys[0] == "id") { + return nil + } + return fm +} + +// dayCountOperand renders a counting preset's operand (§6.2): the whole day +// count in [0, maxDayCount] the format admits, which is also what the query +// engine reads out of the stored value — domain.Value.Int64 is int64(float) +// for a number and 0 for every other kind, so a null, a string or a list all +// mean "0 days, i.e. today" to the engine already. +// +// The stored value is untrusted like everything else in a snapshot, and the +// slot has exactly one written form, so the junk cannot travel: writing it +// verbatim hands back a document this package's own Validate refuses (§11, +// I1), and dropping the member says "today" while claiming nothing. Writing +// the count the engine reads says what the filter does. A count outside the +// bound is the one case where the two part ways — the engine would keep +// counting and the format cannot spell it — so it is pinned to the bound and +// reported. +func (e *exporter) dayCountOperand(f *model.BlockContentDataviewFilter) float64 { + if f.Value == nil { + // no operand at all is not a fault to report: it is the stored shape + // of "today", and 0 is how this format spells it + return 0 + } + raw := f.Value.GetNumberValue() // 0 for every non-number kind + count := int64(raw) + bounded := count + switch { + case bounded < 0: + bounded = 0 + case bounded > maxDayCount: + bounded = maxDayCount + } + if _, isNumber := f.Value.GetKind().(*types.Value_NumberValue); !isNumber || float64(count) != raw || count != bounded { + e.warn("", "the %s filter on %q carries %v as its day count, which is not one this format can write "+ + "(a whole number between 0 and %d); %d is written instead", + datePresetNames.name(f.QuickOption), f.RelationKey, protoValueToJSON(f.Value), maxDayCount, bounded) + } + return float64(bounded) +} + +// dvValueToJSON converts a filter value or custom-order entry: option names +// for select properties (§3), object references through the §9 reference +// renderer (full id, plus the informative `#name` suffix where the shape +// asks for it), verbatim otherwise. +func (e *exporter) dvValueToJSON(dv *model.BlockContentDataview, key string, v *types.Value) any { + format, ok := e.dvFormat(dv, key) + if ok { + switch format { + case model.RelationFormat_status, model.RelationFormat_tag: + return e.mapValueStrings(v, func(id string) string { return e.optionName(key, id) }) + case model.RelationFormat_object, model.RelationFormat_file: + return e.mapValueStrings(v, e.objectRef) + } + } + return protoValueToJSON(v) +} + +// mapValueStrings applies fn to a string value or each element of a string +// list, keeping the single/list shape. +func (e *exporter) mapValueStrings(v *types.Value, fn func(string) string) any { + if s, ok := v.GetKind().(*types.Value_StringValue); ok { + return fn(s.StringValue) + } + if l, ok := v.GetKind().(*types.Value_ListValue); ok { + out := make([]any, 0, len(l.ListValue.Values)) + for _, el := range l.ListValue.Values { + if s := el.GetStringValue(); s != "" { + out = append(out, fn(s)) + } else { + out = append(out, protoValueToJSON(el)) + } + } + return out + } + return protoValueToJSON(v) +} + +// +// ---- import ---- +// + +type jsonDvProperty struct { + Property string `json:"property"` + Format string `json:"format"` +} + +type jsonView struct { + Id string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + GroupBy string `json:"group_by"` + CoverProperty string `json:"cover_property"` + EndProperty string `json:"end_property"` + HideIcon bool `json:"hide_icon"` + CardSize string `json:"card_size"` + CoverFit bool `json:"cover_fit"` + ColoredGroups bool `json:"colored_groups"` + PageSize json.Number `json:"page_size"` + DefaultTemplateId string `json:"default_template_id"` + DefaultTypeId string `json:"default_type_id"` + WrapContent bool `json:"wrap_content"` + ListSize string `json:"list_size"` + AlternateRows bool `json:"alternate_rows"` + Sorts []jsonSort `json:"sorts"` + Filters []jsonFilter `json:"filters"` + Columns []jsonViewColumn `json:"columns"` + Groups []jsonViewGroup `json:"groups"` + ObjectOrders []jsonObjectOrder `json:"object_orders"` +} + +type jsonSort struct { + Property string `json:"property"` + Direction string `json:"direction"` + CustomOrder []any `json:"custom_order"` + EmptyPlacement string `json:"empty_placement"` + IncludeTime bool `json:"include_time"` + NoCollate bool `json:"no_collate"` + Id string `json:"id"` +} + +type jsonFilter struct { + Operator string `json:"operator"` + Filters []jsonFilter `json:"filters"` + + Property string `json:"property"` + Condition string `json:"condition"` + Value any `json:"value"` + DatePreset string `json:"date_preset"` + IncludeTime bool `json:"include_time"` + NestedProperty string `json:"nested_property"` + Id string `json:"id"` +} + +type jsonViewColumn struct { + Property string `json:"property"` + Hidden bool `json:"hidden"` + // pixels, stored as an int32 (§6.2) — so json.Number, bounded by the + // schema to int32 range, rather than a float64 that would truncate + Width json.Number `json:"width"` + Aggregation string `json:"aggregation"` + Align string `json:"align"` +} + +type jsonViewGroup struct { + Id string `json:"id"` + Hidden bool `json:"hidden"` + BackgroundColor string `json:"background_color"` +} + +type jsonObjectOrder struct { + GroupId string `json:"group_id"` + ObjectIds []string `json:"object_ids"` +} + +func (imp *importer) dataviewFromJSON(jb *jsonBlock) (*model.BlockContentDataview, error) { + dv := &model.BlockContentDataview{ + TargetObjectId: imp.objectRef(jb.ObjectId), + IsCollection: jb.IsCollection, + Source: jb.Source, + } + var props []jsonDvProperty + if len(jb.Properties) > 0 { + if err := jsonUnmarshal(jb.Properties, &props); err != nil { + return nil, fmt.Errorf("dataview properties: %w", err) + } + } + for _, p := range props { + key := imp.propertyKeyAt(p.Property, "dataview `properties`") + dv.RelationLinks = append(dv.RelationLinks, &model.RelationLink{ + Key: key, + Format: imp.declaredFormat(key, p.Format), + }) + } + for _, jv := range jb.Views { + viewId := jv.Id + if viewId == "" { + viewId = imp.genId() + } + view := &model.BlockContentDataviewView{ + Id: viewId, + Type: viewTypeNames.value(jv.Type), + Name: jv.Name, + GroupRelationKey: imp.propertyKeyAt(jv.GroupBy, "view `group_by`"), + CoverRelationKey: imp.propertyKeyAt(jv.CoverProperty, "view `cover_property`"), + EndRelationKey: imp.propertyKeyAt(jv.EndProperty, "view `end_property`"), + HideIcon: jv.HideIcon, + CardSize: cardSizeNames.value(jv.CardSize), + CoverFit: jv.CoverFit, + GroupBackgroundColors: jv.ColoredGroups, + PageLimit: jsonInt32(jv.PageSize), + DefaultTemplateId: jv.DefaultTemplateId, + DefaultObjectTypeId: jv.DefaultTypeId, + WrapContent: jv.WrapContent, + ListSize: listSizeNames.value(jv.ListSize), + AlternateRows: jv.AlternateRows, + } + for _, js := range jv.Sorts { + view.Sorts = append(view.Sorts, imp.sortFromJSON(js, dv)) + } + for _, jf := range jv.Filters { + view.Filters = append(view.Filters, imp.filterFromJSON(jf, dv)) + } + for _, jc := range jv.Columns { + view.Relations = append(view.Relations, &model.BlockContentDataviewRelation{ + Key: imp.propertyKeyAt(jc.Property, "view column `property`"), + IsVisible: !jc.Hidden, + Width: jsonInt32(jc.Width), + Formula: aggregationNames.value(jc.Aggregation), + Align: alignNames.value(jc.Align), + }) + } + if len(jv.Groups) > 0 { + groupOrder := &model.BlockContentDataviewGroupOrder{ViewId: viewId} + for i, jg := range jv.Groups { + groupOrder.ViewGroups = append(groupOrder.ViewGroups, &model.BlockContentDataviewViewGroup{ + GroupId: jg.Id, + Index: int32(i), // derived from array order (§6.2) + Hidden: jg.Hidden, + BackgroundColor: jg.BackgroundColor, + }) + } + dv.GroupOrders = append(dv.GroupOrders, groupOrder) + } + for _, jo := range jv.ObjectOrders { + oo := &model.BlockContentDataviewObjectOrder{ViewId: viewId, GroupId: jo.GroupId} + for _, id := range jo.ObjectIds { + oo.ObjectIds = append(oo.ObjectIds, imp.objectRef(id)) + } + dv.ObjectOrders = append(dv.ObjectOrders, oo) + } + dv.Views = append(dv.Views, view) + } + return dv, nil +} + +// impDvFormat rehydrates the cached per-node format from the dataview's +// properties list and bundle; unresolvable keys get format 0 (§6.2). +func (imp *importer) impDvFormat(dv *model.BlockContentDataview, key string) model.RelationFormat { + for _, rl := range dv.RelationLinks { + if rl != nil && rl.Key == key { + return rl.Format + } + } + if f, ok := imp.resolveFormat(key); ok { + return f + } + return 0 +} + +func (imp *importer) sortFromJSON(js jsonSort, dv *model.BlockContentDataview) *model.BlockContentDataviewSort { + key := imp.propertyKeyAt(js.Property, "sort `property`") + s := &model.BlockContentDataviewSort{ + RelationKey: key, + Type: sortDirectionNames.value(js.Direction), + Format: imp.impDvFormat(dv, key), + IncludeTime: js.IncludeTime, + Id: js.Id, + EmptyPlacement: emptyPlacementNames.value(js.EmptyPlacement), + NoCollate: js.NoCollate, + } + for _, entry := range js.CustomOrder { + s.CustomOrder = append(s.CustomOrder, imp.dvValueFromJSON(dv, key, js.Property, entry)) + } + return s +} + +func (imp *importer) filterFromJSON(jf jsonFilter, dv *model.BlockContentDataview) *model.BlockContentDataviewFilter { + if jf.Operator != "" { + f := &model.BlockContentDataviewFilter{ + Operator: model.BlockContentDataviewFilter_And, + } + if jf.Operator == "or" { + f.Operator = model.BlockContentDataviewFilter_Or + } + for _, nf := range jf.Filters { + f.NestedFilters = append(f.NestedFilters, imp.filterFromJSON(nf, dv)) + } + return f + } + key := imp.propertyKeyAt(jf.Property, "filter `property`") + f := &model.BlockContentDataviewFilter{ + Id: jf.Id, + RelationKey: key, + RelationProperty: jf.NestedProperty, + Condition: conditionNames.value(jf.Condition), + QuickOption: datePresetNames.value(jf.DatePreset), + Format: imp.impDvFormat(dv, key), + IncludeTime: jf.IncludeTime, + } + if jf.Value != nil { + f.Value = imp.dvValueFromJSON(dv, key, jf.Property, jf.Value) + } + return f +} + +// dvValueFromJSON reverses dvValueToJSON: option names back to ids where a +// resolver knows them, object references through the §9 reference reader +// (the informative `#name` suffix trimmed unread), everything else verbatim +// (§3, §9a). +// +// The objects/files arm is BACK, and it is not the one the deleted `refs` +// legend had (§9a): that one inverted an indirection table, this one strips +// a suffix that was never an address. A bare id passes through it unchanged, +// so a document written without suffixes imports exactly as it did before +// the arm existed. +func (imp *importer) dvValueFromJSON(dv *model.BlockContentDataview, key, slug string, v any) *types.Value { + format := imp.impDvFormat(dv, key) + switch format { + case model.RelationFormat_status, model.RelationFormat_tag: + return mapJSONStrings(v, func(name string) string { return imp.resolveOption(key, slug, name) }) + case model.RelationFormat_object, model.RelationFormat_file: + return mapJSONStrings(v, imp.objectRef) + } + return jsonToProtoValue(v) +} + +func mapJSONStrings(v any, fn func(string) string) *types.Value { + switch x := v.(type) { + case string: + return &types.Value{Kind: &types.Value_StringValue{StringValue: fn(x)}} + case []any: + vals := make([]*types.Value, 0, len(x)) + for _, el := range x { + if s, ok := el.(string); ok { + vals = append(vals, &types.Value{Kind: &types.Value_StringValue{StringValue: fn(s)}}) + } else { + vals = append(vals, jsonToProtoValue(el)) + } + } + return &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: vals}}} + } + return jsonToProtoValue(v) +} + +func (e *exporter) viewColumnToJSON(r *model.BlockContentDataviewRelation) *omap { + // an unwritable stored key drops the column, warned, like the nameless + // one the caller already skips (§3) + slug := e.slotPropertySlug(r.Key, "a view column") + if slug == "" { + return nil + } + cm := &omap{} + cm.set(memberProperty, slug) + // hidden is the inverse of proto isVisible; omitted means visible (§6.2) + cm.setNonEmpty("hidden", !r.IsVisible) + cm.setNonEmpty("width", r.Width) + if r.Formula != model.BlockContentDataviewRelation_None { + cm.setNonEmpty("aggregation", aggregationNames.name(r.Formula)) + } + if r.Align != model.Block_AlignLeft { + cm.setNonEmpty("align", alignNames.name(r.Align)) + } + // deprecated per-column date/time fields are dropped (§6.2) + return cm +} diff --git a/pkg/lib/anyblockjson/dataview_test.go b/pkg/lib/anyblockjson/dataview_test.go new file mode 100644 index 0000000000..1b11d6478b --- /dev/null +++ b/pkg/lib/anyblockjson/dataview_test.go @@ -0,0 +1,134 @@ +package anyblockjson + +// dataview_test.go — the dataview block's own round-trip rules (§6.2). + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// A filter's value is data, and a falsy one is the most ordinary data there +// is: `done = false` is how a task view spells "not finished yet". Eliding +// it leaves `done equal ` — a different query, in a document that +// still validates, and one no round-trip check can notice, because both +// generations lose it identically. +// +// Measured over 38,061 production objects when this was found: 151 of the +// 1,494 filters carrying a value carried a falsy one — 122 on `done` — in +// 70 documents. +// +// How this can fail: put setNonEmpty back at dataview.go's filter value and +// the false/0/"" rows come back nil. +func TestDataview_AFalsyFilterValueIsStillTheQuery(t *testing.T) { + for name, tc := range map[string]struct { + stored *types.Value + want string + }{ + "false is a checkbox filter": {boolVal(false), `"value": false`}, + "true still survives": {boolVal(true), `"value": true`}, + "zero is a number filter": {num(0), `"value": 0`}, + "one still survives": {num(1), `"value": 1`}, + "the empty string is a filter": {str(""), `"value": ""`}, + } { + t.Run(name, func(t *testing.T) { + // given + snap := filterValueSnapshot(tc.stored) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + _, back, err := Unmarshal(data, testOptions()) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), tc.want) + require.NotNil(t, back.GetBlocks(), "the filter value survives the round trip") + var got *types.Value + for _, b := range back.GetBlocks() { + if dv := b.GetDataview(); dv != nil && len(dv.GetViews()) > 0 && len(dv.GetViews()[0].GetFilters()) > 0 { + got = dv.GetViews()[0].GetFilters()[0].GetValue() + } + } + require.NotNil(t, got, "the value came back nil — the query changed meaning") + assert.Equal(t, tc.stored.String(), got.String()) + }) + } +} + +func boolVal(b bool) *types.Value { + return &types.Value{Kind: &types.Value_BoolValue{BoolValue: b}} +} + +func filterValueSnapshot(v *types.Value) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "bafyreifiltroot", ChildrenIds: []string{"dv"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dv", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{{ + Id: "view1", Name: "All", + Filters: []*model.BlockContentDataviewFilter{{ + Id: "f1", + RelationKey: "done", + Condition: model.BlockContentDataviewFilter_Equal, + Value: v, + }}, + }}, + }}}, + }, + Details: fields(map[string]*types.Value{"id": str("bafyreifiltroot")}), + } +} + +// An ABSENT format says the document did not speak; the chain answers (§3). +// It is not a declaration of text — which is what it used to mean, and that +// silently overrode the bundled table: listing a bundled DATE property +// without its format pinned it to longtext, so the view's filters stopped +// being dates. Omitting the whole properties list resolved correctly, which +// made naming a property strictly worse than staying silent about it. +// +// Canonical export always writes a format, so absence only ever arrives +// from a hand-written document. +// +// How this can fail: let declaredFormatWith fall through to longtext for an +// empty name again and the middle case reads back as text. +func TestDataview_AnAbsentFormatResolvesThroughTheChain(t *testing.T) { + const filter = `"views":[{"name":"All","filters":[{"property":"due_date","condition":"greater","value":"2026-01-01T00:00:00Z"}]}]` + for name, tc := range map[string]struct { + props string + want model.RelationFormat + }{ + "declared date": {`"properties":[{"property":"due_date","format":"date"}],`, model.RelationFormat_date}, + "format omitted": {`"properties":[{"property":"due_date"}],`, model.RelationFormat_date}, + "no properties list": {``, model.RelationFormat_date}, + "declared text stands": {`"properties":[{"property":"due_date","format":"text"}],`, model.RelationFormat_longtext}, + } { + t.Run(name, func(t *testing.T) { + // given + doc := `{"version":2,"blocks":[{"type":"dataview",` + tc.props + filter + `}]}` + + // when + _, snap, err := Unmarshal([]byte(doc), testOptions()) + require.NoError(t, err) + + // then + var got model.RelationFormat = -1 + for _, b := range snap.GetBlocks() { + if dv := b.GetDataview(); dv != nil { + for _, v := range dv.GetViews() { + for _, f := range v.GetFilters() { + got = f.GetFormat() + } + } + } + } + assert.Equal(t, tc.want, got, "naming a property must never be worse than staying silent") + }) + } +} diff --git a/pkg/lib/anyblockjson/dataviewid_test.go b/pkg/lib/anyblockjson/dataviewid_test.go new file mode 100644 index 0000000000..fb0b41250d --- /dev/null +++ b/pkg/lib/anyblockjson/dataviewid_test.go @@ -0,0 +1,141 @@ +package anyblockjson + +// The primary dataview keeps the editor's fixed block id (§7). Without it the +// editor's WithDataviewIDIfNotExists finds no "dataview" block and adds a +// second, empty one next to the configured one. + +import ( + "fmt" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// blockIds returns every non-root block id in document order. +func blockIds(t *testing.T, snap *model.SmartBlockSnapshotBase, rootId string) []string { + t.Helper() + ids := make([]string, 0, len(snap.Blocks)) + for _, b := range snap.Blocks { + if b.Id != rootId { + ids = append(ids, b.Id) + } + } + return ids +} + +func importDoc(t *testing.T, doc string) *model.SmartBlockSnapshotBase { + t.Helper() + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + return snap +} + +func TestImport_PrimaryDataviewGetsFixedId(t *testing.T) { + t.Run("type document", func(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "wikiCategory", + "blocks": [{"type": "dataview", "views": [{"name": "All"}]}]}` + snap := importDoc(t, doc) + assert.Equal(t, []string{"dataview"}, blockIds(t, snap, "t1")) + }) + + // sets and collections are kind:page — the convention is not type-specific, + // so the rule must not key on kind. + t.Run("collection document", func(t *testing.T) { + doc := `{"version": 2, "id": "c1", "type": "collection", + "blocks": [{"type": "dataview", "is_collection": true, "views": [{"name": "All"}]}]}` + snap := importDoc(t, doc) + assert.Equal(t, []string{"dataview"}, blockIds(t, snap, "c1")) + }) + + // objectId means the block views *another* set: an inline dataview, which + // must keep a generated id or it would shadow the object's own. + t.Run("inline view keeps generated id", func(t *testing.T) { + doc := `{"version": 2, "id": "p1", + "blocks": [{"type": "dataview", "object_id": "otherSet", "views": [{"name": "All"}]}]}` + snap := importDoc(t, doc) + assert.Equal(t, []string{"g1"}, blockIds(t, snap, "p1")) + }) + + t.Run("nested dataview keeps generated id", func(t *testing.T) { + doc := `{"version": 2, "id": "p1", "blocks": [ + {"type": "callout", "text": "wrapper"}, + {"type": "dataview", "indent": 1, "views": [{"name": "All"}]}]}` + snap := importDoc(t, doc) + assert.Equal(t, []string{"g1", "g2"}, blockIds(t, snap, "p1")) + }) + + t.Run("only the first is pinned", func(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", "blocks": [ + {"type": "dataview", "views": [{"name": "A"}]}, + {"type": "dataview", "views": [{"name": "B"}]}]}` + snap := importDoc(t, doc) + ids := blockIds(t, snap, "t1") + require.Len(t, ids, 2) + assert.Equal(t, "dataview", ids[0]) + assert.NotEqual(t, "dataview", ids[1]) + }) + + // an explicit id stays authoritative; pinning must not mint a duplicate. + t.Run("explicit claim wins", func(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", "blocks": [ + {"type": "dataview", "views": [{"name": "A"}]}, + {"type": "dataview", "id": "dataview", "views": [{"name": "B"}]}]}` + snap := importDoc(t, doc) + ids := blockIds(t, snap, "t1") + require.Len(t, ids, 2) + assert.NotEqual(t, "dataview", ids[0]) + assert.Equal(t, "dataview", ids[1]) + }) +} + +// omitIds used to break every dataview-backed object: the export dropped the +// fixed id and the re-import could not put it back. +func TestRoundtrip_OmitIdsKeepsPrimaryDataview(t *testing.T) { + snapshot := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "t1", ChildrenIds: []string{"dataview"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dataview", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{{ + Id: "v1", + Name: "Section directory", + Relations: []*model.BlockContentDataviewRelation{ + {Key: "name", IsVisible: true, Width: 180}, + }, + }}, + }}}, + }, + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "t1"}}, + }}, + Key: "wikiCategory", + } + + for _, omit := range []bool{false, true} { + t.Run(fmt.Sprintf("omitIds=%v", omit), func(t *testing.T) { + opts := testOptions() + opts.OmitIds = omit + data, err := Marshal(model.SmartBlockType_STType, snapshot, opts) + require.NoError(t, err) + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + var dv *model.Block + for _, b := range back.Blocks { + if b.GetDataview() != nil { + dv = b + } + } + require.NotNil(t, dv, "dataview block survived") + assert.Equal(t, "dataview", dv.Id) + require.Len(t, dv.GetDataview().Views, 1) + require.Len(t, dv.GetDataview().Views[0].Relations, 1) + assert.Equal(t, int32(180), dv.GetDataview().Views[0].Relations[0].Width) + }) + } +} diff --git a/pkg/lib/anyblockjson/datefilter_test.go b/pkg/lib/anyblockjson/datefilter_test.go new file mode 100644 index 0000000000..79bf657cb9 --- /dev/null +++ b/pkg/lib/anyblockjson/datefilter_test.go @@ -0,0 +1,554 @@ +package anyblockjson + +// `less` on a date matches objects that have no value for it: the filter's +// value is set and the record's is not, so domain.Value.Compare returns 1 — +// precisely what Less tests for (database/filter.go). A freshness view +// written the obvious way therefore lists every never-dated object as +// overdue. + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/gogo/protobuf/types" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func dateFilterDoc(filters string) string { + return `{"version": 2, "id": "p1", "blocks": [{"type": "dataview", + "object_id": "someSet", + "properties": [{"property": "verifiedUntil", "format": "date"}, + {"property": "status", "format": "select"}], + "views": [{"name": "Needs review", "filters": [` + filters + `]}]}]}` +} + +func warningsFor(t *testing.T, doc string) []Issue { + t.Helper() + var got []Issue + require.NoError(t, ValidateWarn([]byte(doc), func(i Issue) { got = append(got, i) })) + return got +} + +func TestValidate_UnguardedDateLessWarns(t *testing.T) { + for _, cond := range []string{"less", "less_or_equal"} { + t.Run(cond, func(t *testing.T) { + w := warningsFor(t, dateFilterDoc( + `{"property": "verifiedUntil", "condition": "`+cond+`", "date_preset": "today"}`)) + require.Len(t, w, 1) + assert.Contains(t, w[0].Message, "no verifiedUntil") + assert.Contains(t, w[0].Message, "not_empty") + }) + } +} + +func TestValidate_GuardedDateLessIsClean(t *testing.T) { + t.Run("not_empty sibling in the implicit top-level AND", func(t *testing.T) { + assert.Empty(t, warningsFor(t, dateFilterDoc( + `{"property": "verifiedUntil", "condition": "not_empty"}, + {"property": "verifiedUntil", "condition": "less", "date_preset": "today"}`))) + }) + + t.Run("not_empty in an enclosing and-group", func(t *testing.T) { + assert.Empty(t, warningsFor(t, dateFilterDoc( + `{"operator": "and", "filters": [ + {"property": "verifiedUntil", "condition": "not_empty"}, + {"property": "verifiedUntil", "condition": "less", "date_preset": "today"}]}`))) + }) + + // the real shape from the wiki: the guarded pair lives inside an OR branch + t.Run("and-group nested under an or-group", func(t *testing.T) { + assert.Empty(t, warningsFor(t, dateFilterDoc( + `{"operator": "or", "filters": [ + {"operator": "and", "filters": [ + {"property": "verifiedUntil", "condition": "not_empty"}, + {"property": "verifiedUntil", "condition": "less", "date_preset": "today"}]}, + {"property": "status", "condition": "in", "value": ["Needs update"]}]}`))) + }) + + t.Run("exists guards too", func(t *testing.T) { + assert.Empty(t, warningsFor(t, dateFilterDoc( + `{"property": "verifiedUntil", "condition": "exists"}, + {"property": "verifiedUntil", "condition": "less", "date_preset": "today"}`))) + }) +} + +// `… OR dueDate IS EMPTY` deliberately INCLUDES the undated objects, so the +// "also matches objects with no X" warning would contradict the filter's own +// text — the canonical worked example (`done = false AND (dueDate < +// currentWeek() OR dueDate IS EMPTY)`) must not warn on every execution. +func TestValidate_EmptySiblingUnderOrSuppressesTheWarning(t *testing.T) { + t.Run("empty on the same property under the same OR", func(t *testing.T) { + assert.Empty(t, warningsFor(t, dateFilterDoc( + `{"operator": "or", "filters": [ + {"property": "verifiedUntil", "condition": "less", "date_preset": "current_week"}, + {"property": "verifiedUntil", "condition": "empty"}]}`))) + }) + + t.Run("empty on a DIFFERENT property does not suppress", func(t *testing.T) { + w := warningsFor(t, dateFilterDoc( + `{"operator": "or", "filters": [ + {"property": "verifiedUntil", "condition": "less", "date_preset": "current_week"}, + {"property": "status", "condition": "empty"}]}`)) + require.Len(t, w, 1) + assert.Contains(t, w[0].Message, "no verifiedUntil") + }) +} + +// an OR sibling guarantees nothing — the comparison is reachable without it +func TestValidate_NotEmptyUnderOrDoesNotGuard(t *testing.T) { + w := warningsFor(t, dateFilterDoc( + `{"operator": "or", "filters": [ + {"property": "verifiedUntil", "condition": "not_empty"}, + {"property": "verifiedUntil", "condition": "less", "date_preset": "today"}]}`)) + require.Len(t, w, 1) + assert.Contains(t, w[0].Message, "no verifiedUntil") +} + +func TestValidate_DateFilterNonTriggers(t *testing.T) { + t.Run("greater is unaffected", func(t *testing.T) { + // an unset value compares as 1, and Greater tests for -1 + assert.Empty(t, warningsFor(t, dateFilterDoc( + `{"property": "verifiedUntil", "condition": "greater", "date_preset": "today"}`))) + }) + + t.Run("less on a non-date property", func(t *testing.T) { + assert.Empty(t, warningsFor(t, dateFilterDoc( + `{"property": "status", "condition": "less", "value": "x"}`))) + }) +} + +// numberOfDaysAgo / numberOfDaysNow read their operand from `value` +// (getDateRange calls f.Value.Int64()). With no value the count is 0, so +// "edited in the last 30 days" silently becomes "edited today". +func TestValidate_CountingPresetNeedsValue(t *testing.T) { + for _, preset := range []string{"number_of_days_ago", "number_of_days_now"} { + t.Run(preset+" without value", func(t *testing.T) { + err := Validate([]byte(dateFilterDoc( + `{"property": "verifiedUntil", "condition": "greater", "date_preset": "` + preset + `"}`))) + require.Error(t, err) + assert.Contains(t, err.Error(), "needs a day count") + }) + t.Run(preset+" with value", func(t *testing.T) { + assert.NoError(t, Validate([]byte(dateFilterDoc( + `{"property": "verifiedUntil", "condition": "greater", "date_preset": "`+preset+`", "value": 30}`)))) + }) + } + + // a zero count is explicit and legal — it just has to be written down + t.Run("explicit zero is accepted", func(t *testing.T) { + assert.NoError(t, Validate([]byte(dateFilterDoc( + `{"property": "verifiedUntil", "condition": "greater", "date_preset": "number_of_days_ago", "value": 0}`)))) + }) + + // fixed-period presets take no operand + t.Run("fixed presets need no value", func(t *testing.T) { + for _, preset := range []string{"today", "last_week", "current_month", "next_year"} { + assert.NoError(t, Validate([]byte(dateFilterDoc( + `{"property": "verifiedUntil", "condition": "greater", "date_preset": "`+preset+`"}`))), preset) + } + }) +} + +// export must not elide a zero count, or the round trip loses which day the +// filter meant and the document stops validating +func TestRoundtrip_ZeroDayCountSurvives(t *testing.T) { + for _, days := range []int64{0, 30} { + snapshot := dataviewSnapshot() + snapshot.Blocks[1].GetDataview().Views[0].Filters = []*model.BlockContentDataviewFilter{{ + RelationKey: "due", + Condition: model.BlockContentDataviewFilter_Greater, + QuickOption: model.BlockContentDataviewFilter_NumberOfDaysAgo, + Format: model.RelationFormat_date, + Value: &types.Value{Kind: &types.Value_NumberValue{NumberValue: float64(days)}}, + }} + data, err := Marshal(model.SmartBlockType_Page, snapshot, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(data), `"date_preset": "number_of_days_ago"`) + assert.Contains(t, string(data), `"value": `+fmt.Sprint(days)) + + require.NoError(t, Validate(data), "exported document must validate") + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + var got *model.BlockContentDataviewFilter + for _, b := range back.Blocks { + if dv := b.GetDataview(); dv != nil && len(dv.Views) > 0 && len(dv.Views[0].Filters) > 0 { + got = dv.Views[0].Filters[0] + } + } + require.NotNil(t, got) + assert.Equal(t, float64(days), got.Value.GetNumberValue()) + } +} + +// The day count is only ever read where the preset's day range is actually +// applied. transformDateFilter computes the range for every date filter but +// substitutes it for six conditions only — equal, in, less, greater, +// less_or_equal, greater_or_equal (pkg/lib/database/quickoptions.go); every +// other condition returns the filter unchanged, so the preset is inert and a +// missing count means nothing at all rather than "today". +// +// This is where the rule collided with export: `value` is dropped on +// presence-only leaves (§11), so a stored filter with `empty` and a counting +// preset marshalled into a document the package's own validation rejected. A +// 36 808-object sweep found two of them. +func TestValidate_CountingPresetOnlyWhereTheRangeApplies(t *testing.T) { + t.Run("applied: a missing count is still an error", func(t *testing.T) { + for _, cond := range []string{"equal", "in", "less", "greater", "less_or_equal", "greater_or_equal"} { + err := Validate([]byte(dateFilterDoc( + `{"property": "verifiedUntil", "condition": "` + cond + `", "date_preset": "number_of_days_ago"}`))) + require.Error(t, err, cond) + assert.Contains(t, err.Error(), "needs a day count", cond) + } + }) + + t.Run("not applied: the preset is inert, so nothing is missing", func(t *testing.T) { + // presence-only leaves are the ones export strips the value from + for _, cond := range []string{"empty", "not_empty", "exists", "not_equal", "not_in"} { + assert.NoError(t, Validate([]byte(dateFilterDoc( + `{"property": "verifiedUntil", "condition": "`+cond+`", "date_preset": "number_of_days_ago"}`))), cond) + } + }) +} + +// I1 for the same pair: a stored filter with a presence-only condition and a +// counting preset must marshal into a document Validate accepts. +func TestExport_CountingPresetOnPresenceOnlyLeaf(t *testing.T) { + for _, cond := range []model.BlockContentDataviewFilterCondition{ + model.BlockContentDataviewFilter_Empty, + model.BlockContentDataviewFilter_NotEmpty, + model.BlockContentDataviewFilter_Exists, + } { + snapshot := dataviewSnapshot() + snapshot.Blocks[1].GetDataview().Views[0].Filters = []*model.BlockContentDataviewFilter{{ + RelationKey: "due", + Condition: cond, + QuickOption: model.BlockContentDataviewFilter_NumberOfDaysAgo, + Format: model.RelationFormat_date, + }} + data, err := Marshal(model.SmartBlockType_Page, snapshot, testOptions()) + require.NoError(t, err) + assert.NoError(t, Validate(data), "Marshal must not emit what Validate rejects (%v):\n%s", cond, data) + } +} + +// bareDateFilterDoc is dateFilterDoc without the properties list — the shape +// a hand-written dataview actually has, where the only thing that says what +// `due_date` is, is the bundled table. +func bareDateFilterDoc(filters string) string { + return `{"version": 2, "id": "p1", "blocks": [{"type": "dataview", + "object_id": "someSet", + "views": [{"name": "Needs review", "filters": [` + filters + `]}]}]}` +} + +// The day count is only read on a DATE filter. transformDateFilter returns a +// filter of any other format untouched — before getDateRange is reached at +// all (pkg/lib/database/quickoptions.go) — so a counting preset on a text or +// select property is stored UI state that decides nothing, and the count it +// does not carry is not missing. Demanding one there rejected a document the +// app runs exactly as written. +// +// The fixture reaches the date path through the format and nothing else: +// every case below is the same leaf under the same condition, and only the +// property's resolved format moves. Where it resolves to date the error is +// still there, which is what shows the check ran. +func TestValidate_CountingPresetOnlyOnADateProperty(t *testing.T) { + const leaf = `{"property": "%s", "condition": "greater", "date_preset": "number_of_days_ago"}` + + t.Run("a declared non-date property is inert", func(t *testing.T) { + // status is declared `select` two lines above the filter + assert.NoError(t, Validate([]byte(dateFilterDoc(fmt.Sprintf(leaf, "status"))))) + }) + + t.Run("the declaration outranks the bundled table", func(t *testing.T) { + // the same key the bundle calls a date, declared as text by the + // block that owns the filter — impDvFormat reads the properties + // list first, so this filter imports as a text filter + doc := `{"version": 2, "blocks": [{"type": "dataview", + "properties": [{"property": "due_date", "format": "text"}], + "views": [{"filters": [` + fmt.Sprintf(leaf, "due_date") + `]}]}]}` + assert.NoError(t, Validate([]byte(doc))) + }) + + t.Run("a property the bundle knows as a date still errors", func(t *testing.T) { + // no properties list at all: `due_date` resolves through the §3 + // chain to dueDate, whose bundled format is date, which is the + // format import attaches — the rule has to reach that document + err := Validate([]byte(bareDateFilterDoc(fmt.Sprintf(leaf, "due_date")))) + require.Error(t, err) + assert.Contains(t, err.Error(), "needs a day count") + }) + + t.Run("a declared date property still errors", func(t *testing.T) { + err := Validate([]byte(dateFilterDoc(fmt.Sprintf(leaf, "verifiedUntil")))) + require.Error(t, err) + assert.Contains(t, err.Error(), "needs a day count") + }) + + t.Run("an unknown property is not assumed to be a date", func(t *testing.T) { + // neither declared nor bundled: import gives it format 0, which is + // not date, so the preset is inert there too + assert.NoError(t, Validate([]byte(bareDateFilterDoc(fmt.Sprintf(leaf, "whenever"))))) + }) +} + +// The same gate on the fragment surface (API v2 filters): the format comes +// from the space through Options.ResolveFormat — via the reader's vocabulary, +// the way import resolves the same term — instead of from a properties list. +// ResolveProperties is a different seam and answers nothing here; a fixture +// wired to it leaves the format unresolved and the rule never runs. +func TestUnmarshalFilters_CountingPresetOnlyOnADateProperty(t *testing.T) { + countingLeaf := func(prop string) json.RawMessage { + return json.RawMessage(`[{"property":"` + prop + `","condition":"greater","date_preset":"number_of_days_ago"}]`) + } + + t.Run("a non-date property is inert", func(t *testing.T) { + _, err := UnmarshalFilters(countingLeaf("status"), fragFilterOpts()) + assert.NoError(t, err) + }) + + t.Run("a date property still errors", func(t *testing.T) { + _, err := UnmarshalFilters(countingLeaf("dueDate"), fragFilterOpts()) + require.Error(t, err) + assert.Contains(t, err.Error(), "needs a day count") + }) + + t.Run("the documented slug reaches the same resolver", func(t *testing.T) { + // `due_date` is what the API surface documents; it resolves to + // dueDate through the bundled vocabulary before the format is + // looked up, exactly as importer.filterFromJSON resolves it + _, err := UnmarshalFilters(countingLeaf("due_date"), fragFilterOpts()) + require.Error(t, err) + assert.Contains(t, err.Error(), "needs a day count") + }) +} + +// A preset under a condition that does not apply is inert: transformDateFilter +// substitutes the range for six conditions and returns every other filter +// unchanged, so the view matches on the condition alone and the preset is UI +// state. That is worth saying — the author wrote "the last week" and got +// something else — but it is a WARNING, because export has to stay lossless: +// stored filters carry these pairs, and refusing them would make an +// unexportable object out of every one that has one (§11, I1). +func TestValidate_InertPresetWarns(t *testing.T) { + for _, cond := range []string{"not_equal", "not_in", "empty", "not_empty", "exists", "contains"} { + t.Run(cond, func(t *testing.T) { + doc := dateFilterDoc( + `{"property": "verifiedUntil", "condition": "` + cond + `", "date_preset": "current_week"}`) + w := warningsFor(t, doc) // warningsFor requires Validate to pass + require.Len(t, w, 1) + assert.Equal(t, "/blocks/0/views/0/filters/0", w[0].Path) + assert.Contains(t, w[0].Message, `date_preset "current_week" is ignored`) + assert.Contains(t, w[0].Message, cond) + assert.Contains(t, w[0].Message, "greater_or_equal", "the message names the conditions that do apply") + }) + } + + t.Run("a leaf with no condition", func(t *testing.T) { + // an absent condition is proto None, which substitutes nothing either + w := warningsFor(t, dateFilterDoc(`{"property": "verifiedUntil", "date_preset": "today"}`)) + require.Len(t, w, 1) + assert.Contains(t, w[0].Message, "a leaf with no condition") + }) + + t.Run("the six that apply say nothing", func(t *testing.T) { + for _, cond := range []string{"equal", "in", "greater", "greater_or_equal"} { + assert.Empty(t, warningsFor(t, dateFilterDoc( + `{"property": "verifiedUntil", "condition": "`+cond+`", "date_preset": "current_week"}`)), cond) + } + // less and less_or_equal apply too; their one warning is the + // unguarded-comparison trap, not this rule + for _, cond := range []string{"less", "less_or_equal"} { + w := warningsFor(t, dateFilterDoc( + `{"property": "verifiedUntil", "condition": "`+cond+`", "date_preset": "current_week"}`)) + require.Len(t, w, 1, cond) + assert.NotContains(t, w[0].Message, "is ignored", cond) + } + }) + + t.Run("the document still imports, preset and all", func(t *testing.T) { + doc := dateFilterDoc(`{"property": "verifiedUntil", "condition": "empty", "date_preset": "current_week"}`) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + var got *model.BlockContentDataviewFilter + for _, b := range snap.Blocks { + if dv := b.GetDataview(); dv != nil && len(dv.Views) > 0 && len(dv.Views[0].Filters) > 0 { + got = dv.Views[0].Filters[0] + } + } + require.NotNil(t, got) + assert.Equal(t, model.BlockContentDataviewFilter_CurrentWeek, got.QuickOption, + "a warning must not cost the document the field it warned about") + }) +} + +// I1 for the same pairing, from the export side: Marshal writes a preset +// beside a presence-only condition because the stored filter has one, so the +// verdict on its own output may be a warning and may never be a refusal. +func TestExport_InertPresetWarnsButNeverRejects(t *testing.T) { + for _, cond := range []model.BlockContentDataviewFilterCondition{ + model.BlockContentDataviewFilter_Empty, + model.BlockContentDataviewFilter_NotEmpty, + model.BlockContentDataviewFilter_Exists, + model.BlockContentDataviewFilter_NotEqual, + } { + snapshot := dataviewSnapshot() + snapshot.Blocks[1].GetDataview().Views[0].Filters = []*model.BlockContentDataviewFilter{{ + RelationKey: "due", + Condition: cond, + QuickOption: model.BlockContentDataviewFilter_CurrentWeek, + Format: model.RelationFormat_date, + }} + data, err := Marshal(model.SmartBlockType_Page, snapshot, testOptions()) + require.NoError(t, err) + require.Contains(t, string(data), `"date_preset": "current_week"`, "export keeps the preset") + + var warnings []Issue + require.NoError(t, ValidateWarn(data, func(i Issue) { warnings = append(warnings, i) }), + "Marshal must not emit what Validate rejects (%v):\n%s", cond, data) + require.Len(t, warnings, 1, "%v: %v", cond, warnings) + assert.Contains(t, warnings[0].Message, "is ignored") + } +} + +// The day-count rule reads the OPERAND, not the member. It used to check only +// that "value" was present, so `"value": null` — and a string, and a boolean — +// passed both Validate and Unmarshal (no I2 break: both accepted) while the +// engine read the very 0 the message warns about: domain.Value.Int64 answers +// 0 for every kind that is not a number. The bound is the one the compact +// grammar already applies to daysAgo(n) (§6.2.1); two forms of one filter +// language admit the same filters. +func TestValidate_CountingPresetOperandIsADayCount(t *testing.T) { + leaf := func(value string) string { + return dateFilterDoc(`{"property": "verifiedUntil", "condition": "greater", + "date_preset": "number_of_days_ago", "value": ` + value + `}`) + } + + t.Run("refused", func(t *testing.T) { + for name, tc := range map[string]struct{ value, says string }{ + "null": {"null", "null counts as 0 days"}, + "a string": {`"30"`, "a string counts as 0 days"}, + "a boolean": {"true", "a boolean counts as 0 days"}, + "an array": {"[30]", "an array counts as 0 days"}, + "a fraction": {"3.5", "3.5 is not a whole day count"}, + "a negative": {"-1", "-1 is not a whole day count"}, + "past the bound": {"36501", "36501 is not a whole day count"}, + } { + t.Run(name, func(t *testing.T) { + // when + err := Validate([]byte(leaf(tc.value))) + + // then + require.Error(t, err, "an operand the engine reads as 0 is the trap the message describes") + var ve *ValidationError + require.ErrorAs(t, err, &ve) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/blocks/0/views/0/filters/0/value", ve.Issues[0].Path, + "the fault is the operand, not the leaf that carries it (§13)") + assert.Contains(t, ve.Issues[0].Message, tc.says) + + // and I2: Unmarshal reaches the same verdict + _, _, err = Unmarshal([]byte(leaf(tc.value)), Options{GenerateId: seqIds("g")}) + require.Error(t, err) + assert.Equal(t, ve.Issues, err.(*ValidationError).Issues) + }) + } + }) + + // a number no float64 can hold is refused, once: checkNumbers owns that + // fault and addresses the same pointer, and one fault is one issue (§12) + t.Run("a number out of float64 range is somebody else's issue", func(t *testing.T) { + err := Validate([]byte(leaf("1e400"))) + var ve *ValidationError + require.ErrorAs(t, err, &ve) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/blocks/0/views/0/filters/0/value", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, "out of range: values must fit a 64-bit float") + }) + + t.Run("accepted", func(t *testing.T) { + for _, value := range []string{"0", "1", "30", "36500"} { + t.Run(value, func(t *testing.T) { + require.NoError(t, Validate([]byte(leaf(value)))) + _, _, err := Unmarshal([]byte(leaf(value)), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + }) + } + }) + + // the gate is unchanged: an operand nothing reads is not a fault + t.Run("still inert where the preset is", func(t *testing.T) { + assert.NoError(t, Validate([]byte(dateFilterDoc( + `{"property": "status", "condition": "greater", + "date_preset": "number_of_days_ago", "value": null}`))), + "a non-date property never reaches the preset's range") + }) +} + +// Export's half of the same rule: a stored operand that is not a day count +// has no written form, so it may not travel verbatim — that document is one +// this package's own Validate refuses (§11, I1). It is written as the count +// the query engine reads out of it, and the caller is told. +func TestExport_CountingPresetOperandIsWritableOrReported(t *testing.T) { + marshal := func(t *testing.T, value *types.Value) (string, []Issue) { + t.Helper() + var warned []Issue + root := &model.Block{Id: "o1", ChildrenIds: []string{"dv"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + snap := &model.SmartBlockSnapshotBase{Blocks: []*model.Block{root, {Id: "dv", + Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + RelationLinks: []*model.RelationLink{{Key: "dueDate", Format: model.RelationFormat_date}}, + Views: []*model.BlockContentDataviewView{{Id: "v1", + Filters: []*model.BlockContentDataviewFilter{{ + RelationKey: "dueDate", + Condition: model.BlockContentDataviewFilter_Greater, + QuickOption: model.BlockContentDataviewFilter_NumberOfDaysAgo, + Format: model.RelationFormat_date, + Value: value, + }}}}, + }}}}} + data, err := Marshal(model.SmartBlockType_Page, snap, + Options{OnWarning: func(i Issue) { warned = append(warned, i) }}) + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal may never emit what its own Validate refuses:\n%s", data) + return string(data), warned + } + + t.Run("a whole count in range travels untouched", func(t *testing.T) { + data, warned := marshal(t, &types.Value{Kind: &types.Value_NumberValue{NumberValue: 30}}) + assert.Contains(t, data, `"value": 30`) + assert.Empty(t, warned) + }) + + t.Run("no operand is the stored shape of today, and stays quiet", func(t *testing.T) { + data, warned := marshal(t, nil) + assert.Contains(t, data, `"value": 0`) + assert.Empty(t, warned) + }) + + t.Run("a string operand is written as the count the engine reads", func(t *testing.T) { + data, warned := marshal(t, str("a week")) + assert.Contains(t, data, `"value": 0`) + require.Len(t, warned, 1) + assert.Contains(t, warned[0].Message, `carries a week as its day count`) + assert.Contains(t, warned[0].Message, "0 is written instead") + }) + + t.Run("a count past the bound is pinned to it and reported", func(t *testing.T) { + data, warned := marshal(t, &types.Value{Kind: &types.Value_NumberValue{NumberValue: 40000}}) + assert.Contains(t, data, `"value": 36500`) + require.Len(t, warned, 1) + assert.Contains(t, warned[0].Message, "36500 is written instead") + }) + + t.Run("a fraction is truncated the way the engine truncates it", func(t *testing.T) { + data, warned := marshal(t, &types.Value{Kind: &types.Value_NumberValue{NumberValue: 3.7}}) + assert.Contains(t, data, `"value": 3`) + require.Len(t, warned, 1) + }) +} diff --git a/pkg/lib/anyblockjson/dictionary.go b/pkg/lib/anyblockjson/dictionary.go new file mode 100644 index 0000000000..9762445b12 --- /dev/null +++ b/pkg/lib/anyblockjson/dictionary.go @@ -0,0 +1,536 @@ +package anyblockjson + +// dictionary.go implements §2f: the bundle-level property dictionary, +// properties.json. Every other document in this format describes one object +// and index.json describes the set; the dictionary says what the set's +// PROPERTIES mean — one file naming every property the bundle's objects use, +// in place of the ~9,500 relation documents per account that restated the +// bundled table field for field (measured: 9,675 of 10,617 relation +// documents are installed copies of the 194 bundled relations, and 98% of +// those are field-identical to bundle/relations.json). +// +// It is a sibling of index.json, not a section inside it, deliberately: an +// index says WHERE things are, a dictionary says WHAT THEY MEAN (§2f). And +// it is the third home of $defs/propertyDefinition (§2e) — a dictionary +// entry, a type's property-definition entry and a relation document's +// property_settings are one shape in three places, which is why the Go +// surface here is []PropertyDefinition rather than a fourth field list. +// +// Self-sufficiency is the constraint that shapes it: a third-party reader +// must be able to interpret a backup WITHOUT shipping bundle/relations.json, +// so every entry carries its `format`. Dropping bundled relation documents +// with no dictionary was considered and rejected for exactly this reason — +// the reader could no longer tell a date from a string, which is the same +// "stands alone" property that keeps a space id off the envelope. + +import ( + "bytes" + _ "embed" + "fmt" + "math" + "sort" + "strconv" + "strings" + "sync" + + "github.com/santhosh-tekuri/jsonschema/v6" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" +) + +//go:embed schema/properties.schema.json +var propertiesSchemaJSON []byte + +// PropertiesFileName is the name a bundle's property dictionary must have, +// at the bundle root beside index.json — IndexFileName's rule (§2f). +const PropertiesFileName = "properties.json" + +// PropertyDictionary is a bundle's properties.json (§2f). +type PropertyDictionary struct { + // Installed lists the BUNDLED properties present in the space — + // presence, not definition. This field holds STORED keys; the wire + // spells them as display names ("Due date", not `dueDate`) — + // the dictionary is aligned with every other slot. + // 98% of installed copies + // are field-identical to the bundled table, so the key is the whole of + // what a restore needs. A key that also appears in Properties is + // installed AND divergent: the entry overrides the table. + Installed []string + // Properties carries one definition per property the bundle's objects + // actually reference — used-only (§2f) — plus a full entry for every + // installed copy that diverges from the bundled table. Keys are STORED + // keys, never document spellings: a document's property_internal_keys legend + // binds its labels to stored keys, and the stored key is what the + // dictionary answers for. + Properties []PropertyDefinition +} + +var compilePropertiesSchema = sync.OnceValues(func() (*jsonschema.Schema, error) { + doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(propertiesSchemaJSON)) + if err != nil { + return nil, fmt.Errorf("decode embedded properties schema: %w", err) + } + c := jsonschema.NewCompiler() + // the object schema is added alongside because a dictionary entry is a + // $ref into it (§2e): the three homes of propertyDefinition share one + // $defs rather than a copy in each that drifts — the same wiring the + // index schema uses for its icon. + objectDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaJSON)) + if err != nil { + return nil, fmt.Errorf("decode embedded schema: %w", err) + } + if err := c.AddResource(SchemaURL, objectDoc); err != nil { + return nil, fmt.Errorf("add schema resource: %w", err) + } + if err := c.AddResource(PropertiesSchemaURL, doc); err != nil { + return nil, fmt.Errorf("add properties schema resource: %w", err) + } + sch, err := c.Compile(PropertiesSchemaURL) + if err != nil { + return nil, fmt.Errorf("compile properties schema: %w", err) + } + return sch, nil +}) + +// jsonDictionary is the decoded properties.json. Entries decode through the +// same JSON layer as a type document's property-definition entries +// (TypeProperty) so the two doors cannot disagree about which members travel +// — `section` never arrives, because the schema refuses it on a dictionary +// entry before this decode runs. +type jsonDictionary struct { + Installed []string `json:"installed"` + Properties []TypeProperty `json:"properties"` +} + +// UnmarshalPropertyDictionary validates data against the properties schema +// and decodes it (§2f). Errors wrap *ValidationError with path-addressed +// issues, like Unmarshal and UnmarshalIndex. +// +// An `installed` key the bundled table does not know is TOLERATED, not +// refused, and the asymmetry with MarshalPropertyDictionary is deliberate: +// the bundled table grows independently of the format version, so a backup +// written by a newer app lists keys an older reader has never heard of — +// refusing them would make every backup unreadable one app version back. +// The reader installs the keys it knows and skips the rest; a WRITER, which +// checks against its own table, has no such excuse. +func UnmarshalPropertyDictionary(data []byte) (*PropertyDictionary, error) { + return UnmarshalPropertyDictionaryWarn(data, nil) +} + +// UnmarshalPropertyDictionaryWarn is UnmarshalPropertyDictionary with a sink +// for warning-grade issues, the way ValidateWarn is for object documents. +// +// The dictionary had no such sink, and it is the one file in the format whose +// keys are STORED keys while every other slot spells the snake_case label — +// so the likeliest authoring mistake, writing the label, produced NOTHING on +// the way in. An `installed` key outside the bundled table read clean and +// failed only on the way back out; a `properties` entry keyed by the label +// read clean and quietly minted a second property beside the bundled one. +func UnmarshalPropertyDictionaryWarn(data []byte, onWarning func(Issue)) (*PropertyDictionary, error) { + return unmarshalPropertyDictionary(data, onWarning) +} + +func unmarshalPropertyDictionary(data []byte, warn func(Issue)) (*PropertyDictionary, error) { + raw, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + if err != nil { + return nil, &ValidationError{Issues: []Issue{{Message: fmt.Sprintf("invalid JSON: %v", err)}}} + } + doc, ok := raw.(map[string]any) + if !ok { + return nil, &ValidationError{Issues: []Issue{{Message: "property dictionary must be a JSON object"}}} + } + // the dictionary shares the format version and its rules with object + // documents (§10): gate on it here, before the schema can turn a newer + // version into a generic "value must be 1" that says nothing about why + if err := checkVersion(doc); err != nil { + return nil, err + } + if issues := misroutedIssues(data, KindPropertyDictionary); len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + sch, err := compilePropertiesSchema() + if err != nil { + return nil, fmt.Errorf("embedded properties schema: %w", err) + } + if err := sch.Validate(raw); err != nil { + return nil, &ValidationError{Issues: schemaIssues(err, keySlotReport{})} + } + if issues := dictionaryDuplicateIssues(doc); len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + + var jd jsonDictionary + if err := jsonUnmarshal(data, &jd); err != nil { + return nil, fmt.Errorf("decode property dictionary: %w", err) + } + d := &PropertyDictionary{Installed: installedKeys(jd.Installed, warn)} + for i, tp := range jd.Properties { + // an entry's `internal_key` IS the stored key and skips the chain — + // a stored id is its own address (§3) and the fold match below could + // rebind it onto a bundled twin. + // + // When an entry states BOTH, the `property` spelling wins + // (authoredKey): export writes the pair from one stored key, so they + // always agree in anything this package produced, and where an author + // makes them disagree the spelling is what the document's own values + // resolve through. Neither order is safe on a disagreeing pair — + // honouring the internal_key would point the entry at a property no + // value in the document uses — so the disagreement is REPORTED rather + // than silently resolved. + term, isInternal := tp.authoredKey() + if tp.Property != "" && tp.InternalKey != "" { + if resolved, _ := dictionaryStoredKey(tp.Property); resolved != tp.InternalKey { + warnIssue(warn, fmt.Sprintf("/properties/%d", i), + "this entry states property %q and internal_key %q, and they name different "+ + "properties (%q resolves to %q). The spelling wins, because it is what the "+ + "document's own values resolve through — state one, or make them agree", + tp.Property, tp.InternalKey, tp.Property, resolved) + } + } + storedKey := term + if !isInternal { + storedKey = dictionaryEntryKey(i, term, warn) + } + // entries speak STORED keys in every key slot — the entry identity + // and `object_types` alike — so there is no legend to run and no + // vocabulary to consult: the definition is built by the same shared + // builder both doors of the §2a array use, with the slots passed + // through verbatim. `format` resolves per key exactly as a + // property_settings format does (§3): "text" on a bundled + // short-text key stays short text, and on anything else is longtext. + def := tp.definition(storedKey, declaredFormatWith(Options{}, storedKey, tp.Format), + mapStrings(tp.ObjectTypes, dictionaryStoredTypeKey)) + d.Properties = append(d.Properties, def) + } + return d, nil +} + +// dictionaryKeySpelling renders an ENTRY's stored key the way the dictionary +// spells it: the bundled spelling for a bundled property — its display name +// from the shipped table (bundledname.go) — the stored key verbatim for +// anything else (§2f). +// +// Only `properties` needs the condition. `installed` admits bundled keys +// and nothing else — it names rows to restore from the bundled table — so +// it names unconditionally. An ENTRY, by contrast, is how a bundle declares +// a property the bundled table does NOT have, so its key population is +// mixed: of 6,426 entries in a 77-space export, 515 are space-minted bson +// ids. For those the condition is load-bearing rather than cosmetic: the +// dictionary has no legend, so its spelling must be a pure function of the +// key, and the only pure spelling a space-minted key has is itself — +// nothing may ever be derived from a bson id. +func dictionaryKeySpelling(storedKey string) string { + if bundle.HasRelation(domain.RelationKey(storedKey)) { + return bundledPropertySpelling(storedKey) + } + return storedKey +} + +// dictionaryTypeSpelling renders a TARGET type key the way the dictionary +// spells it: the display name for a bundled type ("Property" for the type +// stored as `relation`), the stored key verbatim for anything else (§2f) — +// the same rule the entry's own key follows, for the same reason. +// +// A type document's `object_types` reaches the same answer by a different +// road: it spells through the exporter's per-document ledger and binds the +// term in that document's `type_internal_keys` legend. The dictionary has no legend, +// so its spelling must be a PURE FUNCTION of the key, which is what makes +// the bundled name table the right instrument and a ledger the wrong one. +// +// Measured before this rule existed: type documents spelled 5,377 of 5,377 +// target types as slugs, while dictionary entries spelled 232 of 803 in +// camelCase — the same concept, two spellings, one bundle. +func TypeKeySpelling(typeKey string) string { return dictionaryTypeSpelling(typeKey) } + +// StoredTypeKey inverts TypeKeySpelling. +func StoredTypeKey(spelling string) string { return dictionaryStoredTypeKey(spelling) } + +func dictionaryTypeSpelling(typeKey string) string { + if _, err := bundle.GetType(domain.TypeKey(typeKey)); err == nil { + return bundledTypeSpelling(typeKey) + } + return typeKey +} + +// dictionaryStoredTypeKey inverts dictionaryTypeSpelling, by the chain every +// slot in the format follows: an exact stored key names itself, then the +// bundled name table, then a single fold match, and an ambiguity is never +// resolved by guess. The stored-key step running FIRST is deliberate and +// pinned: `relation` is a bundled type's stored key, so it still names that +// type verbatim even though its wire spelling is the display name "Property". +func dictionaryStoredTypeKey(spelling string) string { + if _, err := bundle.GetType(domain.TypeKey(spelling)); err == nil { + return spelling + } + if key, ok := bundledTypeKeyBySpelling(spelling); ok { + return key + } + if candidates := BundledTypeKeysByFold(spelling); len(candidates) == 1 { + return candidates[0] + } + return spelling +} + +// dictionaryStoredKey resolves a dictionary spelling back to the stored key +// it names, following the same chain every other slot in the format follows: +// an exact stored key wins, then the bundled name table, then a single fold +// match, and an ambiguity is never resolved by guess +// (BundledPropertyKeysByFold). +// +// ok is false only when the spelling folds onto more than one bundled +// property, which cannot happen for a spelling this package wrote — +// TestDictionaryKeys_TheBundledTableStaysUnambiguous pins that — but can for +// one an author invents. +func dictionaryStoredKey(spelling string) (stored string, ambiguous []string) { + if bundle.HasRelation(domain.RelationKey(spelling)) { + return spelling, nil // a stored key names itself + } + if key, ok := bundledPropertyKeyBySpelling(spelling); ok { + return key, nil // the bundled name table, before the fold + } + candidates := BundledPropertyKeysByFold(spelling) + switch len(candidates) { + case 1: + return candidates[0], nil + case 0: + return spelling, nil // a space-minted key, or a newer app's + default: + names := append([]string(nil), candidates...) + sort.Strings(names) + return spelling, names + } +} + +// installedKeys reads the `installed` list into stored keys, reporting a key +// that names no bundled property. +// +// Every key here is meant to be bundled — `installed` names rows to restore +// from the bundled table, and a key outside it tells a reader to install +// nothing. Such a key is TOLERATED rather than refused, and the tolerance is +// about VERSION SKEW rather than custom properties: the bundled table grows independently of the +// format version, so a backup written by a newer app lists keys an older +// reader has never heard of, and refusing them would make every backup +// unreadable one app version back. What was missing is that nothing SAID so — +// the document read clean and only re-rendering it failed. +func installedKeys(raw []string, warn func(Issue)) []string { + if len(raw) == 0 { + return raw + } + out := make([]string, 0, len(raw)) + for i, spelling := range raw { + path := fmt.Sprintf("/installed/%d", i) + stored, ambiguous := dictionaryStoredKey(spelling) + switch { + case len(ambiguous) > 0: + warnIssue(warn, path, "installed key %q folds onto more than one bundled property (%s), "+ + "so which is meant cannot be decided here — write one of them", + spelling, strings.Join(quoteAll(ambiguous), ", ")) + case !bundle.HasRelation(domain.RelationKey(stored)): + warnIssue(warn, path, "installed key %q is not a bundled property, so a reader "+ + "restoring this bundle installs NOTHING for it. Give it a full entry in "+ + "`properties`, where its definition travels with it — or, if it comes from a "+ + "newer app whose bundled table has it, expect this reader to skip it", spelling) + } + out = append(out, stored) + } + return out +} + +// dictionaryEntryKey resolves an entry's key, reporting an ambiguity. +func dictionaryEntryKey(i int, spelling string, warn func(Issue)) string { + stored, ambiguous := dictionaryStoredKey(spelling) + if len(ambiguous) > 0 { + warnIssue(warn, fmt.Sprintf("/properties/%d/"+memberProperty, i), + "%q folds onto more than one bundled property (%s), so which is meant cannot be "+ + "decided here — write one of them", + spelling, strings.Join(quoteAll(ambiguous), ", ")) + } + return stored +} + +func mapStrings(in []string, f func(string) string) []string { + if len(in) == 0 { + return in + } + out := make([]string, len(in)) + for i, s := range in { + out[i] = f(s) + } + return out +} + +func quoteAll(names []string) []string { + out := make([]string, 0, len(names)) + for _, n := range names { + out = append(out, strconv.Quote(n)) + } + return out +} + +func warnIssue(warn func(Issue), path, format string, args ...any) { + if warn != nil { + warn(Issue{Path: path, Message: fmt.Sprintf(format, args...)}) + } +} + +// dictionaryDuplicateIssues refuses a key stated twice, in either list. Two +// entries for one key are two definitions of one property with no rule for +// which wins — the canonical form has exactly one slot per key, the way a +// document's `properties` object structurally has, and Marshal refuses the +// same input (§11 I1: the two sides owe the same answer). +func dictionaryDuplicateIssues(doc map[string]any) []Issue { + var issues []Issue + seenInstalled := map[string]int{} + installed, _ := doc["installed"].([]any) + for i, raw := range installed { + key, _ := raw.(string) + if first, dup := seenInstalled[key]; dup { + issues = append(issues, Issue{ + Path: fmt.Sprintf("/installed/%d", i), + Message: fmt.Sprintf("%q is already listed at /installed/%d — the dictionary has one slot per key", + key, first), + }) + continue + } + seenInstalled[key] = i + } + seenEntries := map[string]int{} + entries, _ := doc["properties"].([]any) + for i, raw := range entries { + entry, _ := raw.(map[string]any) + // the identity an entry states, spelling first — the same order + // authoredKey runs (§2e) + key, _ := entry[memberProperty].(string) + if key == "" { + key, _ = entry[memberInternalKey].(string) + } + if key == "" { + continue // the schema's required/minLength verdict already stands + } + if first, dup := seenEntries[key]; dup { + issues = append(issues, Issue{ + Path: fmt.Sprintf("/properties/%d/"+memberProperty, i), + Message: fmt.Sprintf("%q is already defined at /properties/%d — one property, one definition", + key, first), + }) + continue + } + seenEntries[key] = i + } + return issues +} + +// MarshalPropertyDictionary renders a dictionary in the canonical byte form +// (§4): `installed` and `properties` each sorted by key, one slot per key. +// It refuses what UnmarshalPropertyDictionary refuses — a duplicated key — +// and two things only a writer can check: an entry whose key has no written +// form, and an `installed` key its own bundled table does not know, which +// would tell the reader to install nothing (the repair is a full entry in +// `properties`, where the format travels with it). +func MarshalPropertyDictionary(d *PropertyDictionary) ([]byte, error) { + if d == nil { + return nil, fmt.Errorf("nil property dictionary") + } + doc := &omap{} + doc.set("$schema", PropertiesSchemaURL) + doc.set("version", FormatVersion) + + // stored keys in, NAMES out (§2f): every key here is a bundled property, + // and a bundled property's written spelling is its display name + // everywhere else in the format. The dictionary used to be the one file + // that spelled a property one way while every document beside it spelled + // it another (`dueDate` against the then-current `due_date`). + installed := make([]string, 0, len(d.Installed)) + for _, key := range d.Installed { + if _, err := bundle.GetRelation(domain.RelationKey(key)); err != nil { + return nil, fmt.Errorf("installed key %q is not a bundled property: `installed` restores from the "+ + "bundled table, so a key outside it tells the reader to install nothing — give it a full "+ + "entry in `properties` instead", key) + } + // the bundled spelling unconditionally, not dictionaryKeySpelling: + // the check above has already established this key is bundled, and + // `installed` admits nothing else — it is a list of rows to restore + // from the bundled table, so a space-minted key has no meaning in it. + installed = append(installed, bundledPropertySpelling(key)) + } + sort.Strings(installed) + for i, key := range installed { + if i > 0 && installed[i-1] == key { + return nil, fmt.Errorf("installed key %q is listed twice: the dictionary has one slot per key", key) + } + } + doc.setNonEmpty("installed", stringsToAny(installed)) + + defs := append([]PropertyDefinition(nil), d.Properties...) + sort.Slice(defs, func(i, j int) bool { return defs[i].Key < defs[j].Key }) + var entries []any + for i, def := range defs { + if i > 0 && defs[i-1].Key == def.Key { + return nil, fmt.Errorf("property %q is defined twice: one property, one definition", def.Key) + } + entry, err := dictionaryEntryOmap(def) + if err != nil { + return nil, err + } + entries = append(entries, entry) + } + doc.setNonEmpty("properties", entries) + return marshalCanonical(doc) +} + +// dictionaryEntryOmap renders one entry: the propertyDefinition members in +// the §2e order, its `property` in the dictionary's spelling — the display +// name for a bundled property, the stored key verbatim for a space-minted +// one (§2f) — and its +// `internal_key` the stored key verbatim, the export-fidelity half an author +// never has to write. There is still no legend to write: the spelling is a +// pure function of the key, so a reader inverts it without one. `format` is +// written unconditionally — required by the schema, because an entry without +// one is readable only by a reader shipping the bundled table (§2f) — and a +// stored format outside the enum is an ERROR for relationFormatName's +// reason: writing "text" for a format that is not text would be a permanent +// silent format rewrite, the disease the dictionary must not reintroduce. +func dictionaryEntryOmap(def PropertyDefinition) (*omap, error) { + if !isWritablePropertyKey(string(def.Key)) { + return nil, fmt.Errorf("property dictionary: %s", unwritableKeyReason("property key", string(def.Key))) + } + spelling := dictionaryKeySpelling(string(def.Key)) + name := formatName(def.Format) + if name == "" { + return nil, fmt.Errorf("property %q: format %d has no name in this format: "+ + "the entry cannot state what the property holds", def.Key, def.Format) + } + m := &omap{} + m.set(memberProperty, spelling) + m.set(memberInternalKey, string(def.Key)) + m.setNonEmpty("name", def.Name) + m.set("format", name) + m.setNonEmpty("options", optionsToAny(def.Options)) + m.setNonEmpty("object_types", stringsToAny(mapStrings(def.ObjectTypes, dictionaryTypeSpelling))) + m.setNonEmpty("description", def.Description) + if def.IncludeTime != nil { + // a pointer false is a declaration, not an absence — the same + // distinction the §2a array preserves through both its doors + m.set("include_time", *def.IncludeTime) + } + // the schema bounds max_count to a non-negative int32, and setNonEmpty + // would have written whatever the caller held: a negative or + // out-of-range value produced bytes this file's own Unmarshal refuses, + // which is §11 I1 broken on the shortest possible path. Refused by name, + // the same treatment `format` gets above — an entry that cannot be read + // back is not an entry. + if def.MaxCount < 0 || def.MaxCount > math.MaxInt32 { + return nil, fmt.Errorf("property %q: max_count %d is outside the range an entry can "+ + "state (0..%d)", def.Key, def.MaxCount, math.MaxInt32) + } + m.setNonEmpty("max_count", def.MaxCount) + m.setNonEmpty("readonly", def.Readonly) + if def.DefaultValue != nil { + // canonicalize through the same value pipeline every property value + // takes (§3), so a map-shaped default has sorted members and the + // output is byte-stable + m.set("default_value", protoValueToJSON(jsonToProtoValue(def.DefaultValue))) + } + return m, nil +} diff --git a/pkg/lib/anyblockjson/dictionary_test.go b/pkg/lib/anyblockjson/dictionary_test.go new file mode 100644 index 0000000000..7a1e3e1358 --- /dev/null +++ b/pkg/lib/anyblockjson/dictionary_test.go @@ -0,0 +1,291 @@ +package anyblockjson + +// dictionary_test.go pins §2f: the property dictionary is a bundle-level +// document with its own schema, its entries are the third home of +// $defs/propertyDefinition, and Marshal/Unmarshal refuse the same +// malformations so a dictionary one side produces is never one the other +// rejects (§11 I1's shape, at the bundle level). + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func boolPtr(b bool) *bool { return &b } + +// The dictionary round-trips byte-stably through its own two entry points, +// with every propertyDefinition member travelling: a member the schema +// admits and the codec sheds would make the dictionary quietly mean less +// than it says — the exact seam failure §2e pins for the type-document door. +// +// How this can fail: drop a member from dictionaryEntryOmap or from the +// TypeProperty decode path (the member vanishes on the way round); stop +// sorting entries or `installed` (the second marshal reorders and the byte +// check goes red); or route default_value around the canonicalizing value +// pipeline (a map-shaped default re-marshals with unstable member order). +func TestPropertyDictionary_RoundTripBytesStable(t *testing.T) { + // given + in := &PropertyDictionary{ + Installed: []string{"tag", "dueDate"}, + Properties: []PropertyDefinition{ + { + Key: "6a32d4856761631534b22f85", + Name: "Budget", + Format: model.RelationFormat_number, + }, + { + Key: "693c14f2aa11631534b22f01", + Name: "Owner", + Format: model.RelationFormat_object, + ObjectTypes: []string{"participant"}, + Description: "Who carries it", + MaxCount: 1, + Readonly: true, + DefaultValue: map[string]any{"b": 2.0, "a": 1.0}, + }, + { + Key: "5f1e0a7788aa631534b22f02", + Name: "Stage", + Format: model.RelationFormat_status, + Options: []OptionDefinition{{Name: "Now", Color: "red"}, {Name: "Later"}}, + IncludeTime: boolPtr(false), // a pointer false is a declaration, not an absence + }, + }, + } + + // when + data, err := MarshalPropertyDictionary(in) + require.NoError(t, err) + got, err := UnmarshalPropertyDictionary(data) + require.NoError(t, err) + data2, err := MarshalPropertyDictionary(got) + require.NoError(t, err) + + // then + assert.Equal(t, string(data), string(data2), "Marshal ∘ Unmarshal must be byte-stable") + require.Len(t, got.Properties, 3) + byKey := map[domain.RelationKey]PropertyDefinition{} + for _, def := range got.Properties { + byKey[def.Key] = def + } + owner := byKey["693c14f2aa11631534b22f01"] + assert.Equal(t, model.RelationFormat_object, owner.Format) + assert.Equal(t, []string{"participant"}, owner.ObjectTypes) + assert.Equal(t, "Who carries it", owner.Description) + assert.Equal(t, int64(1), owner.MaxCount) + assert.True(t, owner.Readonly) + assert.Equal(t, map[string]any{"a": 1.0, "b": 2.0}, owner.DefaultValue) + stage := byKey["5f1e0a7788aa631534b22f02"] + require.NotNil(t, stage.IncludeTime) + assert.False(t, *stage.IncludeTime) + assert.Equal(t, []OptionDefinition{{Name: "Now", Color: "red"}, {Name: "Later"}}, stage.Options) + assert.Equal(t, []string{"dueDate", "tag"}, got.Installed, "installed is sorted on the way out") +} + +// `format` resolves per key, exactly as a property_settings format does +// (§3): "text" names both stored text formats and the entry's stored KEY is +// what disambiguates — a bundled short-text property keeps its stored format +// through the dictionary even though the document never spells it. +// +// How this can fail: decode the format name literally (formatNames.value) +// instead of through declaredFormatWith — the bundled `name` key comes back +// longtext and the assertion on shorttext goes red. +func TestPropertyDictionary_TextResolvesPerKey(t *testing.T) { + // given: `name` is bundled shorttext; the minted key has no stored format + data := []byte(`{"version":2,"properties":[ + {"property":"name","format":"text"}, + {"property":"6a32d4856761631534b22f85","format":"text"}]}`) + + // when + got, err := UnmarshalPropertyDictionary(data) + + // then + require.NoError(t, err) + require.Len(t, got.Properties, 2) + assert.Equal(t, model.RelationFormat_shorttext, got.Properties[0].Format) + assert.Equal(t, model.RelationFormat_longtext, got.Properties[1].Format) +} + +// The schema is the gate: an entry is a LAYER over propertyDefinition that +// requires `format`, refuses `section` (a type-owned member with no meaning +// off a type document) and closes itself, and the root refuses undeclared +// members and gates the version the way every surface of this format does +// (§10). +// +// How this can fail, case by case: drop `format` from the entry's required +// list (first case green on an entry a bundled-table-free reader cannot +// interpret); remove the entry's unevaluatedProperties gate (section and the +// typo'd member both pass); remove additionalProperties: false at the root +// (the misplaced member passes); route UnmarshalPropertyDictionary around +// checkVersion (the newer-version error loses NewerFormat and both-versions +// wording). +func TestPropertyDictionary_SchemaRefusals(t *testing.T) { + t.Run("an entry without format is refused", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"properties":[{"property":"dueDate","name":"End Date"}]}`)) + require.Error(t, err, "self-sufficiency: an entry without a format is readable only with the bundled table in hand") + }) + t.Run("section is a type-owned member and is refused", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"properties":[{"property":"tag","format":"multi_select","section":"featured"}]}`)) + require.Error(t, err) + }) + t.Run("an unknown entry member is refused through the layer", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"properties":[{"property":"tag","format":"multi_select","formats":"x"}]}`)) + require.Error(t, err) + }) + t.Run("an undeclared root member is refused", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"props":[]}`)) + require.Error(t, err) + }) + t.Run("a newer version is refused by the gate, both versions named", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version":3}`)) + require.Error(t, err) + var ve *ValidationError + require.ErrorAs(t, err, &ve) + assert.True(t, ve.NewerFormat, "the dedicated newer-format verdict, not a generic const failure") + }) + t.Run("null object_types stays a relation-only shape", func(t *testing.T) { + // the shared shape admits null because a relation's STORED value can + // hold one (§2d); a dictionary entry describes rather than mirrors a + // store slot, so its layer narrows it back to an array + _, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"properties":[{"property":"assignee","format":"objects","object_types":null}]}`)) + require.Error(t, err) + }) +} + +// One key, one slot, on both sides: Unmarshal refuses a duplicated key with +// the first occurrence named, and Marshal refuses the same input rather than +// emitting a file its own Unmarshal rejects (§11 I1 at the bundle level). +// +// How this can fail: delete dictionaryDuplicateIssues (the read side +// accepts two definitions of one property with no rule for which wins), or +// delete either duplicate check in MarshalPropertyDictionary (the write +// side emits what the read side refuses). +func TestPropertyDictionary_OneSlotPerKey(t *testing.T) { + t.Run("a duplicated entry key is refused on read", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"properties":[ + {"property":"dueDate","format":"date"},{"property":"dueDate","format":"text"}]}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/properties/1/property") + }) + t.Run("a duplicated installed key is refused on read", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"installed":["tag","tag"]}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/installed/1") + }) + t.Run("marshal refuses the duplicate entry too", func(t *testing.T) { + _, err := MarshalPropertyDictionary(&PropertyDictionary{Properties: []PropertyDefinition{ + {Key: "dueDate", Format: model.RelationFormat_date}, + {Key: "dueDate", Format: model.RelationFormat_longtext}, + }}) + require.Error(t, err) + }) + t.Run("marshal refuses the duplicate installed key too", func(t *testing.T) { + _, err := MarshalPropertyDictionary(&PropertyDictionary{Installed: []string{"tag", "tag"}}) + require.Error(t, err) + }) +} + +// `installed` restores from the bundled table, so the two sides treat an +// unknown key differently ON PURPOSE: the writer checks against its own +// table and refuses (a key it cannot name tells the reader to install +// nothing — the repair is a full entry, where the format travels along), +// while the reader TOLERATES one, because the bundled table grows +// independently of the format version and a backup written by a newer app +// must stay readable one app version back. +// +// How this can fail: drop the writer-side bundled check (first case goes +// green and a typo'd installed key ships, silently installing nothing), or +// "fix" the asymmetry by refusing unknown keys on read (second case red, +// and every forward-written backup with it). +func TestPropertyDictionary_InstalledDiscipline(t *testing.T) { + t.Run("the writer refuses a key its table cannot name", func(t *testing.T) { + _, err := MarshalPropertyDictionary(&PropertyDictionary{Installed: []string{"notABundledKey"}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "properties", "the error names the repair: a full entry") + }) + t.Run("the reader tolerates one", func(t *testing.T) { + got, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"installed":["aKeyFromANewerApp"]}`)) + require.NoError(t, err) + assert.Equal(t, []string{"aKeyFromANewerApp"}, got.Installed) + }) +} + +// Marshal never emits what its own Unmarshal rejects (§11 I1): an entry key +// with no written form, and a format outside the enum, both fail the export +// with the fault named rather than shipping a file that validates nowhere. +// +// How this can fail: drop the isWritablePropertyKey guard (the control +// character ships and the schema's pattern refuses the file on read), or +// make dictionaryEntryOmap fall back to "text" for an unnameable format (a +// permanent silent format rewrite — the disease §2d killed). +func TestPropertyDictionary_MarshalRefusesTheUnwritable(t *testing.T) { + t.Run("a key with a control character", func(t *testing.T) { + _, err := MarshalPropertyDictionary(&PropertyDictionary{Properties: []PropertyDefinition{ + {Key: "bad\x00key", Format: model.RelationFormat_longtext}, + }}) + require.Error(t, err) + }) + t.Run("a format outside the enum", func(t *testing.T) { + _, err := MarshalPropertyDictionary(&PropertyDictionary{Properties: []PropertyDefinition{ + {Key: "custom", Format: model.RelationFormat(999)}, + }}) + require.Error(t, err) + assert.NotContains(t, strings.ToLower(err.Error()), `"text"`, "no fallback spelling") + }) +} + +// §11 I1 on the shortest possible path: what MarshalPropertyDictionary +// writes, UnmarshalPropertyDictionary must be able to read. `max_count` went +// out through setNonEmpty unchecked while the schema bounds it to a +// non-negative int32, so a negative or oversized value produced bytes this +// same file refuses. +// +// Reachable rather than theoretical: the value is whatever the caller holds, +// and a relation's stored relationMaxCount is an untrusted number like any +// other detail. +// +// How this can fail: put setNonEmpty back without the bound and the two +// cases below marshal cleanly into a document that fails its own reader. +func TestPropertyDictionary_MaxCountStaysWithinWhatItCanRead(t *testing.T) { + for name, count := range map[string]int64{ + "negative": -1, + "beyond int32": 1 << 40, + } { + t.Run(name, func(t *testing.T) { + // given + d := &PropertyDictionary{Properties: []PropertyDefinition{{ + Key: "estimated_hours", Format: model.RelationFormat_number, MaxCount: count, + }}} + + // when + _, err := MarshalPropertyDictionary(d) + + // then + require.Error(t, err, "an entry that cannot be read back is not an entry") + assert.Contains(t, err.Error(), "max_count") + }) + } + + t.Run("an ordinary bound still writes and reads back", func(t *testing.T) { + // given + d := &PropertyDictionary{Properties: []PropertyDefinition{{ + Key: "estimated_hours", Format: model.RelationFormat_number, MaxCount: 1, + }}} + + // when + data, err := MarshalPropertyDictionary(d) + require.NoError(t, err) + back, err := UnmarshalPropertyDictionary(data) + + // then + require.NoError(t, err, "I1: what Marshal writes, Unmarshal reads") + require.Len(t, back.Properties, 1) + assert.Equal(t, int64(1), back.Properties[0].MaxCount) + }) +} diff --git a/pkg/lib/anyblockjson/dictionarywarn_test.go b/pkg/lib/anyblockjson/dictionarywarn_test.go new file mode 100644 index 0000000000..0ad89202e3 --- /dev/null +++ b/pkg/lib/anyblockjson/dictionarywarn_test.go @@ -0,0 +1,387 @@ +package anyblockjson + +// dictionarywarn_test.go — the property dictionary spells a property the way +// every other slot in the format does (§2f): the display name for a bundled +// key, the stored key verbatim for a space-minted one — and the bundled name +// table it leans on stays unambiguous. + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" +) + +const dictHead = `"$schema":"https://schemas.anytype.io/anyblock/2/properties.schema.json","version":2,` + +func readDict(t *testing.T, doc string) (*PropertyDictionary, []Issue) { + t.Helper() + var warns []Issue + d, err := UnmarshalPropertyDictionaryWarn([]byte(doc), func(i Issue) { warns = append(warns, i) }) + require.NoError(t, err) + return d, warns +} + +// The dictionary spells a bundled property the way every document slot does: +// by its display name. One spelling for one concept — an object document +// says "Due date" in its `properties` map and the dictionary beside it says +// "Due date" in `installed`. +// +// How this can fail: emit the stored key here and the dictionary becomes the +// odd file out; spell the name on the way out without inverting it on the +// way in and `installed` names nothing the bundled table has. +func TestDictionary_SpellsPropertiesTheWayDocumentsDo(t *testing.T) { + d, warns := readDict(t, `{`+dictHead+`"installed":["Due date","Creation date"]}`) + + assert.Equal(t, []string{"dueDate", "createdDate"}, d.Installed, + "read as the stored keys they name — the wire spelling is the name, the codec keeps stored keys") + assert.Empty(t, warns, "the canonical spelling must be silent") + + out, err := MarshalPropertyDictionary(d) + require.NoError(t, err) + assert.Contains(t, string(out), `"Due date"`) + assert.NotContains(t, string(out), `"dueDate"`, "the stored key must not survive a round trip") +} + +// A stored key still names itself — an exact match wins before folding is +// consulted, which is the ladder every slot in the format follows. And the +// pre-change derived slug (`due_date`) still resolves through the fold +// layer with no compatibility table: ToSnake only inserts `_` and +// lowercases, so the old slug sits in its stored key's fold class. Both +// re-render to the canonical name. +func TestDictionary_LegacySpellingsStillNameTheirProperty(t *testing.T) { + for _, legacy := range []string{"dueDate", "due_date"} { + t.Run(legacy, func(t *testing.T) { + d, warns := readDict(t, `{`+dictHead+`"installed":["`+legacy+`"]}`) + + assert.Equal(t, []string{"dueDate"}, d.Installed) + assert.Empty(t, warns) + + out, err := MarshalPropertyDictionary(d) + require.NoError(t, err) + assert.Contains(t, string(out), `"Due date"`, "re-rendering settles on the canonical spelling") + }) + } +} + +// An entry is how a bundle declares a property, and its key population is +// MIXED: a bundled key gets its display name, a space-minted one is a bson +// id and must survive verbatim — the dictionary has no legend, so its +// spelling must be a pure function of the key, and the only pure spelling a +// space-minted key has is itself. +func TestDictionary_AnEntryKeyIsNamedOnlyWhenItIsBundled(t *testing.T) { + t.Run("a bundled key travels as its name", func(t *testing.T) { + d, warns := readDict(t, `{`+dictHead+ + `"properties":[{"property":"Due date","name":"Due date","format":"date"}]}`) + + require.Len(t, d.Properties, 1) + assert.EqualValues(t, "dueDate", d.Properties[0].Key, + "the name names the bundled property, so the entry defines THAT property") + assert.Empty(t, warns) + }) + + t.Run("a space-minted key survives verbatim", func(t *testing.T) { + const bson = "6a32d4856761631534b22f85" + d, warns := readDict(t, `{`+dictHead+ + `"properties":[{"property":"`+bson+`","name":"Aroma notes","format":"text"}]}`) + + require.Len(t, d.Properties, 1) + assert.EqualValues(t, bson, d.Properties[0].Key) + assert.Empty(t, warns) + + out, err := MarshalPropertyDictionary(d) + require.NoError(t, err) + assert.Contains(t, string(out), `"`+bson+`"`, "nothing is ever derived from a bson id") + }) +} + +// `installed` names rows to restore from the bundled table, so a key outside +// it tells a reader to install nothing. It is TOLERATED rather than refused, +// and the tolerance is about VERSION SKEW, not custom properties: the bundled +// table grows independently of the format version, so a backup written by a +// newer app lists keys this build has never heard of, and refusing them would +// make every backup unreadable one app version back. +// +// How this can fail: turn the warning into an error and a newer app's backup +// stops reading; drop the warning and a bundle that installs nothing for a +// property ships with a clean bill of health. +func TestDictionary_AKeyFromANewerAppIsToleratedAndReported(t *testing.T) { + d, warns := readDict(t, `{`+dictHead+`"installed":["some_key_this_build_has_never_heard_of"]}`) + + assert.Equal(t, []string{"some_key_this_build_has_never_heard_of"}, d.Installed, + "kept verbatim — it may be the newer app's") + require.Len(t, warns, 1) + assert.Contains(t, warns[0].Message, "installs NOTHING for it") + assert.Contains(t, warns[0].Message, "newer app", "the tolerance is explained, not just the fault") +} + +// UnmarshalPropertyDictionary is UnmarshalPropertyDictionaryWarn with no +// sink: the same verdicts, the warnings discarded — the relationship Validate +// and ValidateWarn have. +func TestDictionary_TheSinklessDoorAgrees(t *testing.T) { + doc := `{` + dictHead + `"installed":["Due date"]}` + + quiet, err := UnmarshalPropertyDictionary([]byte(doc)) + require.NoError(t, err) + loud, _ := readDict(t, doc) + assert.Equal(t, loud, quiet) +} + +// The whole design rests on one fact: a bundled property's wire spelling +// names it UNIQUELY. If two bundled keys ever spelled or folded together, a +// dictionary spelling either of them would be undecidable, and the format +// would have to go back to storing camelCase. +// +// The spelling under guard is the NAME-AWARE one (bundledname.go): a key +// whose display name uniquely inverts spells the name, everything else its +// stored key. This is not a test of today's table — it is the condition +// under which a NEW bundled relation may be added: if adding one breaks +// this, the name needs to change (the audioGenre "Genre" → "Audio genre" +// rename is exactly that), not the dictionary loosened. +// +// How this can fail: name a new bundled relation "Due date", or "Due-Date" +// (one fold class with the existing name) — this fails naming the pair. +func TestDictionaryKeys_TheBundledTableStaysUnambiguous(t *testing.T) { + keys := bundledRelationKeys() + require.NotEmpty(t, keys) + + for spelling, owners := range keysBySpelling(keys, dictionaryKeySpelling) { + assert.Lenf(t, owners, 1, + "bundled properties %v all spell as %q — a dictionary naming it could mean any of them", + owners, spelling) + } + for fold, owners := range keysBySpelling(keys, func(k string) string { + return FoldKeyTerm(dictionaryKeySpelling(k)) + }) { + assert.Lenf(t, owners, 1, + "bundled properties %v all FOLD to %q — a near-miss spelling could not be recovered", + owners, fold) + } + + // and every spelling names its own key back, through the very lookup the + // dictionary reader uses (StoredX(SpellingX(k)) == k is the round-trip + // half of the guard) + for _, key := range keys { + spelling := dictionaryKeySpelling(key) + stored, ambiguous := dictionaryStoredKey(spelling) + assert.Emptyf(t, ambiguous, "spelling %q of %q is ambiguous", spelling, key) + assert.Equalf(t, key, stored, "spelling %q must name %q back", spelling, key) + } + + // the same round trip through the vocabulary door the codec uses — the + // two readers must agree on every bundled key. A key that spells ITSELF + // (its name is shared, unwritable or absent) resolves verbatim — the + // vocabulary answers "not a slug" and the chain treats the term as the + // stored key it is. + for _, key := range keys { + spelling := (BundledKeyVocabulary{}).PropertySlug(key) + back, ok := (BundledKeyVocabulary{}).PropertyKey(spelling) + if spelling == key { + assert.Equalf(t, key, back, "verbatim spelling %q must pass through", spelling) + continue + } + require.Truef(t, ok, "the vocabulary must invert its own spelling %q", spelling) + assert.Equalf(t, key, back, "vocabulary spelling %q must name %q back", spelling, key) + } + + // A guard that cannot fail is not a guard: this is the collision the real + // table must never contain, and the detector must see it. + t.Run("the detector sees a planted collision", func(t *testing.T) { + sameName := func(string) string { return "Genre" } + planted := keysBySpelling([]string{"genre", "audioGenre"}, sameName) + require.Len(t, planted["Genre"], 2, + "two keys spelling alike must land in one bucket") + + folded := keysBySpelling([]string{"gitHubStars", "githubStars"}, func(k string) string { + return FoldKeyTerm(k) + }) + require.Len(t, folded["githubstars"], 2, + "two keys folding alike must land in one bucket") + }) +} + +// keysBySpelling groups keys by the spelling a reader would see. +func keysBySpelling(keys []string, spell func(string) string) map[string][]string { + out := map[string][]string{} + for _, key := range keys { + s := spell(key) + out[s] = append(out[s], key) + } + return out +} + +// The same guard for TYPES. A bundled type's spelling is what the manifest +// keys on and what a dictionary entry's `object_types` names, so an ambiguity +// here would be undecidable in two places at once. +// +// How this can fail: add a bundled type whose NAME spells onto an existing +// one's — this fails naming the pair, and the new type needs a different +// name (the space "Space" → "Space settings" rename is exactly that). +func TestDictionaryKeys_TheBundledTypeTableStaysUnambiguous(t *testing.T) { + keys := make([]string, 0, 32) + for _, tk := range bundle.ListTypesKeys() { + keys = append(keys, tk.String()) + } + require.NotEmpty(t, keys) + + for spelling, owners := range keysBySpelling(keys, TypeKeySpelling) { + assert.Lenf(t, owners, 1, + "bundled types %v all spell as %q — a manifest key or an object_types member "+ + "naming it could mean any of them", owners, spelling) + } + // the SPELLING folds stay unique. The full fold TABLE holds one known + // collision beyond them — the name "Space" (spaceView) folds onto the + // stored key `space` — which only degrades the forgiving layer for + // near-misses of that one pair, both measured at 0 documents; the + // spellings themselves stay exact and unambiguous, which is what this + // guard protects. + for fold, owners := range keysBySpelling(keys, func(k string) string { + return FoldKeyTerm(TypeKeySpelling(k)) + }) { + assert.Lenf(t, owners, 1, "bundled types %v all FOLD to %q", owners, fold) + } + // StoredTypeKey(TypeKeySpelling(k)) == k — for every key + for _, key := range keys { + assert.Equalf(t, key, StoredTypeKey(TypeKeySpelling(key)), + "the spelling of type %q must name it back", key) + } + // and through the vocabulary door the codec uses + for _, key := range keys { + spelling := (BundledKeyVocabulary{}).TypeSlug(key) + back, ok := (BundledKeyVocabulary{}).TypeKey(spelling) + if spelling == key { + assert.Equalf(t, key, back, "verbatim spelling %q must pass through", spelling) + continue + } + require.Truef(t, ok, "the vocabulary must invert its own spelling %q", spelling) + assert.Equalf(t, key, back, "vocabulary spelling %q must name type %q back", spelling, key) + } +} + +// One spelling for one concept, across the two slots that name a type outside +// a document: a dictionary entry's target types and the bundle manifest. +// +// A type DOCUMENT reaches the same answer by a different road — it spells +// through the exporter's per-document ledger and binds the term in that +// document's own `type_internal_keys` legend. The dictionary and the manifest +// have no legend, so their spelling has to be a pure function of the key. +func TestDictionary_TargetTypesSpellLikeEverythingElse(t *testing.T) { + d, warns := readDict(t, `{`+dictHead+ + `"properties":[{"property":"Assignee","name":"Assignee","format":"objects",`+ + `"object_types":["Space member","object_type","6a83296f61fab2265263ae34"]}]}`) + require.Empty(t, warns) + require.Len(t, d.Properties, 1) + + assert.Equal(t, []string{"participant", "objectType", "6a83296f61fab2265263ae34"}, + d.Properties[0].ObjectTypes, + "a name resolves to its stored type key, a legacy slug through the fold, a minted key stays verbatim") + + out, err := MarshalPropertyDictionary(d) + require.NoError(t, err) + assert.Contains(t, string(out), `"Space member"`, "written back in the format's spelling") + assert.Contains(t, string(out), `"Type"`, "objectType's name") + assert.NotContains(t, string(out), `"objectType"`) + assert.Contains(t, string(out), `"6a83296f61fab2265263ae34"`, "and a minted key is never renamed") +} + +// bundledRelationKeys lists every relation key this build's bundled table +// holds, read from the table itself rather than a hand-kept list. +func bundledRelationKeys() []string { + urls := bundle.ListRelationsUrls() + out := make([]string, 0, len(urls)) + for _, url := range urls { + key := url + if i := strings.LastIndex(url, "/"); i >= 0 { + key = url[i+1:] + } + out = append(out, strings.TrimPrefix(key, "_br")) + } + return out +} + +// An entry may state a `property` spelling and an `internal_key`, and export +// writes both from ONE stored key — so in anything this package produced they +// agree and the precedence never matters. An AUTHOR can make them disagree, +// and then neither order is safe: the spelling is what the document's own +// values resolve through, while the internal_key is the exact key the entry +// claims to define. Honouring either silently leaves the other pointing +// somewhere else. +// +// So the disagreement is reported. The spelling still wins — the code's +// comment used to say the opposite of what authoredKey does, which is the +// contradiction this pins shut. +// +// How this can fail: let the pair disagree in silence and a type's recommended +// list points at a property no value in the document uses. +func TestDictionary_ADisagreeingIdentityPairIsReported(t *testing.T) { + t.Run("they disagree", func(t *testing.T) { + d, warns := readDict(t, `{`+dictHead+ + `"properties":[{"property":"Due date","internal_key":"6a32d4856761631534b22f85",`+ + `"name":"Due date","format":"date"}]}`) + + require.Len(t, warns, 1) + assert.Contains(t, warns[0].Message, "name different properties") + assert.EqualValues(t, "dueDate", d.Properties[0].Key, + "the spelling wins, as authoredKey has always done") + }) + + t.Run("they agree — the pair export writes", func(t *testing.T) { + _, warns := readDict(t, `{`+dictHead+ + `"properties":[{"property":"Due date","internal_key":"dueDate",`+ + `"name":"Due date","format":"date"}]}`) + assert.Empty(t, warns, "the agreeing pair is the normal case and must be silent") + }) + + t.Run("only one stated", func(t *testing.T) { + for _, entry := range []string{ + `{"property":"Due date","format":"date"}`, + `{"internal_key":"6a32d4856761631534b22f85","format":"date"}`, + } { + _, warns := readDict(t, `{`+dictHead+`"properties":[`+entry+`]}`) + assert.Empty(t, warns, entry) + } + }) +} + +// An inline option says what the option MEANS — its name, its colour, and by +// its position where it sits. Its stored key says what the option IS, and +// that is the one thing about it derivable from nothing. +// +// Everything else about an option can be reconstructed. The api key is +// regenerated from the name by the app's own rule at creation: measured over +// a 77-space export, all 514 real option api keys are reproduced by it — 470 +// by the api slug and 44 by the transliterate fallback, for names like `$$` +// that slug to nothing. Not one survived a rename, so none needs to travel. +// The order is the array position (§2f). The property is the entry holding it. +// +// So `internal_key` is what an inline vocabulary was missing to be complete +// rather than merely descriptive. +// +// How this can fail: render it on an option that has none and the compact +// bare-name form disappears for every colourless option; drop it from the +// object form and a vocabulary can never state identity. +func TestDictionary_AnOptionCarriesItsStoredKey(t *testing.T) { + d, warns := readDict(t, `{`+dictHead+ + `"properties":[{"property":"Status","name":"Status","format":"select","options":[`+ + `{"name":"To Do","color":"ice","internal_key":"63454ad0c493f68e301890db"},`+ + `{"name":"Done","color":"lime"},`+ + `"Someday"]}]}`) + require.Empty(t, warns) + require.Len(t, d.Properties, 1) + + opts := d.Properties[0].Options + require.Len(t, opts, 3) + assert.Equal(t, "63454ad0c493f68e301890db", opts[0].InternalKey) + assert.Empty(t, opts[1].InternalKey, "an option may carry none") + assert.Equal(t, "Someday", opts[2].Name, "and the bare-name form still means a colourless option") + + out, err := MarshalPropertyDictionary(d) + require.NoError(t, err) + assert.Contains(t, string(out), `"internal_key": "63454ad0c493f68e301890db"`) + assert.Contains(t, string(out), `"name": "To Do"`, "names with spaces survive intact") + assert.Contains(t, string(out), `"Someday"`, + "an option with neither colour nor key stays the compact bare name") +} diff --git a/pkg/lib/anyblockjson/documentkind.go b/pkg/lib/anyblockjson/documentkind.go new file mode 100644 index 0000000000..59a9f9d1f5 --- /dev/null +++ b/pkg/lib/anyblockjson/documentkind.go @@ -0,0 +1,165 @@ +package anyblockjson + +// documentkind.go — which of the format's three grammars a document follows, +// and what to say when it is handed to the wrong reader (§2c, §2f, §13). +// +// A bundle holds three species of file: object documents (including types, +// relations and templates), one bundle index, and one property dictionary. +// They are different grammars, not variants of one, and until this file +// existed nothing in a document reliably told them apart: +// +// - `version` is `const: 1` in all three schemas, so it discriminates +// nothing; +// - `$schema` is written by all three writers but is not required, so a +// hand-authored document may carry none — 9 of 9 bundle files written by +// small models against the schemas carried none; +// - which left the FILENAME, and a filename is not part of the format. A +// bundle unpacked flat, renamed, or streamed over an API has lost it. +// +// The cost was not theoretical. Handed to the object reader, a perfectly good +// index reported `/name: property "name" is not allowed` — on the field whose +// whole job is to name the space — and a dictionary reported `/properties: +// got array, want object` and `/installed: property "installed" is not +// allowed`, on its headline field. Each of those sends an author to repair a +// file that was already correct. This package's own eval harness hit it too. +// +// Requiring `$schema` would settle it, and is deliberately NOT what this +// does: the format is meant to be hand-authorable, and demanding a 60-byte +// URL as the price of a three-line document is a worse trade than reading the +// document's shape. So `$schema` stays optional — but when present it must +// name one of the three real schemas, which is what catches the invented +// `.../relation.schema.json` an author reached for. + +import ( + "encoding/json" + "strings" +) + +// DocumentKind names which of the format's three grammars data follows. +// +// It reads the declared `$schema` first — that is the author's own statement +// and outranks any inference — and falls back to shape for a document that +// declares none. It never fails: anything it cannot place is KindObject, the +// grammar that covers every document a space actually contains, so a reader +// dispatching on this behaves exactly as it did before for ordinary +// documents. +func DocumentKind(data []byte) string { + kind, _ := documentKindOf(data) + return kind +} + +// documentKindOf is DocumentKind with the part that matters to a reader +// deciding whether to OVERRIDE its caller: decided reports whether the +// document carried evidence at all. A document that declares no `$schema` +// and holds no member unique to one grammar is not evidence of anything — +// `{"version": 2}` is a legal start to all three — so a reader must take the +// caller's word for it rather than infer. +func documentKindOf(data []byte) (kind string, decided bool) { + var probe struct { + Schema string `json:"$schema"` + Installed json.RawMessage `json:"installed"` + Properties json.RawMessage `json:"properties"` + Manifest json.RawMessage `json:"manifest"` + Widgets json.RawMessage `json:"widgets"` + Entrypoint json.RawMessage `json:"entrypoint"` + } + if json.Unmarshal(data, &probe) != nil { + return KindObject, false + } + switch { + case strings.HasSuffix(probe.Schema, "/index.schema.json"): + return KindIndex, true + case strings.HasSuffix(probe.Schema, "/properties.schema.json"): + return KindPropertyDictionary, true + case strings.HasSuffix(probe.Schema, "/object.schema.json"): + return KindObject, true + } + // No declaration: infer from members no other grammar has. `properties` + // is the one member two grammars share, and they disagree about its TYPE + // — an object maps keys to values, a dictionary lists definitions — so + // the array spelling is itself a discriminator. + if len(probe.Installed) > 0 || isJSONArray(probe.Properties) { + return KindPropertyDictionary, true + } + if len(probe.Manifest) > 0 || len(probe.Widgets) > 0 || len(probe.Entrypoint) > 0 { + return KindIndex, true + } + return KindObject, false +} + +// The three grammars DocumentKind names. +const ( + KindObject = "object" + KindIndex = "index" + KindPropertyDictionary = "property_dictionary" +) + +// misroutedIssues reports a document handed to the reader for a different +// grammar. want is the grammar this reader implements. +// +// It runs BEFORE schema validation, so the author is told what the document +// is instead of being walked through the ways it fails to be something it +// never claimed to be. +func misroutedIssues(data []byte, want string) []Issue { + got, decided := documentKindOf(data) + if !decided || got == want { + return nil + } + return []Issue{{Message: "this is " + articleFor(got) + ", not " + + articleFor(want) + " — read it with " + readerFor(got) + + ". " + evidenceFor(data, got)}} +} + +func articleFor(kind string) string { + switch kind { + case KindIndex: + return "a bundle index" + case KindPropertyDictionary: + return "a property dictionary" + default: + return "an object document" + } +} + +func readerFor(kind string) string { + switch kind { + case KindIndex: + return "UnmarshalIndex" + case KindPropertyDictionary: + return "UnmarshalPropertyDictionary" + default: + return "Unmarshal or Validate" + } +} + +// evidenceFor names what the verdict was read from, so an author who +// disagrees with it can see which member decided and fix that member rather +// than guessing. +func evidenceFor(data []byte, kind string) string { + if _, schemaURL, ok := DetectFormat(data); ok && schemaURL != "" { + return "It declares `$schema: " + schemaURL + "`." + } + switch kind { + case KindIndex: + return "It declares no `$schema`, and carries members only an index has." + case KindPropertyDictionary: + return "It declares no `$schema`, and carries `installed` or a `properties` ARRAY, " + + "which an object document — where `properties` maps keys to values — cannot." + default: + return "It declares no `$schema`." + } +} + +func isJSONArray(raw json.RawMessage) bool { + for _, b := range raw { + switch b { + case ' ', '\t', '\r', '\n': + continue + case '[': + return true + default: + return false + } + } + return false +} diff --git a/pkg/lib/anyblockjson/documentkind_test.go b/pkg/lib/anyblockjson/documentkind_test.go new file mode 100644 index 0000000000..d11f93a79a --- /dev/null +++ b/pkg/lib/anyblockjson/documentkind_test.go @@ -0,0 +1,126 @@ +package anyblockjson + +// documentkind_test.go — telling the format's three grammars apart (§2c, §2f). + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A reader handed a file with no filename to go by has to place it. The +// declared `$schema` is the author's own statement and outranks inference; +// shape answers for a document that declares none. +// +// How this can fail: match `$schema` exactly instead of by suffix and every +// document written against another format version becomes unplaceable; infer +// from a member two grammars share and ordinary objects start being read as +// dictionaries. +func TestDocumentKind_PlacesTheThreeGrammars(t *testing.T) { + t.Run("the declaration is believed", func(t *testing.T) { + for _, tc := range []struct{ doc, want string }{ + {`{"$schema": "` + SchemaURL + `", "version": 2}`, KindObject}, + {`{"$schema": "` + IndexSchemaURL + `", "version": 2}`, KindIndex}, + {`{"$schema": "` + PropertiesSchemaURL + `", "version": 2}`, KindPropertyDictionary}, + } { + assert.Equal(t, tc.want, DocumentKind([]byte(tc.doc)), tc.doc) + } + }) + + // `$schema` is decorative for VALIDITY — only `version` gates the format + // — so a bundle written against a later schema still reads, and it must + // still be placeable. Matching the whole URL would break that. + t.Run("the version segment is ignored", func(t *testing.T) { + assert.Equal(t, KindIndex, DocumentKind([]byte( + `{"$schema": "https://schemas.anytype.io/anyblock/9/index.schema.json", "version": 2}`))) + }) + + t.Run("a schema nobody publishes decides nothing", func(t *testing.T) { + // an author reached for `.../relation.schema.json`, which does not + // exist; it must fall through to shape rather than be believed + assert.Equal(t, KindObject, DocumentKind([]byte( + `{"$schema": "https://schemas.anytype.io/anyblock/2/relation.schema.json", + "version": 2, "kind": "property", "internal_key": "estimate"}`))) + }) + + t.Run("shape places a document that declares nothing", func(t *testing.T) { + for _, tc := range []struct{ name, doc, want string }{ + {"installed is a dictionary's alone", `{"version": 2, "installed": ["done"]}`, KindPropertyDictionary}, + {"and so is a properties ARRAY", `{"version": 2, "properties": [{"property": "k", "format": "text"}]}`, KindPropertyDictionary}, + {"a properties MAP is an object's", `{"version": 2, "properties": {"name": "Note"}}`, KindObject}, + {"a manifest is an index's", `{"version": 2, "manifest": {"properties": "properties.json"}}`, KindIndex}, + {"so are widgets", `{"version": 2, "widgets": [{"target": "page-home"}]}`, KindIndex}, + } { + assert.Equal(t, tc.want, DocumentKind([]byte(tc.doc)), tc.name) + } + }) +} + +// Handed to the wrong reader, a correct document used to be walked through +// the ways it fails to be something it never claimed to be: an index was told +// `/name: property "name" is not allowed` — on the field whose whole job is +// to name the space — and a dictionary `/properties: got array, want object`. +// Both send an author to repair a file that is already right. +// +// How this can fail: report the misroute on a document that carries no +// evidence at all, and `{"version": 3}` stops getting the newer-format +// verdict it needs. +func TestDocumentKind_TheWrongReaderSaysSo(t *testing.T) { + index := `{"$schema": "` + IndexSchemaURL + `", "version": 2, "name": "Company Wiki"}` + dict := `{"$schema": "` + PropertiesSchemaURL + `", "version": 2, "installed": ["done"]}` + object := `{"$schema": "` + SchemaURL + `", "version": 2, "properties": {"name": "Note"}}` + + t.Run("an index read as an object", func(t *testing.T) { + err := Validate([]byte(index)) + require.Error(t, err) + assert.Contains(t, err.(*ValidationError).Issues[0].String(), "this is a bundle index") + assert.Contains(t, err.(*ValidationError).Issues[0].String(), "UnmarshalIndex") + assert.NotContains(t, err.Error(), `"name" is not allowed`, + "the old verdict blamed the field that names the space") + }) + + t.Run("a dictionary read as an object", func(t *testing.T) { + err := Validate([]byte(dict)) + require.Error(t, err) + assert.Contains(t, err.(*ValidationError).Issues[0].String(), "this is a property dictionary") + assert.NotContains(t, err.Error(), "want object") + }) + + t.Run("an object read as an index, or as a dictionary", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(object)) + require.Error(t, err) + assert.Contains(t, err.Error(), "an object document") + + _, err = UnmarshalPropertyDictionary([]byte(object)) + require.Error(t, err) + assert.Contains(t, err.Error(), "an object document") + }) + + t.Run("each grammar still reads its own", func(t *testing.T) { + require.NoError(t, Validate([]byte(object))) + _, err := UnmarshalIndex([]byte(index)) + require.NoError(t, err) + _, err = UnmarshalPropertyDictionary([]byte(dict)) + require.NoError(t, err) + }) + + // `{"version": 2}` is a legal start to all three grammars. A reader that + // guessed here would override its caller on no evidence. + t.Run("a document with no evidence is left to its caller", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version": 3}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "newer version", + "the version gate must still be what answers") + + require.NoError(t, Validate([]byte(`{"version": 2}`))) + }) + + // The identity a reader dispatches on must not become the thing that + // decides validity: a stale or invented $schema stays decorative. + t.Run("a stale schema url is still valid", func(t *testing.T) { + require.NoError(t, Validate([]byte( + `{"$schema": "https://schemas.anytype.io/anyblock/9/object.schema.json", + "version": 2, "blocks": [{"type": "paragraph", "text": "fine"}]}`))) + }) +} diff --git a/pkg/lib/anyblockjson/dropcensus_test.go b/pkg/lib/anyblockjson/dropcensus_test.go new file mode 100644 index 0000000000..9eec5db240 --- /dev/null +++ b/pkg/lib/anyblockjson/dropcensus_test.go @@ -0,0 +1,161 @@ +package anyblockjson + +// dropcensus_test.go — the term census must not reserve a spelling for a +// detail key the properties emit is going to DROP. +// +// A dropped key is written nowhere, so a generation-2 census cannot hold +// it. If generation 1 let it contest a spelling anyway, the rival it +// suffixed un-suffixes on the next export and the round trip stops being a +// fixpoint. The attribution keys were the only population modelled; four +// more drop inside buildProperties, and each gets an arm here. +// +// Every arm has the same shape: a bundled key that WILL be dropped, a +// custom property carrying the same display name, and the assertion that +// the custom property keeps its plain name in generation 1 and in +// generation 2 — byte-identical output across the round trip. + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +// droppedKeyCase is one drop population: the snapshot that carries both the +// dropped key and its custom name-twin, plus what the twin must be spelled. +type droppedKeyCase struct { + name string + sbType model.SmartBlockType + droppedKey string + sharedName string + extraFields map[string]*types.Value + // writtenVerbatim marks the one population export DOES write — the + // attribution keys, whose value is written under the stored key and + // then dropped by import. The two generations differ by that one line + // and cannot be byte-compared; every other population is written + // nowhere, so its two generations must be identical. + writtenVerbatim bool +} + +func TestDroppedPropertyKeyNeverContestsASpelling(t *testing.T) { + const twin = "6a7663db61fab21cd4b90aa1" + + cases := []droppedKeyCase{{ + // DroppedEmptySystemProperty: `isHidden: false` says nothing a + // reader could act on, so it is not written + name: "an empty system-stamped flag", + sbType: model.SmartBlockType_Page, + droppedKey: "isHidden", + sharedName: "Hidden", + extraFields: map[string]*types.Value{ + "isHidden": pbtypes.Bool(false), + }, + }, { + // typeProvenanceKeys: a type document does not carry its own + // install provenance + name: "a type document's install provenance", + sbType: model.SmartBlockType_STType, + droppedKey: "origin", + sharedName: "Origin", + extraFields: map[string]*types.Value{ + "origin": pbtypes.Int64(3), + "uniqueKey": pbtypes.String("ot-custom"), + "type": pbtypes.String("objectType"), + "layout": pbtypes.Int64(int64(model.ObjectType_objectType)), + "recommendedLayout": pbtypes.Int64(int64(model.ObjectType_basic)), + }, + }, { + // DroppedParticipantProvenanceKey: a participant's createdDate is a + // load timestamp, re-stamped on every cold build + name: "a participant's load timestamp", + sbType: model.SmartBlockType_Participant, + droppedKey: "createdDate", + sharedName: "Creation date", + extraFields: map[string]*types.Value{ + "createdDate": pbtypes.Int64(1700000000), + "identity": pbtypes.String("A5qTLyde3S1q9NRyFeSeN6UWwa6VwwXEJbMACJwMfez3BGVD"), + }, + }, { + // a name-over-number key holding a string its vocabulary cannot + // name: there is no way to write it, so it is dropped + name: "an unnameable named-enum value", + sbType: model.SmartBlockType_Page, + droppedKey: "layoutAlign", + sharedName: "Layout align", + extraFields: map[string]*types.Value{ + "layoutAlign": pbtypes.String("not a name this vocabulary holds"), + }, + }, { + // the population that was already modelled, kept here so all five + // are pinned in one place. This one is WRITTEN — under its stored + // key — and dropped on the way back in. + name: "an attribution key", + sbType: model.SmartBlockType_Page, + droppedKey: "creator", + sharedName: "Created by", + extraFields: map[string]*types.Value{ + "creator": pbtypes.String("_participant_a_b_A5qTLyde3S1q9NRyFeSeN6UWwa6VwwXEJbMACJwMfez3BGVD"), + }, + writtenVerbatim: true, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // given — the dropped key's name-twin is a live custom property + vocab := nameVocab{names: map[string]string{twin: tc.sharedName}} + fields := map[string]*types.Value{ + "id": pbtypes.String("o1"), + twin: pbtypes.String("the twin's value"), + "name": pbtypes.String("An object"), + } + for k, v := range tc.extraFields { + fields[k] = v + } + snap := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: fields}} + opts := Options{Keys: vocab, SpaceId: "a.b"} + + // when + data, err := Marshal(tc.sbType, snap, opts) + require.NoError(t, err) + + // then — the twin keeps the plain name it will re-derive once + // the dropped key is gone + require.NoError(t, Validate(data), "I1:\n%s", data) + var doc struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "the twin's value", doc.Properties[tc.sharedName], + "a key the emit drops must not degrade the rival it contests") + if tc.writtenVerbatim { + assert.Contains(t, doc.Properties, tc.droppedKey, + "a yielding claimant is written under its own stored key") + } else { + assert.NotContains(t, doc.Properties, tc.droppedKey, + "the dropped key claims no spelling of its own") + } + + // the fixpoint: generation 2 sees no dropped key at all, so a + // spelling it had contested would move + _, back, err := Unmarshal(data, opts) + require.NoError(t, err) + again, err := Marshal(tc.sbType, back, opts) + require.NoError(t, err) + var doc2 struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(again, &doc2)) + assert.Equal(t, "the twin's value", doc2.Properties[tc.sharedName], + "generation 2 re-derives the same spelling — the fixpoint the rule exists for") + if !tc.writtenVerbatim { + assert.Equal(t, string(data), string(again), + "a key written nowhere leaves the two generations byte-identical") + } + }) + } +} diff --git a/pkg/lib/anyblockjson/exhaustivelegend_test.go b/pkg/lib/anyblockjson/exhaustivelegend_test.go new file mode 100644 index 0000000000..0bdbc3d06f --- /dev/null +++ b/pkg/lib/anyblockjson/exhaustivelegend_test.go @@ -0,0 +1,254 @@ +package anyblockjson + +// exhaustivelegend_test.go — the legend names every spelling the bundled +// table cannot speak for, including the ones written verbatim. +// +// The old rule asked the bundled table to INVERT the term. A table that does +// not know a term answers the term itself (chain step 4), so every custom key +// spelled verbatim "inverted" trivially and owed nothing — and the document +// said nothing at all about the one population no reader can resolve without +// it. The key is unambiguous the day it is written; the loss happens later, +// when the relation is deleted and the freed spelling becomes some other +// relation's api key. Every document already written then re-points, offline, +// with nothing in it to say otherwise. That is the corpse-AFTER-export hole, +// and only the writer's own document can close it, because the writer had +// nothing to warn about at the time. + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The hole, end to end. The writer has NO vocabulary that binds `initiative` +// — nothing to warn about, nothing to record under the old rule — and the +// reader is a space where the relation has since been deleted and its +// spelling reassigned. +func TestExport_AVerbatimCustomKeyNamesItself(t *testing.T) { + // given — an object holding a space-local relation keyed `initiative`, + // exported by a package-only writer + snap := customKeySnapshot(map[string]*types.Value{ + "initiative": str("value of the relation that was deleted later"), + "dueDate": str("2026-07-06T08:44:05Z"), + }) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + + // then — the entry is there, and only for the term the bundled table + // cannot speak for: `dueDate` is spelled "Due date", which every reader's + // table binds, so it owes nothing + require.NoError(t, err) + require.NoError(t, Validate(data)) + doc := decodeDoc(t, data) + assert.Equal(t, map[string]string{"initiative": "initiative"}, doc.PropertyKeys) + + // and the reader that binds the spelling elsewhere is overruled by the + // document. corpseVocabulary is fully conforming — the writer could not + // have been warned, and was not running it + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: corpseVocabulary{}}) + require.NoError(t, err) + assert.Equal(t, "value of the relation that was deleted later", + back.Details.Fields["initiative"].GetStringValue(), + "without the entry this value lands on "+corpsePropKey+", silently") + assert.NotContains(t, back.Details.Fields, corpsePropKey) + assert.Contains(t, back.Details.Fields, "dueDate", + "a bundled key still needs no entry: `due_date` is bound by every reader's table") +} + +// The type namespace, same hole, with the loss that costs more: an object's +// TYPE. Nothing in a package-only export knows `initiative` will be someone +// else's slug tomorrow. +func TestExport_AVerbatimCustomTypeKeyNamesItself(t *testing.T) { + // given + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{"id": str("o1")}), + ObjectTypes: []string{"ot-initiative"}, + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data)) + var doc struct { + Type string `json:"type"` + TypeKeys map[string]string `json:"type_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "initiative", doc.Type) + assert.Equal(t, map[string]string{"initiative": "initiative"}, doc.TypeKeys) + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: corpseVocabulary{}}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-initiative"}, back.ObjectTypes, + "without the entry the object comes back typed as ot-"+customTypeKey) +} + +// The exact boundary of "exhaustive": a BUNDLED key spelled as its display +// name still owes nothing, because the table that binds it ships with every +// reader. Without this the rule would be "emit everything", the legend would +// double the size of an ordinary document, and nothing would be bought. +func TestExport_ABundledSpellingStillOwesNothing(t *testing.T) { + snap := customKeySnapshot(map[string]*types.Value{ + "dueDate": str("2026-07-06T08:44:05Z"), + "pluralName": str("A"), + }) + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + + doc := decodeDoc(t, data) + assert.Contains(t, doc.Properties, "Due date") + assert.Contains(t, doc.Properties, "Plural name") + assert.Empty(t, doc.PropertyKeys, + "every spelling here is one the bundled table binds to the key it came from") +} + +// decoyVocabulary is the corpse policy taken to its limit: it binds EVERY +// spelling the bundled table does not know to a decoy stored key, in both +// namespaces. It conforms — it spells nothing, it never touches a spelling +// the bundled table binds, and it refuses spellings the format could not +// write anyway — so §11.1's preconditions do not exclude it, and a document +// that survives it survives any space that reassigned any of its spellings. +type decoyVocabulary struct{} + +const decoyPrefix = "decoy-" + +func (decoyVocabulary) PropertySlug(key string) string { return key } +func (decoyVocabulary) TypeSlug(key string) string { return key } + +func (decoyVocabulary) PropertyKey(slug string) (string, bool) { + if key, ok := (BundledKeyVocabulary{}).PropertyKey(slug); ok { + return key, true + } + if !isWritablePropertyKey(slug) { + return slug, false + } + return decoyPrefix + slug, true +} + +func (decoyVocabulary) TypeKey(slug string) (string, bool) { + if key, ok := (BundledKeyVocabulary{}).TypeKey(slug); ok { + return key, true + } + if !isWritablePropertyKey(slug) { + return slug, false + } + return decoyPrefix + slug, true +} + +// The corpus form of the rule, and the one that keeps it exhaustive as the +// format grows: over the whole hostile corpus, a reader that has reassigned +// every non-bundled spelling in the document resolves the SAME stored keys as +// a reader with no vocabulary at all. It can only do that by reading the +// legend, because that is the one thing it is told to consult first. +// +// The comparison is against the package-only reader rather than against the +// snapshot, deliberately: export legitimately drops keys (unwritable ones, +// stripped ones, blocks it does not model), and this invariant is about the +// legend's completeness, not about what export chooses to carry. +func TestInvariant_TheLegendSurvivesAReaderThatReassignedEverySpelling(t *testing.T) { + reassigned := 0 + for n := 0; n < 300; n++ { + sbType, snap := hostileSnapshot(n) + o := Options{ResolveOptions: hostileOptions} + if sbType == model.SmartBlockType_STType { + o.ResolveProperties = hostileTypePropResolver{} + } + data, err := Marshal(sbType, snap, o) + if err != nil { + continue + } + _, plain, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "seed %d produced:\n%s", n, data) + _, decoyed, err := Unmarshal(data, + Options{GenerateId: seqIds("g"), Keys: decoyVocabulary{}}) + require.NoError(t, err, "seed %d produced:\n%s", n, data) + + require.Equal(t, keyCensus(plain), keyCensus(decoyed), + "seed %d: a reader that reassigned every non-bundled spelling read "+ + "different stored keys out of:\n%s", n, data) + for _, k := range keyCensus(plain) { + if strings.HasPrefix(k, decoyPrefix) { + t.Fatalf("seed %d: the decoy leaked into the package-only read: %s", n, k) + } + } + reassigned += len(keyCensus(plain)) + } + // the corpus has to REACH the rule, or a green sweep says nothing. Every + // custom key in every document is one this vocabulary would reassign. + require.Greater(t, reassigned, 500, + "the corpus stopped producing keys for this invariant to be about") +} + +// keyCensus lists every stored key an imported snapshot names, at every slot +// the legend covers, sorted. It is the observable the invariant compares: two +// reads of one document must name the same relations and the same types. +func keyCensus(snap *model.SmartBlockSnapshotBase) []string { + seen := map[string]struct{}{} + add := func(k string) { + if !strings.HasSuffix(k, ":") { + seen[k] = struct{}{} + } + } + if snap.Details != nil { + for k := range snap.Details.Fields { + add("detail:" + k) + } + } + for _, t := range snap.ObjectTypes { + add("type:" + t) + } + for _, b := range snap.Blocks { + if b == nil { + continue + } + switch c := b.Content.(type) { + case *model.BlockContentOfRelation: + add("block:" + orEmpty(c.Relation).Key) + case *model.BlockContentOfLink: + for _, k := range orEmpty(c.Link).Relations { + add("link:" + k) + } + case *model.BlockContentOfDataview: + dv := orEmpty(c.Dataview) + for _, rl := range dv.RelationLinks { + if rl != nil { + add("dv:" + rl.Key) + } + } + for _, v := range dv.Views { + if v == nil { + continue + } + add("group:" + v.GroupRelationKey) + add("cover:" + v.CoverRelationKey) + add("end:" + v.EndRelationKey) + for _, r := range v.Relations { + if r != nil { + add("col:" + r.Key) + } + } + for _, s := range v.Sorts { + if s != nil { + add("sort:" + s.RelationKey) + } + } + for _, f := range flattenFilters(v.Filters) { + add("filter:" + f.RelationKey) + } + } + } + } + return sortedKeys(seen) +} diff --git a/pkg/lib/anyblockjson/export.go b/pkg/lib/anyblockjson/export.go new file mode 100644 index 0000000000..de114376bf --- /dev/null +++ b/pkg/lib/anyblockjson/export.go @@ -0,0 +1,2858 @@ +package anyblockjson + +// export.go serializes a snapshot into canonical AnyBlock JSON (§2–§7, +// §9–§9a). + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// FormatResolver reports the format of a property key, when known. Bundle +// properties are resolved internally; the resolver covers custom keys (§3). +type FormatResolver func(key domain.RelationKey) (model.RelationFormat, bool) + +// OptionResolver maps select/multiSelect option ids to names on export and +// names to ids on import (creating options is the import wiring's job, §3). +// +// OptionName has TWO duties, and the second one is not an export call: +// +// 1. export — what is this option id called? The name is what the document +// writes for the value (§3), and the id it stood for rides along in the +// `option_ids` legend (§9a). +// 2. import — is this id a live option of this relation HERE? That is the +// liveness question every `option_ids` entry is checked against +// (optionrefs.go): the legend is a hint, honoured only for an id the +// target space still serves under that key, and OptionName answering is +// precisely what "still serves" means. Nothing else asks it, so a +// resolver's answer here is the whole of the check. +// +// A resolver that cannot answer OptionName gives up the legend entirely: it +// says "no id is live", so every entry fails step 1 of §3's chain and every +// value falls back to name resolution, exactly as it did before `option_ids` +// existed — including the two losses the legend was added to close, a name +// shared by two options of one property (the first one answers) and an option +// renamed since the export (nothing answers, and the wiring mints a second +// option under the stale name). That is a legitimate position for a resolver +// with no option store to consult, and returning false is then the honest +// answer; it is not a stub to leave in place unexamined, because it disables +// a feature for everything that reader imports, silently. `OptionId` without +// `OptionName` is the shape to look at twice. +type OptionResolver interface { + OptionName(key domain.RelationKey, id string) (string, bool) + OptionId(key domain.RelationKey, name string) (string, bool) +} + +// ParticipantResolver names the space member a participant id stands for. +// The derived attribution properties — `creator` and `lastModifiedBy` — are +// written as the member's RESOLVABLE id with the name riding as the +// informative `#name` suffix: `#` (§3, §9). +// +// The id is the primary content and the name is a caption, which is the +// general §9 reference shape and a deliberate reversal of the earlier +// name-only spelling. Name-only broke the API v2 contract — a consumer that +// wants the author's avatar or profile needs an id to resolve, and two +// members sharing a display name are indistinguishable by it (76 of 2,478 +// production participants share one). The participant fold keeps the +// readable half honest: the id is ~48 characters, not the 135 the composite +// was. +// +// It has ONE direction on purpose: there is no `ParticipantId(name)`. A +// display name is a label, not an address, so nothing could invert it +// honestly — and nothing needs to. Both properties are `source: derived`, +// `maxCount: 1`, `readonly: true`: their value is recovered from the object +// tree root's own signature on every rebuild (`treeSource.GetCreationInfo`), +// and import DROPS both keys whatever they carry (§3). +// +// A resolver that cannot answer returns false and the id is written bare — +// resolvable either way, just without the caption. Nil resolver, same +// answer, everywhere. +type ParticipantResolver interface { + ParticipantName(id string) (string, bool) +} + +// Options configures Marshal and Unmarshal (§13). +type Options struct { + ResolveFormat FormatResolver // optional; nil = bundle-only resolution (§3) + ResolveOptions OptionResolver // optional; nil = option values pass through as ids + ResolveProperties PropertyResolver // optional; nil = type documents keep raw recommended-relation ids (§2a) + // ResolveParticipants names the member behind a participant id, for the + // derived attribution properties only (export; nil = `creator` and + // `lastModifiedBy` are omitted, §3). + ResolveParticipants ParticipantResolver + // ResolveObjectNames names the object behind a reference, for the + // informative `#name` suffix only (export, behind RefNames; nil = every + // reference is written bare, §9). Import never consults it — the suffix + // is trimmed unread. + ResolveObjectNames ObjectNameResolver + // SpaceId is the space this codec run reads from or writes into — the + // wiring supplies it exactly as it supplies the resolvers. It enables + // the participant fold (§9): export folds + // `_participant__` to the bare identity, and import + // rebuilds the composite against this space. Empty disables the fold in + // BOTH directions: a composite id passes through verbatim and a bare + // identity is left alone, because folding on export without the paired + // import being able to rebuild would land a bare identity in a snapshot + // slot where a composite belongs — silent corruption of exactly the slot + // the fold exists to fix. + SpaceId string + // RefNames turns on the informative `#name` suffix on object references + // (export only, §9). Off by default — the export/backup shape stays + // minimal and stable under renames of referenced objects — and opted + // into by read shapes, the way CompactBlockLabels is. + RefNames bool + Keys KeyVocabulary // optional; nil = BundledKeyVocabulary (the derived table — keyvocab.go) + // Legend is the enclosing document's three legends, for the FRAGMENT + // entry points only (fragment.go, filters.go, BuildRecommendedLists). + // Marshal and Unmarshal ignore it: a whole document carries its own. + // + // A fragment has no envelope, so it has no legend of its own — and + // without one the §3 chain loses its first and highest step. A block cut + // out of a document that said `{"priority": "6a32d485…"}` resolved + // `priority` through the READER's vocabulary alone, which is the exact + // misresolution the legend exists to prevent, reintroduced at the seam + // that edits live objects. Hand the fragment the document's legend and + // chain step 1 is back. + Legend Legend + OmitIds bool // export only: drop every id (§9) + CompactBlockLabels bool // export only: relabel doc-local block/row/column/view ids to short suffixes (§9a; lossy, legend-less) + CompactIds bool // export only: alias for CompactBlockLabels — object refs are never compacted (§9a) + GenerateId func() string // import only: id generator for missing ids; nil = random 24-hex + NormalizeIndent bool // import only: clamp over-deep indents instead of rejecting (§4) + OnWarning func(Issue) // optional sink for warning-grade issues, both directions (indent clamps, unrepresentable dates, …) +} + +// Legend carries the three legends of the document a fragment was cut out +// of, so the fragment entry points can run the §3 chain from step 1 instead +// of starting at the reader's vocabulary. The field names and the semantics +// are the envelope's: `property_internal_keys` and `type_internal_keys` values are +// AUTHORITATIVE, an `option_ids` value is a liveness-checked hint (§3). +// +// The zero value is "no legend", which is what a caller that assembled the +// fragment itself has, and is the behaviour every fragment entry point had +// before this field existed. +type Legend struct { + // PropertyKeys maps a property spelling to the stored relation key it + // names (§3) — the enclosing document's `property_internal_keys`. + PropertyKeys map[string]string + // TypeKeys is the same for the type namespace — the enclosing document's + // `type_internal_keys`. + TypeKeys map[string]string + // OptionIds maps {property spelling: {option name: option id}} — the + // enclosing document's `option_ids` (§9a). + OptionIds map[string]map[string]string +} + +// empty reports whether the legend says nothing. +func (l Legend) empty() bool { + return len(l.PropertyKeys) == 0 && len(l.TypeKeys) == 0 && len(l.OptionIds) == 0 +} + +// fragmentDoc is the synthetic envelope a fragment entry point resolves +// against: no blocks, no properties, just the legend the caller handed over. +// Every fragment importer used to build `&jsonDoc{}` here, and an empty +// legend makes chain step 1 unconditionally silent. +func (o Options) fragmentDoc() *jsonDoc { + return &jsonDoc{ + PropertyKeys: o.Legend.PropertyKeys, + TypeKeys: o.Legend.TypeKeys, + OptionIds: o.Legend.OptionIds, + } +} + +// compactBlockLabels reports whether doc-local id relabeling is on. +func (o Options) compactBlockLabels() bool { return o.CompactBlockLabels || o.CompactIds } + +const ( + compactIdMinLen = 5 + + // well-known internal keys that get lifted into the envelope + detailKeyId = "id" + detailKeyType = "type" + // typeKeyTemplate is the type key `kind: "template"` names. Nothing + // RESOLVES to it any more: `kind` is the sole authority on whether a + // document is a template (§2), so the spelling `template` is an ordinary + // type term that a legend or a vocabulary may bind wherever it likes. + // + // ONE raw string comparison survives, and it consults no vocabulary: + // buildDoc keeps `kind` explicit when the term it is about to write is + // literally `template`, so a Page never emits the shape that used to mean + // a template. validate.go's matching refusal died at the freeze — the + // version gate answers for every pre-freeze document now (§15 #9). + typeKeyTemplate = "template" + storeKeyItems = "objects" + // codeLangField is the internal fields key holding a code block's + // language (§5.1) + codeLangField = "lang" +) + +// propertiesKeptOnExport are the internal properties the importer +// meaningfully preserves; everything else in LocalAndDerivedRelationKeys is +// stripped (§3). +// +// It mirrors the pb importer's own preserve-list +// (`core/block/import/pb/converter.go`), which is where "meaningfully +// preserves" is decided — and `creator` was never on it. It sat here anyway, +// so export wrote a participant id that every write path on the other side +// discarded. It is now on derivedAttributionProperties instead (validate.go): +// stripped as a VALUE, written as a name, dropped on import. Keeping it in +// both places would have been a contradiction resolved by list order, since +// strippedDetailKeys folds the dropped keys in after this one. +var propertiesKeptOnExport = map[string]bool{ + "createdDate": true, + "lastModifiedDate": true, + "isFavorite": true, + "isArchived": true, + "resolvedLayout": true, +} + +// wellKnownPropertyOrder puts the §3 magic keys first in the properties +// object; all remaining keys follow alphabetically (canonical order +// decision). +// +// It held `iconEmoji` and `iconImage` until §2b lifted both into the typed +// `icon` envelope field, where they sit above `properties` entirely — a +// stronger version of the same idea, since the reader now meets the icon +// before the property list rather than at the top of it. +var wellKnownPropertyOrder = []string{"name", "description"} + +// MarshalPropertyValue converts one property value to its JSON form under +// the §3 rules (dates → RFC 3339, select options → names, object/file → +// id lists, scalars wrap into lists for list-shaped formats). It is the +// row-level building block for API list surfaces that carry requested +// property values without a full document export. The result +// marshals with encoding/json. +// +// The second return is this value's share of the `option_ids` legend — +// {option name: option id} for THIS key (§9a) — and it is not optional +// bookkeeping. A select value is written as its option NAME, and a name is +// not an address: two options of one property may share it, and an option +// renamed between this call and the read makes the wiring mint a second +// option under the stale name. The whole-document export has carried the ids +// since §9a; this entry point computed them and threw them away, so every +// caller of the row-level surface silently had the pre-legend behaviour. +// Hand the map back to a value-level reader through Options.Legend.OptionIds +// under this key's spelling, or drop it and accept name resolution knowingly. +// +// The derived attribution keys — `creator`, `lastModifiedBy` — return the +// §3 spelling `#` as a plain string (the folded participant id, +// the member's name as the informative suffix where a resolver names them), +// or **nil** when the stored value holds no id. A row surface cannot omit a +// value its caller asked for, so nil is where the document's "omit it" +// lands; a caller that wants the property absent rather than null drops it +// on nil. +func MarshalPropertyValue(key string, v *types.Value, opts Options) (any, map[string]string) { + e := &exporter{opts: opts} + out := e.propertyValue(key, v) + return out, e.optionIdsFor(key) +} + +// Marshal serializes a snapshot into canonical AnyBlock JSON (§13). +func Marshal(sbType model.SmartBlockType, snapshot *model.SmartBlockSnapshotBase, opts Options) ([]byte, error) { + if snapshot == nil { + return nil, fmt.Errorf("nil snapshot") + } + e := &exporter{opts: opts, snapshot: snapshot, sbType: sbType, blocks: map[string]*model.Block{}, visited: map[string]bool{}} + e.indexBlocks() + // OmitIds writes no id at all, so a label plan has nothing to label: the + // two flags together used to run the census probe — a second full block + // emit — and mint a plan for output that carries no ids. Byte-identical + // either way; it was simply a whole extra emit for nothing. + if opts.compactBlockLabels() && !opts.OmitIds { + e.buildLabelPlan() + } + doc, err := e.buildDoc(sbType) + if err != nil { + return nil, fmt.Errorf("build document: %w", err) + } + return marshalCanonical(doc) +} + +type exporter struct { + opts Options + snapshot *model.SmartBlockSnapshotBase + sbType model.SmartBlockType + blocks map[string]*model.Block + rootId string + visited map[string]bool // emitted block ids: breaks ChildrenIds cycles, dedupes shared children + + // emitted, when non-nil, records the STORED doc-local id of everything + // this run actually writes — blocks, table rows/columns, dataview views. + // It is the id census's population (buildLabelPlan): only the probe run + // sets it, so a normal export pays nothing for it. + emitted map[string]bool + + // icon and cover are the typed envelope fields (§2b), built once: the id + // census reads the object ids they write and buildDoc writes the fields + // themselves, and building twice would report every warning twice. + icon, cover *omap + iconBuilt, coverBuilt bool + + // relTargets is a relation document's translated target-type key list + // (§2d), built once for the same reason: the type-key census + // (seedTypeTermLedger) and buildPropertySettings both read it. + relTargets []string + relTargetsBuilt bool + + localIds map[string]string // block/row/column/view id -> short label (§9a) + + // optionRefs is the second `refs` population: the option id behind every + // name export wrote for a select value (optionrefs.go). Recorded against + // the STORED property key and rendered into the nested `option_ids` map + // (property spelling → option name → id) at envelope-assembly time, when + // the term ledger has settled. + optionRefs map[optionRefPair]string + + // idLabels maps a stored block/row/column id to the id written for it, and + // idsUsed is every id this document has written. One set for every id + // surface, because they share one uniqueness domain (§4): a sanitized + // column id, a compact label and a verbatim block id all land in the same + // document, and any two of them colliding is a document Validate rejects. + idLabels map[string]string + idsUsed map[string]struct{} + + // propertyKeys is the §3 legend: the spelling→stored-key entries this + // document must carry to be invertible by a reader that cannot ask the + // space. + propertyKeys map[string]string + + // termOwner / termByKey / namedKeys are the §3 term ledger — the property + // keys' analogue of idLabels/idsUsed: one claim domain for every key slot, + // because the legend that inverts the terms is one document-wide map. + termOwner map[string]string // term -> the stored key it denotes + termByKey map[string]string // stored key -> the term written for it + namedKeys map[string]bool // census: every stored key any slot may name + // termPlan is the census's collision verdict, computed once per document + // (planKeyTerms): the term each censused key will actually take. For + // nearly every key that is its plain vocabulary spelling; where two + // censused keys claim ONE spelling — names are not unique, and + // collisions are resolved per document, not per space — EVERY claimant + // degrades through the ladder (stored key when readable, else + // ` ()`, else stored key), so which spelling a key gets + // cannot depend on which slot happened to claim first. + termPlan map[string]string + + // typeKeys is the §3 legend for the TYPE namespace, and typeTermOwner / + // typeTermByKey / typeNamedKeys its term ledger. One ledger and one + // legend PER NAMESPACE, deliberately: a property spelling and a type + // spelling may coincide without conflict (§3 — `object_type` the type key + // coexists with `objectType` the layout value, and a space can name a + // relation and a type one word), so a shared claim domain would back a + // key off a spelling the other namespace owns — a spurious conflict, and + // one legend map could not carry both meanings of the shared term at all. + typeKeys map[string]string + typeTermOwner map[string]string + typeTermByKey map[string]string + typeNamedKeys map[string]bool + typeTermPlan map[string]string +} + +// propertySlug renders a stored property key for output and records what the +// document owes a reader who cannot ask the space (§3). It is the term +// ledger's claim step, and the ONLY way a key slot may spell a key: a stored +// key always keeps its own term (§3 verbatim-first — the census reserved +// it), an uncontested spelling goes to its claimant, and a CONTESTED one — +// two censused keys sharing a name, or a name that is a censused stored key +// — degrades EVERY claimant by the census's plan (planKeyTerms), so which +// spelling a key gets never depends on which slot claimed first. The answer +// is remembered, so one key spells the same way in every slot; without the +// ledger, a block slot's blind recordPropertyKey could rebind a term that +// /properties already owns, silently moving that property's value onto a +// different relation. +func (e *exporter) propertySlug(key string) string { + if key == "" { + return key + } + if e.termOwner == nil { + e.seedTermLedger() + } + if term, done := e.termByKey[key]; done { + return term + } + term := e.writableSlug(key) + // the census's collision plan overrides the plain spelling for every + // censused key: names are not unique, so two keys in one document can + // claim one spelling, and the plan degrades EVERY claimant through the + // ladder rather than letting claim order pick a winner. An uncensused + // key (a block the census walk could not see) keeps the old first-claim + // discipline below, which can only over-degrade — always correct, + // merely less compact. + if planned, ok := e.termPlan[key]; ok { + term = planned + } + if term != key { + if _, claimed := e.termOwner[term]; claimed || e.namedKeys[term] { + term = key + } + } + e.termOwner[term] = key + e.termByKey[key] = term + e.recordPropertyKey(term, key) + return term +} + +// seedTermLedger runs the property-key census: every stored key any slot of +// this document may name. Verbatim-first (§3) makes each of those keys its +// own address, so no OTHER key's name may take one as a spelling — the same +// avoid-set discipline seedIdLabels applies to ids. The walk mirrors the emit +// sites (buildProperties, buildTypeProperties, blockToJSON, dataviewToJSON), +// and mirroring them EXACTLY is the whole obligation: an over-reservation +// degrades somebody's spelling to their stored key, which is always correct +// and merely less compact — until the next generation, which does not repeat +// it. Then a second export of the same object differs from the first and the +// round trip stops being a fixpoint. +// +// Three ways that can happen, and two of them are closed. The detail walk +// asks droppedPropertyKey, so the four kind- and value-scoped drops +// buildProperties applies are not censused (the gap that spelled a custom +// "Hidden" once suffixed and once plain, beside an `isHidden: false` nobody +// writes). modelledTypeKeys closes the same gap in the type namespace. What +// remains is a key named ONLY by a block the emit later drops: which blocks +// survive is decided during buildBlocks, so this walk cannot know. +func (e *exporter) seedTermLedger() { + e.termOwner = map[string]string{} + e.termByKey = map[string]string{} + e.namedKeys = map[string]bool{} + name := func(key string) { + if key != "" { + e.namedKeys[key] = true + } + } + if e.snapshot == nil { + return + } + if e.snapshot.Details != nil { + stripped := strippedDetailKeys() + lifted := e.envelopeLiftedKeys() + for k := range e.snapshot.Details.Fields { + if stripped[k] || lifted[k] || !isWritablePropertyKey(k) { + continue + } + // a key buildProperties is going to DROP is written nowhere, so + // it neither claims a spelling nor reserves its own stored key + // as one. Counting it broke the fixpoint: it degraded a rival + // in generation 1 that generation 2 — which no longer holds the + // key at all — spells plainly. Silent here; the claim site + // fires the one warning this predicate has (droppedPropertyKey). + // + // The census still names the key if a BLOCK names it below: a + // dataview filter on a dropped detail key really does spell it, + // and the block survives the drop. + if e.droppedPropertyKey(k, quietWarn) { + continue + } + name(k) + } + // the attribution keys are on the stripped list — their STORED value + // never reaches a document verbatim — but the document still SPELLS + // them when their value holds an id, and the census counts what the + // document spells (§9a). The condition mirrors buildProperties + // exactly: reserve neither more nor less than what is written, or a + // custom relation named "Created by" claims the spelling and the two + // members collapse onto one. + for k := range derivedAttributionProperties { + if _, ok := e.attributionRef(k); ok { + name(k) + } + } + } + if e.typePropsActive() { + for _, l := range recommendedListKeys { + for _, id := range valueStringList(e.detail(l.detailKey)) { + if def, ok := e.resolveTypeProperty(id); ok { + name(string(def.Key)) + } + } + } + } + for _, b := range e.snapshot.Blocks { + if b == nil { + continue + } + switch c := b.Content.(type) { + case *model.BlockContentOfRelation: + name(orEmpty(c.Relation).Key) + case *model.BlockContentOfLink: + for _, k := range orEmpty(c.Link).Relations { + name(k) + } + case *model.BlockContentOfDataview: + dv := orEmpty(c.Dataview) + for _, rl := range dv.RelationLinks { + if rl != nil { + name(rl.Key) + } + } + for _, v := range dv.Views { + if v == nil { + continue + } + name(v.GroupRelationKey) + name(v.CoverRelationKey) + name(v.EndRelationKey) + for _, r := range v.Relations { + if r != nil { + name(r.Key) + } + } + for _, s := range v.Sorts { + if s != nil { + name(s.RelationKey) + } + } + for _, f := range flattenFilters(v.Filters) { + name(f.RelationKey) + } + } + } + } + // the collision pass runs off the finished census, silently — the plan + // is consulted at claim time (propertySlug), where the warnings fire. + // The attribution keys yield: import drops them, so a generation-2 + // census will not hold them, and a spelling they contested would + // otherwise change between generations (planKeyTerms). + yielding := map[string]bool{} + for k := range derivedAttributionProperties { + if e.namedKeys[k] { + yielding[k] = true + } + } + e.termPlan = planKeyTerms(e.namedKeys, + func(k string) string { return e.vetSlug(k, quietWarn) }, + bundledPropertyKeyBySpelling, yielding) +} + +// propertySlugs is the list form, and it lives here rather than on Options for +// the same reason the singular one does: a key list (a link block's shown +// properties) is a key slot like any other, and a spelling written without +// the legend entry that inverts it is a spelling that reads back as a +// different relation. Options carries the vocabulary; only the exporter can +// record what the document owes for using it. +func (e *exporter) propertySlugs(keys []string) []string { + if len(keys) == 0 { + return keys + } + out := make([]string, len(keys)) + for i, key := range keys { + out[i] = e.propertySlug(key) + } + return out +} + +// slotPropertySlug is propertySlug for a reference slot that can DROP: a +// stored key no spelling can carry — over the 128-character bound, or +// holding control bytes (§3) — has no writable form at ANY slot, because +// vetSlug backs every vocabulary spelling off to the stored key when the key +// itself is unwritable, and the legend cannot rescue it either (its values +// are bounded the same way). Emitting it verbatim produced a document +// Marshal's own Validate rejects, now that the block key slots carry the +// schema bound /properties always had (§11, I1). So the slot is dropped, +// with a warning, exactly as an EMPTY stored key already is at the same +// slots. Returns "" for the caller to skip the slot; the buildProperties +// door keeps its own copy of this rule (it drops the whole property). +func (e *exporter) slotPropertySlug(key, slot string) string { + if key != "" && !isWritablePropertyKey(key) { + e.warn("", "%s names a property key that cannot be written in this format "+ + "(%s) and is dropped", slot, unwritableKeyReason("stored key", key)) + return "" + } + return e.propertySlug(key) +} + +// slotPropertySlugs is slotPropertySlug over a key list (a link block's +// shown properties). A dropped entry comes back as "" — the emit sites feed +// stringsToAny, which elides empties, so the drop and the empty-key elision +// share one door. +func (e *exporter) slotPropertySlugs(keys []string, slot string) []string { + if len(keys) == 0 { + return keys + } + out := make([]string, len(keys)) + for i, key := range keys { + out[i] = e.slotPropertySlug(key, slot) + } + return out +} + +// writableSlug is the vocabulary's spelling for a stored key when that +// spelling can actually be written, and the stored key itself otherwise. The +// spelling is the string that becomes a JSON property name (buildProperties) +// or a legend entry's name, and spellings come from display NAMES — arbitrary +// user text, with no length bound — so nothing upstream guarantees the +// shape §3 requires of a spelling. Checking the stored key and then emitting +// the spelling unchecked is how Marshal produced a document its own Validate +// rejects (maxLength 192 vs 128, on /properties and /property_internal_keys at once). +// The stored-key arm covers the mirror case: a spelling for a key that cannot +// be a legend VALUE has no invertible spelling but its own, so the verbatim +// key — always its own address (§3 verbatim-first) — is the one honest +// rendering. +// +// The same fate for a spelling validation refuses on other grounds than +// shape: "id" and "type" are refused as SPELLINGS before any resolution (§2 — +// the legend cannot re-purpose them, so it cannot rescue them either), and a +// DENIED key never takes a spelling at all, because the spelling's legend +// entry would carry a value the §3 deny rule refuses. Both used to make +// Marshal emit what its own Validate rejects — or, for the denied key, emit a +// legend a pre-admission reader would happily resolve. +func (e *exporter) writableSlug(key string) string { + return e.vetSlug(key, e.warn) +} + +// vetSlug is writableSlug with the warning sink explicit, so the census's +// collision planning (planKeyTerms) can ask the same question SILENTLY: the +// plan evaluates every censused key once before any slot claims, and a +// warning fired there would double every real warning and add ones for keys +// the emit later drops. The claim step (propertySlug → writableSlug) fires +// them at the moment the spelling is actually written, exactly as before. +func (e *exporter) vetSlug(key string, warn func(path, format string, args ...any)) string { + slug := e.opts.propertySlug(key) + if slug == key { + return slug + } + if !isWritablePropertyKey(slug) || !isWritablePropertyKey(key) { + warn("/"+memberPropertyInternalKeys, + "the vocabulary spells %q as %q, which cannot be a property spelling in this format; the stored key is written instead", + key, slug) + return key + } + if slug == detailKeyId || slug == detailKeyType { + warn("/"+memberPropertyInternalKeys, + "the vocabulary spells %q as %q, a spelling this format refuses before any resolution — `id` and `type` are the envelope's own members; the stored key is written instead", + key, slug) + return key + } + if _, denied := deniedPropertyKey(key); denied { + // the refusal protects the LEGEND: a denied key must not become a + // legend value (§3). A slug the bundled table binds to this very key + // — and no vocabulary contradicts — needs no legend entry at all + // (recordPropertyKey's rule), so the deny rule never sees it, and + // the reference slots that legitimately NAME a lifted key keep their + // §3 spelling. This is not hypothetical: the Property TYPE document + // lists `relationFormat` in its type_properties and shows it as a + // dataview column in 64 production spaces, and the blanket refusal + // spelled all of them camelCase-verbatim with two warnings each. + if bundledBinds(slug, key, (BundledKeyVocabulary{}).PropertyKey) && + termInverts(slug, key, e.opts.keys().PropertyKey) { + return slug + } + warn("/"+memberPropertyInternalKeys, + "%q cannot be a legend value (import refuses the internal keys export strips), so its spelling %q is not written; the stored key is its own address", + key, slug) + return key + } + return slug +} + +// quietWarn is the silent sink vetSlug/vetTypeSlug take during census +// planning. +func quietWarn(string, string, ...any) {} + +// planKeyTerms is the census's collision pass, run once per namespace per +// document: which term each censused key will take, decided from the whole +// census rather than from claim order. Raw names are not unique — two live +// properties may bear one name — and a document is a map, so a spelling two +// keys share cannot be written twice. The rule is per DOCUMENT, not per +// space: a name ambiguous space-wide but appearing once here spells its +// plain name (measured, genuine in-document collisions are 60 of 28,560 +// documents, 0.21%, across five names), and where a document does collide +// EVERY claimant degrades: +// +// (a) the stored key verbatim, when it is itself readable (not a minted +// 24-hex bson id) — the `producer_region` / `wine_region` shape; +// (b) else ` ()`, tail6 = the stored key's last six hex — +// deterministic, immutable while the key lives, visibly synthetic; +// (c) a residual tie — two claimants minting one suffix, or a suffix the +// census, the plan or the bundled table already answers — falls to +// the full stored key, which is always its own address. +// +// A spelling that equals a censused stored key is contested the same way +// (verbatim-first: the stored key owns its own term at every reader, so no +// claimant may take it). All claimants degrading — rather than first-claim +// keeping the plain name — is what makes the suffix stable across exports +// and the plain name trustworthy: a plain spelling in a document is never +// one of two same-named claimants. +// +// A YIELDING claimant is the one exception, and it exists for the fixpoint: +// the attribution keys are written by export and DROPPED by import, so a +// generation-2 census no longer holds them — a normal claimant they had +// contested would then un-suffix, and the round trip stopped being +// byte-stable exactly in the spaces holding a custom name-twin of +// "Created by". A yielding claimant therefore never contests anyone: alone +// on a spelling it takes it as usual, contested at all it takes its own +// stored key (always readable — the attribution keys are bundled camelCase) +// and the normal claimants keep the verdict they will re-derive without it. +func planKeyTerms(named map[string]bool, spell func(string) string, + bundledBound func(string) (string, bool), yielding map[string]bool) map[string]string { + keys := make([]string, 0, len(named)) + for k := range named { + keys = append(keys, k) + } + sort.Strings(keys) + cand := make(map[string]string, len(keys)) + claims := map[string][]string{} + for _, k := range keys { + t := spell(k) + if t == "" || t == k { + continue // verbatim: always its own address, never contested + } + cand[k] = t + if !yielding[k] { + claims[t] = append(claims[t], k) + } + } + contested := func(t string) bool { return len(claims[t]) > 1 || named[t] } + plan := make(map[string]string, len(cand)) + granted := make(map[string]bool, len(cand)) + for _, k := range keys { + t, ok := cand[k] + if !ok { + continue + } + if yielding[k] { + if len(claims[t]) > 0 || named[t] || granted[t] { + plan[k] = k // yield: the stored key, and nobody else moves + } else { + plan[k] = t + granted[t] = true + } + continue + } + if !contested(t) { + plan[k] = t + granted[t] = true + } + } + // rung (b) candidates are counted first so a residual tie — same name, + // same six-hex tail — sends BOTH claimants to rung (c), not whichever + // sorted first to (b) + suffixCount := map[string]int{} + for _, k := range keys { + if t, ok := cand[k]; ok && !yielding[k] && contested(t) { + if s := DisambiguatedKeySpelling(t, k); s != "" { + suffixCount[s]++ + } + } + } + for _, k := range keys { + t, ok := cand[k] + if !ok || yielding[k] || !contested(t) { + continue + } + s := DisambiguatedKeySpelling(t, k) // "" = rung (a) or unwritable: the key + if s != "" && suffixCount[s] == 1 && !named[s] && !granted[s] && len(claims[s]) == 0 { + if _, bound := bundledBound(s); !bound { + plan[k] = s + granted[s] = true + continue + } + } + plan[k] = k + } + return plan +} + +// recordPropertyKey writes the legend entry a term owes, or nothing when +// every reader's own chain already answers it correctly. One condition, +// two halves, and they ask DIFFERENT questions: +// +// 1. the **bundled table BINDS this spelling to this very key** — it ships +// with every reader, so `due_date` → `dueDate` needs no entry; and +// 2. the **vocabulary in force INVERTS it** — a reader may bind a spelling +// the bundled table binds correctly, and the writer's own space is the +// reader most likely to read the document back. +// +// The asymmetry is the point, and it is what makes the rule EXHAUSTIVE. A +// term that is a stored key written verbatim trivially "inverts" through any +// table, because a table that does not know a term answers the term itself +// (chain step 4) — so asking half 1 as an inversion let every custom key +// pass with no entry at all, and the document said nothing about the one +// population no reader can resolve without it. That silence is the corpse +// hole: the key is live and unambiguous the day it is written, and the +// moment a relation is UI-deleted its stored key stops being live while the +// freed spelling becomes some other relation's api key. Every document +// already written then re-points, offline, with nothing in it to say +// otherwise. Asking half 1 as a BINDING closes it: a spelling the bundled +// table does not bind to this key owes an entry, verbatim or not. +// +// Half 2 stays an inversion, and stays. Dropping it — "one table, not two" — +// loses the SHADOWING-WRITER entries, where the bundled table binds the +// spelling correctly and the vocabulary in force binds it elsewhere: +// measured, it drops `{"task": "task"}` and a template comes back pointing +// at an unrelated custom type, and drops `{"due_date": "dueDate"}` and +// dueDate's value lands on the custom relation that wanted the spelling. +// Both are silent losses of user data, both are pinned by tests. +// +// So identity entries stop being the exception and become the common line: +// every custom key names itself in the legend. That is the byte cost of the +// rule — ~2% on the golden documents — and it buys the document the ability +// to say what its own spellings mean without asking anyone. +// +// An entry the LEGEND cannot hold is not written — see legendEntryRefusal. +func (e *exporter) recordPropertyKey(term, key string) { + if term == "" { + return + } + if bundledBinds(term, key, (BundledKeyVocabulary{}).PropertyKey) && + termInverts(term, key, e.opts.keys().PropertyKey) { + return + } + if reason, refused := legendEntryRefusal(term, key, true); refused { + e.warn("/"+memberPropertyInternalKeys, "%s", reason) + return + } + if e.propertyKeys == nil { + e.propertyKeys = map[string]string{} + } + e.propertyKeys[term] = key +} + +// legendEntryRefusal reports whether a `property_internal_keys` / `type_internal_keys` entry is +// one the format can actually carry, and why not. It is the recording site's +// share of I1 ("Marshal never emits what Validate rejects", §11): every other +// key slot admits before it writes, and the two legends did not — the ONLY +// admission on the way in was writableSlug/writableTypeSlug, which returns +// early when the vocabulary has no spelling for a key, so the stored key reached +// the ledger unvetted. Marshal then wrote `{"a\nb": "a\nb"}` and its own +// Validate rejected the whole document; the object was unexportable and +// nothing said so. Reproduced first-hand at a dataview FILTER slot (which, +// unlike /properties, does not pre-filter unwritable keys) and at the +// envelope `type`, on `a\nb` and on a 140-character key, in both namespaces. +// +// The rule is the two Validate already states — a legend spelling and a +// legend stored key are both writable keys (propertyNameIssues), and a legend +// VALUE in the property namespace is a stored key the deny rule judges +// (§3 §4a) — asked here so the answers cannot drift. +// +// **Refusing the entry loses nothing the document was carrying.** The term is +// written verbatim either way (the ledger backed it off to the stored key +// long before this point), so the object still round-trips through any reader +// whose chain reaches step 4. What is lost is PORTABILITY for that one key: +// a reader whose vocabulary binds the spelling elsewhere has nothing to +// override it with. That is a strictly smaller loss than the alternative, +// which was the whole document, and the warning says which key it applies to. +func legendEntryRefusal(term, key string, deny bool) (string, bool) { + if !isWritablePropertyKey(term) { + return fmt.Sprintf("%s, so no legend entry is written for it; the term is "+ + "spelled verbatim and a reader that binds it elsewhere cannot be corrected", + unwritableKeyReason("legend spelling", term)), true + } + if !isWritablePropertyKey(key) { + return fmt.Sprintf("%s, so no legend entry is written for it; the term is "+ + "spelled verbatim and a reader that binds it elsewhere cannot be corrected", + unwritableKeyReason("legend stored key", key)), true + } + if !deny { + return "", false + } + if reason, denied := deniedPropertyKey(key); denied { + return fmt.Sprintf("legend value: %s — so no legend entry is written for %q; "+ + "the term is spelled verbatim", reason, term), true + } + return "", false +} + +// termInverts reports whether `term`, written for the stored key `key` with +// NO legend entry, reads back as `key` through one reader's table. It asks +// the table the same way the importer does — Options.propertyKey/typeKey +// take the answer and drop the ok flag, and a table that does not know a +// term answers the term itself (chain step 4, verbatim), which is what makes +// the two forms one question. +// +// Export asks it of TWO tables, and the second one is the fix for a defect +// that lost user data. The bundled table is the reader that always exists, +// so a spelling it binds elsewhere has always owed an entry. But the +// vocabulary this export runs under is a reader too — the writer's own +// space, the one most likely to read the document back — and it answers +// FIRST, before the bundled table (importer.propertyKey / typeKey run +// Options' vocabulary, which is the whole chain a node-backed reader has). +// Asking only the bundled table wrote a term with no legend that the +// writer's own vocabulary then bound to a different stored key: +// +// - the type namespace lost data in silence. A type UI-deleted from the +// space vacates its stored key (storeresolver's corpse policy), so +// `initiative` stops being a live stored key while objects still carry +// `ot-initiative`; the same listing binds the freed spelling `initiative` +// to a live type keyed `69bbfc…`. Export wrote `"type": "initiative"` with no +// entry and import bound it to `69bbfc…` — the object came back typed as +// a different type, no error anywhere. +// - the property namespace broke I1 out loud. A vocabulary spelling +// `alpha` as `beta`, over an object holding both `alpha` and `beta`, +// wrote both keys verbatim (the ledger backs `alpha` off its contested +// spelling, correctly) — and then the reader's own vocabulary bound `beta` +// to `alpha`, so two spellings addressed one property and Unmarshal +// refused a document Marshal had just emitted. +// +// The entry fixes both for EVERY reader, not just for the one whose table +// prompted it: the legend is chain step 1, ahead of any vocabulary. What it +// cannot fix is a reader whose table shadows the bundled one in a way the +// writer never saw — that is precondition 2 on KeyVocabulary, and it stays a +// precondition for exactly this reason. +func termInverts(term, key string, table func(string) (string, bool)) bool { + back, _ := table(term) + return back == key +} + +// bundledBinds is the stricter question recordPropertyKey/recordTypeKey ask +// of the BUNDLED table: does the table actually BIND this spelling to this +// key — `ok` and all — rather than merely fail to contradict it? +// +// termInverts cannot answer it. It drops the ok flag on purpose, because +// that is what the importer does, and for the VOCABULARY half that is the +// right question: "would this reader land on the right key?". For the +// bundled half it is the wrong one, because "the table has never heard of +// this term" and "the table binds this term to this key" are the same answer +// there — and they mean opposite things to a reader. The first is exactly +// the population that owes an entry: a key the bundled table cannot speak +// for, whose spelling is up for grabs the moment the key stops being live. +func bundledBinds(term, key string, table func(string) (string, bool)) bool { + back, ok := table(term) + return ok && back == key +} + +// buildPropertyKeys renders the legend in key order, or nil when the document +// needs none — which is every document a package-only reader wrote. +func (e *exporter) buildPropertyKeys() *omap { + if len(e.propertyKeys) == 0 { + return nil + } + slugs := make([]string, 0, len(e.propertyKeys)) + for slug := range e.propertyKeys { + slugs = append(slugs, slug) + } + sort.Strings(slugs) + m := &omap{} + for _, slug := range slugs { + m.set(slug, e.propertyKeys[slug]) + } + return m +} + +// typeSlug renders a stored type key for output and records what the +// document owes a reader who cannot ask the space (§3) — the type +// namespace's claim step, propertySlug on a ledger of its own. The same +// discipline for the same reason: a stored type key named anywhere in the +// document always keeps its own term (verbatim-first), an uncontested +// spelling goes to its claimant, and a contested one degrades every +// claimant by the census's plan. +func (e *exporter) typeSlug(key string) string { + if key == "" { + return key + } + if e.typeTermOwner == nil { + e.seedTypeTermLedger() + } + if term, done := e.typeTermByKey[key]; done { + return term + } + term := e.writableTypeSlug(key) + // the census's collision plan, exactly as propertySlug applies it: every + // claimant of a contested spelling degrades by plan, not by claim order + if planned, ok := e.typeTermPlan[key]; ok { + term = planned + } + if term != key { + if _, claimed := e.typeTermOwner[term]; claimed || e.typeNamedKeys[term] { + term = key + } + } + e.typeTermOwner[term] = key + e.typeTermByKey[key] = term + e.recordTypeKey(term, key) + return term +} + +// typeSlugs is the list form (a type property's object_types, §2a). +func (e *exporter) typeSlugs(keys []string) []string { + if len(keys) == 0 { + return keys + } + out := make([]string, len(keys)) + for i, key := range keys { + out[i] = e.typeSlug(key) + } + return out +} + +// seedTypeTermLedger runs the type-key census: every stored type key any +// slot of this document may name — the snapshot's object types (envelope +// `type`/`template_for`) and the target types of the resolved type-property +// definitions (§2a object_types). Verbatim-first (§3) makes each its own +// address, so no other key's name may take one as a spelling. +func (e *exporter) seedTypeTermLedger() { + e.typeTermOwner = map[string]string{} + e.typeTermByKey = map[string]string{} + e.typeNamedKeys = map[string]bool{} + if e.snapshot == nil { + return + } + for _, key := range e.modelledTypeKeys(false) { + e.typeNamedKeys[key] = true + } + if e.typePropsActive() { + for _, l := range recommendedListKeys { + for _, id := range valueStringList(e.detail(l.detailKey)) { + def, ok := e.resolveTypeProperty(id) + if !ok || !writableTypePropertyKey(def) { + continue + } + for _, key := range def.ObjectTypes { + if key != "" { + e.typeNamedKeys[key] = true + } + } + } + } + } + // a relation document's own target types (§2d) are a type-key slot too, + // and the census must know every key the slot will spell for the same + // reason it knows the §2a targets: verbatim-first (§3) makes each its + // own address, so no other key's name may take one as a spelling + if e.isPropertyDoc() { + for _, key := range e.relationTargetKeys() { + if key != "" { + e.typeNamedKeys[key] = true + } + } + } + // the collision pass, exactly as the property census runs it — with no + // yielding set: nothing in the type namespace is written-then-dropped + e.typeTermPlan = planKeyTerms(e.typeNamedKeys, + func(k string) string { return e.vetTypeSlug(k, quietWarn) }, + bundledTypeKeyBySpelling, nil) +} + +// writableTypeSlug is writableSlug for the type namespace: the vocabulary's +// spelling when it can actually be written and honored, the stored key +// itself otherwise. The shape rule is the same — a spelling becomes a legend +// member name and a stored key a legend value, both bounded by the schema. The +// reserved spelling differs: the type namespace has none. It used to refuse +// to move `template` in either direction, because the envelope's template +// semantics hung off the spelled term — export keyed template_for emission +// off it, validation gated /template_for on it, and import derived the +// smartblock kind from it, so a vocabulary moving the spelling dropped a +// template's target type and one landing another key on it handed that +// machinery to the wrong type. `kind` carries all three now (§2), the term is +// an ordinary type spelling, and the reservation deleted with the ambiguity +// it was protecting: a vocabulary may spell the template type `tmpl`, and the +// legend says so and inverts it. +func (e *exporter) writableTypeSlug(key string) string { + return e.vetTypeSlug(key, e.warn) +} + +// vetTypeSlug is vetSlug for the type namespace — the warning sink explicit +// for the same census-planning reason. +func (e *exporter) vetTypeSlug(key string, warn func(path, format string, args ...any)) string { + slug := e.opts.typeSlug(key) + if slug == key { + return slug + } + if !isWritablePropertyKey(slug) || !isWritablePropertyKey(key) { + warn("/"+memberTypeInternalKeys, + "the vocabulary spells type %q as %q, which cannot be a type spelling in this format; the stored key is written instead", + key, slug) + return key + } + return slug +} + +// recordTypeKey writes the type legend entry a term owes, or nothing when +// every reader's own chain already inverts it — recordPropertyKey's rule +// through the type half of the two tables, identity entries included: a +// stored type key written verbatim whose spelling the bundled table binds to +// a DIFFERENT key (`object_type` the stored key beside bundled `objectType`) +// gets `{"object_type": "object_type"}`, the document's only way to tell a +// storeless reader the term is a stored key — and the same entry, for the +// same reason, when the vocabulary in force is the one that binds it +// elsewhere (`initiative` the stored key of a UI-deleted type, beside the +// live type whose api key is `initiative`). See termInverts. +// +// An entry the legend cannot hold is not written here either +// (legendEntryRefusal) — minus the deny rule, which is the property +// namespace's alone: `strippedDetailKeys` and the importer's resolution +// vectors are relation keys, and Validate states no deny rule over a +// `type_internal_keys` value. +func (e *exporter) recordTypeKey(term, key string) { + if term == "" { + return + } + if bundledBinds(term, key, (BundledKeyVocabulary{}).TypeKey) && + termInverts(term, key, e.opts.keys().TypeKey) { + return + } + if reason, refused := legendEntryRefusal(term, key, false); refused { + e.warn("/"+memberTypeInternalKeys, "%s", reason) + return + } + if e.typeKeys == nil { + e.typeKeys = map[string]string{} + } + e.typeKeys[term] = key +} + +// buildTypeKeys renders the type legend in term order, or nil when the +// document needs none — which is every document that names only bundled and +// verbatim, unshadowed type keys. +// legendTypeTerm answers what this document's own type_internal_keys legend binds a +// term to, falling back to the term itself. It is what buildDoc's emission +// rule reads: like the Validate gate it used to answer to — deleted at the +// freeze (§15 #9) — it reads the document alone and resolves nothing beyond +// it, so a term and the key its own legend binds it to are one spelling of +// one document (§2, §10). +func (e *exporter) legendTypeTerm(term string) string { + if key, ok := e.typeKeys[term]; ok && key != "" { + return key + } + return term +} + +func (e *exporter) buildTypeKeys() *omap { + if len(e.typeKeys) == 0 { + return nil + } + terms := make([]string, 0, len(e.typeKeys)) + for term := range e.typeKeys { + terms = append(terms, term) + } + sort.Strings(terms) + m := &omap{} + for _, term := range terms { + m.set(term, e.typeKeys[term]) + } + return m +} + +// seedIdLabels reserves the id each block will be written with, before any +// sanitizing starts. Without it the first block to need sanitizing could take +// the name of a block that was going to be written verbatim — renaming a +// perfectly good authored id, or duplicating it. +// +// The order below is what makes export's reservations the same id domain +// validation checks (§4). Rows and columns come first, because the derived +// cell ids are built from their labels; then the grid those imply — every +// rowId-colId pair, written or not, since the table owns the id either way and +// the editor materializes the cell at exactly that id the first time it is +// filled (§6.1); then everything else, which yields to the grid, because "a +// non-table block id that collides with a derived cell id is a validation +// error" (§4) and the derived id is the one that cannot move. +// +// Only blocks the emit can REACH are reserved. A snapshot's block list is not +// its block tree: orphaned subtrees outlive the block that held them (state +// apply unlinks without deleting), and a table among them owns a whole grid of +// derived ids — reserving that grid renamed a perfectly good authored id on a +// block the document does contain, on the authority of one nobody can see. The +// walk over-approximates on purpose: it descends every ChildrenIds edge from +// the export's entry point, including the ones blockToJSON declines to follow, +// because under-reserving is the dangerous direction — a grid that IS emitted +// and not reserved is a document Marshal's own Validate rejects (I1). +func (e *exporter) seedIdLabels() { + e.idLabels = map[string]string{} + e.idsUsed = map[string]struct{}{} + if e.snapshot == nil { + return + } + reachable := e.reachableBlocks() + + // snapshot order, not map order: the reservations are order-dependent now, + // so ranging over the id-keyed map would make the output nondeterministic. + // wanted collects the label every block would take verbatim; it is an + // avoid-set rather than a reservation, because sanitizing a row id must + // not take the name of a block written as-is, and that block cannot be + // reserved before the grid it may collide with. + var order []*model.Block + seen := map[string]bool{} + wanted := map[string]struct{}{} + for _, b := range e.snapshot.Blocks { + if b == nil || b.Id == "" || seen[b.Id] || !reachable[b.Id] { + continue + } + seen[b.Id] = true + order = append(order, e.blocks[b.Id]) // the indexed block wins, as everywhere else + wanted[e.localId(b.Id)] = struct{}{} + } + + for _, b := range order { + if !e.isTableInner(b) { + continue + } + if want := e.localId(b.Id); isValidTableInnerId(want) { + e.idLabels[b.Id] = want + e.idsUsed[want] = struct{}{} + } + } + for _, b := range order { + if !e.isTableInner(b) { + continue + } + if _, done := e.idLabels[b.Id]; done { + continue + } + e.idLabels[b.Id] = e.reserveLabel(sanitizeTableInnerId(e.localId(b.Id)), wanted) + } + for _, id := range e.derivedCellIds(reachable) { + e.idsUsed[id] = struct{}{} + } + for _, b := range order { + if e.isTableInner(b) { + continue + } + want := e.localId(b.Id) + if !isBlockIdLabel(want) { // the blockId charset: [A-Za-z0-9_-]{1,64} + continue + } + if _, taken := e.idsUsed[want]; taken { + continue // a derived cell id holds the name; idLabel disambiguates + } + e.idLabels[b.Id] = want + e.idsUsed[want] = struct{}{} + } +} + +// reserveLabel takes base, or the first _n form of it that no id has taken and +// no id is going to take verbatim. +func (e *exporter) reserveLabel(base string, wanted map[string]struct{}) string { + label := base + for n := 2; ; n++ { + _, used := e.idsUsed[label] + _, want := wanted[label] + if !used && !want { + e.idsUsed[label] = struct{}{} + return label + } + suffix := "_" + strconv.Itoa(n) + trimmed := base + if len(trimmed)+len(suffix) > maxIdLen { + trimmed = trimmed[:maxIdLen-len(suffix)] + } + label = trimmed + suffix + } +} + +// derivedCellIds lists the cell id every table the export can REACH implies: +// rowLabel + "-" + colLabel for the whole grid, materialized or not (§6.1). +// It runs after every row and column has its label, and mirrors the structure +// tableToJSON reads — wrappers by layout style, children by content type — so +// that the ids reserved are the ones the exported tables actually derive. A +// table no emit reaches derives nothing, so it reserves nothing: its grid is +// not in the document, and the ids of the blocks that are may not turn on it. +func (e *exporter) derivedCellIds(reachable map[string]bool) []string { + var out []string + for _, b := range e.snapshot.Blocks { + if b == nil || b.Id == "" || e.blocks[b.Id] != b || !reachable[b.Id] { + continue + } + if _, isTable := b.Content.(*model.BlockContentOfTable); !isTable { + continue + } + var cols, rows []string + for _, id := range b.ChildrenIds { + wrapper := e.blocks[id] + l, ok := wrapper.GetContent().(*model.BlockContentOfLayout) + if !ok { + continue + } + for _, innerId := range wrapper.ChildrenIds { + inner := e.blocks[innerId] + if inner == nil { + continue + } + label := e.idLabels[innerId] + if label == "" { + continue + } + switch l.Layout.GetStyle() { + case model.BlockContentLayout_TableColumns: + if _, ok := inner.Content.(*model.BlockContentOfTableColumn); ok { + cols = append(cols, label) + } + case model.BlockContentLayout_TableRows: + if _, ok := inner.Content.(*model.BlockContentOfTableRow); ok { + rows = append(rows, label) + } + } + } + } + for _, row := range rows { + for _, col := range cols { + out = append(out, row+"-"+col) + } + } + } + return out +} + +// reachableBlocks is the ChildrenIds closure of the export's entry point — +// every block the emit can arrive at, and therefore every block whose id the +// document may contain. Everything else is an orphan: present in the snapshot, +// absent from the output, and with no claim on the id domain (§4). +// +// It is deliberately coarser than the emit: blockToJSON stops descending into +// a bookmark, a link or a divider, and drops structural and content-less +// blocks entirely, but this walk follows those edges anyway. Reserving for a +// block the emit turns out to drop costs a disambiguation suffix; failing to +// reserve for one it keeps costs a document that fails its own Validate. +// +// The walk is iterative: a snapshot's block graph is untrusted, and a +// pathological chain is not the place to find out how deep the stack goes. +func (e *exporter) reachableBlocks() map[string]bool { + out := map[string]bool{} + if e.rootId == "" { + return out + } + stack := []string{e.rootId} + for len(stack) > 0 { + id := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if id == "" || out[id] { + continue + } + out[id] = true + if b := e.blocks[id]; b != nil { + stack = append(stack, b.ChildrenIds...) + } + } + return out +} + +func (e *exporter) isTableInner(b *model.Block) bool { + switch b.GetContent().(type) { + case *model.BlockContentOfTableRow, *model.BlockContentOfTableColumn: + return true + } + return false +} + +// idLabel is the one place a stored id becomes the id written in the document. +// It sanitizes with the charset of the position, then disambiguates against +// every id the document has already written, and remembers its answer so the +// same stored id always renders the same way. +func (e *exporter) idLabel(stored string, sanitize func(string) string) string { + if stored == "" { + return "" + } + if e.idLabels == nil { + e.seedIdLabels() + } + if got, ok := e.idLabels[stored]; ok { + return got + } + base := sanitize(e.localId(stored)) + label := base + for n := 2; ; n++ { + if _, taken := e.idsUsed[label]; !taken { + e.idsUsed[label] = struct{}{} + e.idLabels[stored] = label + return label + } + suffix := "_" + strconv.Itoa(n) + trimmed := base + if len(trimmed)+len(suffix) > maxIdLen { + trimmed = trimmed[:maxIdLen-len(suffix)] + } + label = trimmed + suffix + } +} + +// blockLabel renders a block's stored id for output (§9). Stored ids are not +// guaranteed to match the schema's block charset: legacy accounts hold ids +// with dots and slashes, and Options.GenerateId belongs to the caller — the +// convert wiring derives ids from file paths. Writing one verbatim made +// Marshal emit a document its own Validate rejects, i.e. an archive that fails +// at import, discovered long after the export. +func (e *exporter) blockLabel(stored string) string { + return e.idLabel(stored, sanitizeBlockId) +} + +func (e *exporter) detail(key string) *types.Value { + if e.snapshot.Details == nil { + return nil + } + return e.snapshot.Details.Fields[key] +} + +func (e *exporter) objectId() string { + return e.detail(detailKeyId).GetStringValue() +} + +func (e *exporter) indexBlocks() { + children := map[string]bool{} + for _, b := range e.snapshot.Blocks { + if b == nil || b.Id == "" { + continue + } + e.blocks[b.Id] = b + for _, c := range b.ChildrenIds { + children[c] = true + } + } + // the root block's id equals the object id (§2); fall back to the first + // block nobody references + if _, ok := e.blocks[e.objectId()]; ok { + e.rootId = e.objectId() + return + } + for _, b := range e.snapshot.Blocks { + if b != nil && b.Id != "" && !children[b.Id] { + e.rootId = b.Id + return + } + } +} + +// typeKeyIdPrefix is the "ot-" prefix ObjectTypes entries carry. +var typeKeyIdPrefix = domain.TypeKey("").URL() + +// envelopeTypeTerms are the spellings written for the snapshot's object +// types — the `type`/`template_for` slots — each claimed through the type +// term ledger so the legend it owes is recorded (§3). +// +// Two disciplines run here, and both are buildProperties' own: +// +// - **A keyless entry is dropped WITH a warning, and the survivors close +// ranks.** A stored `ot-` (or a bare "") carries no type key: typeSlug +// answers "" for it and setNonEmpty then omits the slot, so a positional +// write lost that entry AND everything behind it. A template stored as +// ["ot-", "ot-task"] emitted no `type` at all, which made `template_for` +// inexpressible too — so the perfectly good `ot-task` vanished beside its +// bad neighbour, silently, and the document read back as no types at all. +// Filtering first is what lets the good sibling survive; the warning is +// what buildProperties already owes an unwritable *property* key, and +// what the import seam refuses outright and path-addressed. +// - **Only the slots actually WRITTEN claim a term.** typeSlug is the term +// ledger's claim step, so spelling an entry no slot emits still records +// the legend entry that spelling owes: a document then carried a +// `type_internal_keys` line naming a type it never mentions, publishing a space's +// spelling→key mapping for nothing. buildProperties cannot do this because +// it filters before it spells; the type side now does the same. +// +// The list still truncates to the positions §2 models — one type, plus the +// target type on a template. That is the format's shape, not a defect, and +// the census (seedTypeTermLedger) still reserves every stored type key the +// snapshot names, so a dropped entry's key can never be taken as another +// key's spelling. +func (e *exporter) envelopeTypeTerms() []string { + keys := e.modelledTypeKeys(true) + terms := make([]string, 0, len(keys)) + for _, key := range keys { + terms = append(terms, e.typeSlug(key)) + } + return terms +} + +// modelledTypeKeys reduces the snapshot's object types to the stored keys the +// envelope will actually spell: keyless entries dropped, survivors closing +// ranks, then the positions §2 models — one type, plus the target type on a +// template. `warn` reports each keyless drop, and only the emitting call +// passes it, because the CENSUS runs this reduction too and must not report +// the same drop twice. +// +// The census has to see exactly this list rather than every object type, +// which is where it started. Reserving a key no slot spells makes export stop +// being a fixpoint: a snapshot whose truncated-away second type is the first +// one's spelling backed that spelling off, while the same object exported after one +// round trip — the second type gone, the census one key smaller — spelled it. +// Two documents, the same object, differing in the term and in a legend line; +// §9's "re-exports diff cleanly" is the promise that breaks. Nothing was +// protected by the wider reservation either: a key the document never names +// cannot be taken as another key's spelling by a reader that never sees it. +// +// The second slot exists exactly when the SMARTBLOCK TYPE is Template, which +// is the whole of §2's template rule. It used to be "when the +// first surviving key is the template key", and that was the bug: a template +// whose object types are ["ot-task", "ot-extra"] — a real shape, since +// nothing in the model requires a template to carry the template key first — +// kept one slot and dropped its target type with a warning. Keyed off the +// smartblock type, the same snapshot writes `{"kind": "template", "type": +// "task", "template_for": "extra"}` and round-trips whole. +func (e *exporter) modelledTypeKeys(warn bool) []string { + keys := make([]string, 0, len(e.snapshot.ObjectTypes)) + stood := make([]int, 0, len(e.snapshot.ObjectTypes)) // where each survivor stood + for i, t := range e.snapshot.ObjectTypes { + key := strings.TrimPrefix(t, typeKeyIdPrefix) + if key == "" { + if warn { + e.warn("/type", + "object type %d (%q) carries no type key and is dropped; the remaining types move up", i, t) + } + continue + } + keys = append(keys, key) + stood = append(stood, i) + } + if len(keys) == 0 { + return nil + } + kept := 1 + if e.sbType == model.SmartBlockType_Template && len(keys) > 1 { + kept = 2 + } + // §3 promises every drop is reported, and the positional one was the + // silent half: a keyless entry warned, a perfectly good SECOND type just + // vanished. It is not a defect — the envelope models these positions and + // no others (§2) — but it is a loss the caller can neither see in the + // document nor infer from it, and the caller is the one holding the + // snapshot that still has the type. + if warn { + for j := kept; j < len(keys); j++ { + e.warn("/type", + "object type %d (%q) is dropped: the envelope carries one type, plus the target type on a template, "+ + "and every position is already taken; it is not written anywhere in this document", + stood[j], e.snapshot.ObjectTypes[stood[j]]) + } + } + return keys[:kept] +} + +func (e *exporter) buildDoc(sbType model.SmartBlockType) (*omap, error) { + doc := &omap{} + doc.set("$schema", SchemaURL) + doc.set("version", FormatVersion) + + typeTerms := e.envelopeTypeTerms() + typeTerm := "" + if len(typeTerms) > 0 { + typeTerm = typeTerms[0] + } + + // kind is omitted whenever derivable (§2), and only Page is + // derivable: `kind` is the sole authority on template-ness, so a Template + // always spells it. The term test that survives is an EMISSION rule and + // resolves nothing — a Page whose type term is literally `template`, or + // whose own type_internal_keys legend binds its term to `template`, keeps + // its explicit kind. + // + // It was an I1 rule until the freeze: Validate refused + // `{"type": "template"}` with no kind as the pre-`kind` spelling of a + // template, so emitting one would have been Marshal writing what Validate + // rejects. That refusal is gone — every document written under the old + // reading declares version 1, which the version gate refuses for the + // whole grammar (§15 #9) — and the rule stays for the reason that + // outlived it: the shape is ambiguous to a reader who remembers the old + // meaning, the authoring subset refuses it outright (§2g), and a + // spelled-out kind costs ~16 bytes on the rare document that has to + // carry it. Both spellings force it, because they are the same document + // said differently: testing only the raw term lets a page whose term + // RESOLVES to the template key through its own legend drop the kind, and + // testing only the resolved key lets a page whose legend rebinds + // `template` ELSEWHERE drop it. + derivable := sbType == model.SmartBlockType_Page && + typeTerm != typeKeyTemplate && e.legendTypeTerm(typeTerm) != typeKeyTemplate + if !derivable { + name := kindNames.name(sbType) + if name == "" { + return nil, fmt.Errorf("smartblock type %v has no kind mapping", sbType) + } + doc.set("kind", name) + } + + // the envelope id: the participant fold applies — a participant + // document's OWN id folds to the bare identity, or a reader could not + // textually join a folded reference to the document it points at (§9) — + // but never the name suffix: the document's name is right below in + // `properties`, and the envelope id is the one slot a reader must be + // able to use verbatim as an address. + doc.setNonEmpty("id", e.opts.foldParticipantRef(e.objectId())) + doc.setNonEmpty("type", typeTerm) + if sbType == model.SmartBlockType_Template && len(typeTerms) > 1 { + doc.setNonEmpty("template_for", typeTerms[1]) + } + doc.setNonEmpty(memberInternalKey, e.snapshot.Key) + // a relation document states its own definition next (§2d): `format`, + // `include_time`, `object_types` — before `icon`, because what a property + // IS outranks what it looks like, and before the legends, so the type + // terms `object_types` claims land in the ledger the legends render + if err := e.buildPropertySettings(doc); err != nil { + return nil, err + } + // the typed icon/cover fields sit above `properties` (§2b): they are what + // a reader looks at first, and they are outside the key namespace the + // legend can rebind + doc.setNonEmpty("icon", e.iconField()) + doc.setNonEmpty("cover", e.coverField()) + // every surface that spells a property or type key runs before the + // envelope is assembled, so the legends they populate land in their + // canonical positions rather than trailing the blocks that filled them + properties := e.buildProperties() + typeSettings := e.buildTypeSettings() + blocks, err := e.buildBlocks() + if err != nil { + return nil, err + } + + doc.setNonEmpty("properties", properties) + // present whenever it has anything to say — with a property resolver its + // property_definitions member is present even when empty, because that + // presence is what tells import to rebuild the four lists (§2a) + doc.setNonEmpty("type_settings", typeSettings) + + doc.setNonEmpty(memberPropertyInternalKeys, e.buildPropertyKeys()) + doc.setNonEmpty(memberTypeInternalKeys, e.buildTypeKeys()) + // option_ids last of the three legends: its outer keys are property + // spellings, so the legend that inverts those precedes it (§2). Written + // unconditionally — this is identity, not compaction — except under + // OmitIds, where a legend of nothing but ids has no place (§9, §9a). + if !e.opts.OmitIds { + doc.setNonEmpty("option_ids", sortedNestedOmap(e.buildOptionIds())) + } + doc.setNonEmpty("blocks", blocks) + + items, store := e.buildStore() + doc.setNonEmpty("items", items) + doc.setNonEmpty("store", store) + doc.setNonEmpty("root", e.buildRootEscape()) + return doc, nil +} + +func (e *exporter) buildStore() ([]any, *omap) { + coll := e.snapshot.Collections + if coll == nil || len(coll.Fields) == 0 { + return nil, nil + } + // the objects key lifts into items only when it is a list; any other + // shape stays in store so nothing is silently dropped + var items []any + objectsLifted := false + if v := coll.Fields[storeKeyItems]; v != nil { + if lv, ok := v.GetKind().(*types.Value_ListValue); ok { + objectsLifted = true + for _, el := range lv.ListValue.GetValues() { + if id := el.GetStringValue(); id != "" { + items = append(items, e.objectRef(id)) + } + } + } + } + store := &omap{} + keys := make([]string, 0, len(coll.Fields)) + for k := range coll.Fields { + if k != storeKeyItems || !objectsLifted { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, k := range keys { + store.set(k, protoValueToJSON(coll.Fields[k])) + } + return items, store +} + +// analyticsRootFields are the analytics keys the ROOT BLOCK's fields carry. +// +// The format already decided analytics do not travel — twice, on details: +// the click-context triple (`data`/`isNew`/`layoutFormat`) "describes the +// click that made the object, not the object", and `analyticsSpaceId` is +// stripped beside a space's invite keys because "a restored space mints its +// own invites and its own analytics identity". +// +// Both rulings watched ONE door. These two ride the other: block fields, not +// details, so the strippedDetailKeys list never saw them and 1,042 of 38,105 +// corpus documents shipped them — analyticsOriginalId on 872, +// analyticsContext on 445 (values like "empty", a client route name). +// +// analyticsOriginalId is the sharper of the two: it is the id of the object +// this one was made FROM, so it is both a tracking identifier and a dangling +// reference — 805 of the 872 name an object that is in no bundle at all. +// +// Deliberately only these two. The same map carries `isLocked` (128) and +// `width` (45), which are real state a reader wants, so this is a named set +// rather than an analytics-prefix sweep — and a prefix rule would be wrong +// anyway: the corpus holds a user's tag named `analytics` and an option +// named "Data, Analytics & Reporting", which are CONTENT. +var analyticsRootFields = map[string]bool{ + "analyticsOriginalId": true, + "analyticsContext": true, +} + +func (e *exporter) buildRootEscape() *omap { + root := e.blocks[e.rootId] + if root == nil { + return nil + } + m := &omap{} + if fields := withoutAnalyticsFields(root.Fields); fields != nil { + m.set("fields", protoStructToJSON(fields)) + } + m.setNonEmpty("background_color", root.BackgroundColor) + return m +} + +// withoutAnalyticsFields returns f without the analytics keys, or nil when +// nothing survives — so a block whose fields were ONLY analytics exports no +// `fields` member at all rather than an empty map. Copies rather than +// mutates: the snapshot belongs to the caller. +func withoutAnalyticsFields(f *types.Struct) *types.Struct { + if f == nil || len(f.Fields) == 0 { + return nil + } + kept := make(map[string]*types.Value, len(f.Fields)) + for k, v := range f.Fields { + if !analyticsRootFields[k] { + kept[k] = v + } + } + if len(kept) == 0 { + return nil + } + return &types.Struct{Fields: kept} +} + +// +// ---- properties ---- +// + +// strippedDetailKeys are the internal/derived properties export removes (§3). +// InternalPropertyKeys reports the property keys this format treats as +// internal: export strips them and import refuses them (§3). It is exported +// for tooling that has to agree with that set — a round-trip checker comparing +// a snapshot with its re-import has to know which keys are *expected* to be +// gone. Two copies of this list have now drifted: cmd/anyblockroundtrip's, and +// the one that moved with it into snapshotdiff. Both reported 10 378 false +// data-loss issues over a 36 808-object account the moment the package added +// the importer's provenance keys. +func InternalPropertyKeys() map[string]bool { + return strippedDetailKeys() +} + +// strippedDetailKeys is the internal-property list, and it is the single +// source of truth for both directions: export removes these keys, and import +// refuses them (§3, §4a — deniedPropertyKey reads this same set). Two lists +// would drift, which is how the import surface ended up strictly wider than +// the export surface. +func strippedDetailKeys() map[string]bool { + stripped := map[string]bool{detailKeyId: true, detailKeyType: true} + for _, k := range bundle.LocalAndDerivedRelationKeys { + if !propertiesKeptOnExport[string(k)] { + stripped[string(k)] = true + } + } + // the importer's own resolution vectors are bundled relations but not + // local/derived ones, so the list above does not cover them; they are + // internal all the same + for k := range neverWritableProperties { + stripped[k] = true + } + // transient state describes the moment the object was written, not the + // object; it means nothing on the other side of an import + for k := range transientProperties { + stripped[k] = true + } + // the attribution keys are stripped as VALUES — the raw stored value + // never reaches a document through the ordinary details walk. What export + // writes under those keys is the §3 attribution spelling `#`, + // put there by buildProperties, and that is not this list's business: + // this list is about stored values (§3). + for k := range derivedAttributionProperties { + stripped[k] = true + } + return stripped +} + +// envelopeLiftedKeys is every stored detail key some ENVELOPE field carries +// instead of `properties`: the four recommended lists when type properties +// are active (§2a), the nine icon/cover keys always (§2b), and the three +// relation-definition keys always (§2d — always even off relation documents, +// where a present value is dropped with a warning, because the refusal in +// `properties` is unconditional and export may not write what import +// refuses, I1/I2). The three sites that must agree about the lift — the term +// census, the properties emit and the id census — all ask this one function, +// because they used to ask typePropDetailKeys separately and a second lift +// list would have had to be added to each of them by hand. +func (e *exporter) envelopeLiftedKeys() map[string]bool { + lifted := liftedDetailKeys() + for k := range e.typePropDetailKeys() { + lifted[k] = true + } + for k := range propertySettingsLiftedDetailKeys() { + lifted[k] = true + } + // the five type_settings members, on TYPE documents only (§2a): off one, + // the same stored keys are ordinary properties — apiObjectKey is real + // data on 9,725 relation documents — so the lift is kind-scoped where + // §2b's and §2d's are unconditional + if e.isTypeDoc() { + for k := range typeSettingsLiftedDetailKeys() { + lifted[k] = true + } + } + return lifted +} + +// droppedPropertyKey answers the four kind- and value-scoped drops +// buildProperties applies to a detail key that has already passed the +// stripped / envelope-lifted / writable filter. It exists as ONE predicate +// because two sites have to agree about it, and for a while they did not: +// the term census (seedTermLedger) reserved a spelling for every surviving +// detail key, and these four populations are written NOWHERE, so a census +// that counted them made export stop being a fixpoint. A dropped key that +// contests a spelling degrades its rival in generation 1; generation 2 has +// no dropped key to contest with, so the rival un-degrades and a second +// export of the same object differs from the first. Reproduced on all four: +// `isHidden: false` beside a custom property named "Hidden" wrote +// "Hidden (b90aa1)" once and "Hidden" the next time. +// +// The warning sink is a parameter for vetSlug's reason: the census asks the +// question about every candidate key before any slot claims, and the one +// arm that warns would fire there for keys the emit is about to drop +// anyway, doubling the real warning. The claim site passes the real sink, +// so the warning still fires exactly where the value is actually lost. +// +// The attribution keys are NOT here. They are the opposite case — written +// by export and dropped by IMPORT — and the census models them as yielding +// claimants instead, which is a weaker thing than not counting them at all: +// a yielding key still takes an uncontested spelling, because it still +// occupies a member of the document it is written into. +func (e *exporter) droppedPropertyKey(k string, warn func(path, format string, args ...any)) bool { + if e.snapshot == nil || e.snapshot.Details == nil { + return false + } + // a TYPE document does not carry its own install provenance (§2a): + // eight keys, each admitted to the drop individually against the + // corpus — the verdicts live on typeProvenanceKeys. Silent, like the + // transient keys: the value describes the install, not the type, and + // the comparator consults the same predicate. + if e.isTypeDoc() { + if _, dropped := typeProvenanceKeys[k]; dropped { + return true + } + } + // a PARTICIPANT document does not carry createdDate (§3): the object + // is derived from the ACL and has no creation change, so the stored + // value is a load timestamp re-stamped on every cold build — the + // only field that drifted across a 1,164-document double-export + // comparison (22/22 participants, created_date only). Same silent + // drop as the type provenance above, and the comparator consults + // the same predicate (participantprovenance.go). + if DroppedParticipantProvenanceKey(e.sbType, k) { + return true + } + // a system-stamped key whose empty value says nothing a reader could + // act on (§15 #12): omitted, so schema documents stop paying ~20% of + // their bytes for it. The whitelist is deliberately short and the + // rule lives in systemtrim.go, where the comparator reads it too. + if DroppedEmptySystemProperty(k, e.snapshot.Details.Fields[k]) { + return true + } + // a name-over-number key can hold a vocabulary NAME or a number and + // nothing else (§3): Validate refuses any other string as an unknown + // name, so a stored string the vocabulary does not name has no + // written form — emitting it verbatim produced a document this + // package's own Validate rejects (I1). Dropped with a warning, the + // §2a typeSettingEnumValue policy; a stored string that IS a name + // survives and imports back as its number. + if vocab, named := namedEnumProperty(k); named { + if s, isStr := e.snapshot.Details.Fields[k].GetKind().(*types.Value_StringValue); isStr && !vocab.has(s.StringValue) { + warn("/properties", "%s %q on %q is not a name its vocabulary can hold and is dropped — "+ + "there is no way to write it", vocab.what, s.StringValue, k) + return true + } + } + return false +} + +func (e *exporter) buildProperties() *omap { + if e.snapshot.Details == nil { + return nil + } + stripped := strippedDetailKeys() + lifted := e.envelopeLiftedKeys() + var keys []string + for k := range e.snapshot.Details.Fields { + if isAttributionProperty(k) { + // stripped as a VALUE like every other derived key, and written + // as `#` — whenever the stored value holds an id at + // all. The name is a caption a resolver may or may not supply; + // the id is complete without it (§3, §9). + if _, ok := e.attributionRef(k); !ok { + continue + } + keys = append(keys, k) + continue + } + if stripped[k] || lifted[k] { + continue + } + // the four kind- and value-scoped drops, in one predicate the term + // census asks too (droppedPropertyKey) + if e.droppedPropertyKey(k, e.warn) { + continue + } + // a stored detail key is not necessarily a property name: real data + // holds an empty key and keys with control characters in them, and + // there is no way to write those (§3). Dropping them is what keeps + // Marshal's output valid — emitting one produced a document its own + // Validate rejects, which is the invariant §11 states. + if !isWritablePropertyKey(k) { + e.warn("/properties", "property key %q cannot be written in this format and is dropped", k) + continue + } + keys = append(keys, k) + } + // the document spells display names (§3), so the canonical alphabetical + // order is over the SPELLINGS, not the stored keys — the reader sorts + // what it sees. Values still resolve through the stored key. + // + // The claims below run over the STORED keys sorted, not over map order: + // which holder keeps a contested spelling must not depend on Go's map + // iteration, or the canonical form is not canonical and export∘import + // byte-stability is a coin flip on exactly the spaces that need it most. + // The collapse discipline itself — a spelling two stored keys agree on + // would merge into one JSON key and lose a value — lives in the term + // ledger (propertySlug), where every OTHER key slot claims through it too. + sort.Strings(keys) + type prop struct{ slug, key string } + props := make([]prop, 0, len(keys)) + for _, k := range keys { + props = append(props, prop{slug: e.propertySlug(k), key: k}) + } + sort.Slice(props, func(i, j int) bool { return props[i].slug < props[j].slug }) + ordered := make([]prop, 0, len(props)) + seen := map[string]bool{} + for _, wk := range wellKnownPropertyOrder { + for _, p := range props { + if p.key == wk { + ordered = append(ordered, p) + seen[p.key] = true + } + } + } + for _, p := range props { + if !seen[p.key] { + ordered = append(ordered, p) + } + } + m := &omap{} + for _, p := range ordered { + // presence of a property key is meaningful — it records that the + // property was set on the object — so values are written verbatim, + // including empty and default ones (§3); the omit-empty canon applies + // to block attributes and envelope fields only + m.set(p.slug, e.propertyValue(p.key, e.snapshot.Details.Fields[p.key])) + } + return m +} + +// warn reports a warning-grade issue through the caller's sink (§13): a thing +// export had to do that the author would want to know about, but that does not +// make the output invalid. Silent when no sink is wired. +func (e *exporter) warn(path, format string, args ...any) { + if e.opts.OnWarning == nil { + return + } + e.opts.OnWarning(Issue{Path: path, Message: fmt.Sprintf(format, args...)}) +} + +func (e *exporter) resolveFormat(key string) (model.RelationFormat, bool) { + return resolveFormatWith(e.opts, key) +} + +// resolveFormatWith applies the §3 resolution order: bundle first, then the +// caller's resolver. +func resolveFormatWith(opts Options, key string) (model.RelationFormat, bool) { + if f, err := bundle.GetRelationFormat(domain.RelationKey(key)); err == nil { + return f, true + } + if opts.ResolveFormat != nil { + return opts.ResolveFormat(domain.RelationKey(key)) + } + return 0, false +} + +// isAttributionProperty reports the two derived properties that name a member +// — `creator` and `lastModifiedBy` (validate.go). Their stored value is a +// participant id and the document writes `#` — the folded id with +// the member's name as the informative suffix (§3, §9). +func isAttributionProperty(key string) bool { + _, ok := derivedAttributionProperties[key] + return ok +} + +// attributionRefOf renders an attribution value as `#` (§3): the +// stored participant id through the §9 participant fold, with the member's +// display name as the informative suffix when a resolver names them. ok is +// false — and the property is then written NOWHERE — only when the stored +// value holds no id at all: a bare id is a complete answer, since the id is +// the resolvable half and the suffix is a caption. +// +// The resolver is asked about the STORED id (the composite the space +// indexes), and the name goes through refNameLabel — the identifier grammar +// admits no `#`, so a raw display name can never break the split, and a +// blank or vanishing name yields a bare id rather than a dangling `#`. The +// suffix does NOT ride Options.RefNames: these two properties are dropped on +// import (no round-trip byte-stability is at stake), and the name is the +// reason the line is worth writing at all — that was measured. +// +// The value is read as a LIST and the first id answers, because a stored +// detail may hold either shape; the relation is `maxCount: 1` and 36,966 real +// objects held exactly one id each, so there is no second id to lose. +func attributionRefOf(v *types.Value, opts Options) (string, bool) { + ids := valueStringList(v) + if len(ids) == 0 { + return "", false + } + // A participant composite whose identity half is EMPTY addresses nobody + // — `_participant__`, 86 characters of the document's own space + // restated with no member behind it. Real data: 9,103 of 37,429 + // production objects store exactly that in lastModifiedBy (derived when + // the writer's identity was blank). It is the id-shaped analogue of a + // blank name, and the property is omitted rather than spelled — the same + // verdict the blank name gets at refNameLabel. + if strings.HasPrefix(ids[0], domain.ParticipantPrefix) { + if _, identity, err := domain.ParseParticipantId(ids[0]); err == nil && identity == "" { + return "", false + } + } + out := opts.foldParticipantRef(ids[0]) + if opts.ResolveParticipants != nil { + if name, ok := opts.ResolveParticipants.ParticipantName(ids[0]); ok { + if label := refNameLabel(name); label != "" { + out += refNameSep + label + } + } + } + return out, true +} + +// attributionRef answers attributionRefOf for a stored detail of this +// snapshot. Both the emit (buildProperties) and the term census +// (seedTermLedger) ask it, and they must agree: the census reserves a +// spelling exactly when the emit is going to write it. +func (e *exporter) attributionRef(key string) (string, bool) { + return attributionRefOf(e.detail(key), e.opts) +} + +func (e *exporter) propertyValue(key string, v *types.Value) any { + // the derived attribution properties are spelled `#` (§3): the + // folded participant id — resolvable — with the member's name as the + // informative suffix. Nil is the "no value" answer, and a whole-document + // export never reaches it: buildProperties omits the key rather than + // writing a null, having asked the same question first. + if isAttributionProperty(key) { + if ref, ok := attributionRefOf(v, e.opts); ok { + return ref + } + return nil + } + // a name-over-number key is stored as a number and named in the format + // (§3); a number outside the vocabulary falls through and exports + // unchanged, and a stored STRING never reaches here — buildProperties + // drops one the vocabulary does not name (I1) and writes a known name + // through the verbatim fall-through below, where import maps it back. + if vocab, named := namedEnumProperty(key); named { + if n, isNum := v.GetKind().(*types.Value_NumberValue); isNum { + if name := vocab.name(n.NumberValue); name != "" { + return name + } + } + } + format, ok := e.resolveFormat(key) + if !ok { + return protoValueToJSON(v) + } + switch format { + case model.RelationFormat_date: + if n, isNum := v.GetKind().(*types.Value_NumberValue); isNum { + if s, ok := formatDateValue(n.NumberValue); ok { + return s + } + // no RFC 3339 form: emitting one anyway would write a string + // parseDate cannot read, so the value would come back as a string + // on a date property and stay that way (byte-stable, so nothing + // corrects it). The raw number round-trips instead. + e.warn("/properties/"+key, + "date %v has no RFC 3339 form (outside years 0000-9999), so it is written as a raw number; "+ + "a value this large is usually milliseconds where seconds belong", n.NumberValue) + } + case model.RelationFormat_status, model.RelationFormat_tag: + // a select vocabulary is a LIST of references too, so the same rule + // applies (§9): the stored `_missing_object` sentinel names an option + // that is gone, and there is nothing left to write — the corpus + // carries `"tag": ["_missing_object"]` beside an EMPTY `option_ids` + // legend, so not even a name survives to show. + // + // Only the sentinel drops here, not a whole existence check: an + // option id lives in the option namespace, and optionName already + // resolves it or leaves it as written. + var out []any + for _, id := range valueStringList(v) { + if id == missingObjectId { + continue + } + out = append(out, e.optionName(key, id)) + } + return out + case model.RelationFormat_object, model.RelationFormat_file: + // a LIST-valued reference slot: an entry the space does not hold — + // the stored sentinel included — is dropped, because a list + // expresses absence by being shorter (§9). The emptied list stays + // `[]`, never omitted: presence of the key is meaningful (§3), and + // dropping dangling entries must not erase the fact that the + // property was set. + var out []any + for _, id := range valueStringList(v) { + if e.droppedMissingListEntry("/properties/"+key, id) { + continue + } + out = append(out, e.objectRef(id)) + } + return out + } + return protoValueToJSON(v) +} + +// optionName is the one site where export substitutes a name for an option id +// (§3), in property values, filter values and custom orders alike — so it is +// also the one site that records what that name stood for (optionrefs.go). +// The legend entry rides with the substitution rather than behind a flag: +// identity is not compaction, and a document that spells a name without +// saying which option it was is the lossy half this legend exists to close. +func (e *exporter) optionName(key, id string) string { + if e.opts.ResolveOptions != nil { + if name, ok := e.opts.ResolveOptions.OptionName(domain.RelationKey(key), id); ok { + e.recordOptionRef(key, name, id) + return name + } + } + return id +} + +// valueStringList reads a value as a list of strings, accepting the single +// string form. +func valueStringList(v *types.Value) []string { + if s := v.GetStringValue(); s != "" { + return []string{s} + } + var out []string + for _, el := range v.GetListValue().GetValues() { + if s := el.GetStringValue(); s != "" { + out = append(out, s) + } + } + return out +} + +// +// ---- blocks ---- +// + +// orEmpty substitutes an empty message for a nil one (proto semantics: a nil +// message equals its zero value). +func orEmpty[T any](p *T) *T { + if p == nil { + return new(T) + } + return p +} + +// isStructural reports blocks that are derivable and dropped on export (§7). +func isStructural(b *model.Block) bool { + switch c := b.Content.(type) { + case *model.BlockContentOfLayout: + return orEmpty(c.Layout).Style == model.BlockContentLayout_Header + case *model.BlockContentOfText: + style := orEmpty(c.Text).Style + return style == model.BlockContentText_Title || + style == model.BlockContentText_Description + case *model.BlockContentOfFeaturedRelations: + return true + } + return false +} + +// isTransparentContainer reports the §7a transparent containers: a block +// that contributes containment and NOTHING else, so the document spells its +// children and not it. Two shapes qualify — a `Layout/Div`, which is the +// editor's fan-out wrapper (state.wrapChildrenToDiv mints one whenever a +// parent exceeds 40 children: a rendering budget, not an authored block), +// and a block whose content oneof is unset, which is legacy data with no +// content to render at all. +// +// The test is on CONTENT, never on the `div-` id prefix the normalizer +// happens to mint: keying on a prefix would make id SPELLING semantically +// load-bearing — the worst thing to freeze — and would leave an authored +// `{"type": "group"}` round-tripping into a permanent wrapper. +func isTransparentContainer(b *model.Block) bool { + switch c := b.Content.(type) { + case nil: + return true + case *model.BlockContentOfLayout: + return orEmpty(c.Layout).Style == model.BlockContentLayout_Div + } + return false +} + +// warnLiftedAttributes reports a transparent container that carried block +// attributes, because the lift drops them with it (§7a) and the loss is +// otherwise invisible. Free on real data — all 7,303 wrappers in the +// production corpus carry none — and it turns a silent loss into a visible +// one for a document that authored a `group` with an attribute on it. +func (e *exporter) warnLiftedAttributes(b *model.Block) { + if e.opts.OnWarning == nil { + return + } + if b.Align == model.Block_AlignLeft && b.VerticalAlign == model.Block_VerticalAlignTop && + b.BackgroundColor == "" && len(b.Fields.GetFields()) == 0 { + return + } + e.warn("/blocks", "block %s is a transparent container: it is lifted, and the attributes on it are dropped", b.Id) +} + +func (e *exporter) buildBlocks() ([]any, error) { + root := e.blocks[e.rootId] + if root == nil { + return nil, nil + } + var out []any + if err := e.appendBlocksFlat(&out, root.ChildrenIds, 0, true); err != nil { + return nil, err + } + return out, nil +} + +// appendBlocksFlat walks a subtree in pre-order and appends each block to out +// with its depth as the indent field — the flat encoding (§4 F1–F2). A block +// dropped by blockToJSON (structural, visited, content-less leaf) drops its +// whole subtree, matching the nested encoding's semantics. +func (e *exporter) appendBlocksFlat(out *[]any, ids []string, depth int, topLevel bool) error { + for _, id := range ids { + b := e.blocks[id] + if b == nil { + continue + } + if topLevel && isStructural(b) { + continue + } + // §7a: a transparent container is not a block. Emit nothing for it + // and walk its children at ITS OWN depth, carrying its topLevel flag + // — the children take the position it held. + // + // Before the depth check, so both the value compared against the + // bound and the emitted indent are post-lift; and marking it visited + // HERE, because the lift skips blockToJSON, which is where that mark + // is set — a ChildrenIds cycle through a chain of containers would + // otherwise recurse until the stack gives out. + if isTransparentContainer(b) { + if b.Id != "" { + if e.visited[b.Id] { + continue + } + e.visited[b.Id] = true + } + e.warnLiftedAttributes(b) + if err := e.appendBlocksFlat(out, b.ChildrenIds, depth, topLevel); err != nil { + return err + } + continue + } + emitDepth := depth + if depth > maxBlockIndent { + if e.opts.OnWarning == nil { + return fmt.Errorf("block %s: nesting depth %d exceeds the format bound %d", id, depth, maxBlockIndent) + } + // read path (C11): degrade rather than fail — clamp the indent and + // keep the content instead of erroring the whole document. + e.opts.OnWarning(Issue{Path: "/blocks", Message: fmt.Sprintf("block %s: nesting depth %d exceeds the bound %d — indent clamped", id, depth, maxBlockIndent)}) + emitDepth = maxBlockIndent + } + m, withChildren, err := e.blockToJSON(b, emitDepth) + if err != nil { + return err + } + if m == nil { + continue + } + e.recordEmitted(b.Id) + *out = append(*out, m) + if withChildren { + if err := e.appendBlocksFlat(out, b.ChildrenIds, depth+1, false); err != nil { + return err + } + } + } + return nil +} + +// recordEmitted notes that this run wrote the block/row/column/view stored +// under id. Only the census probe collects; every other run no-ops. +func (e *exporter) recordEmitted(id string) { + if e.emitted != nil && id != "" { + e.emitted[id] = true + } +} + +func (e *exporter) localId(id string) string { + if e.opts.compactBlockLabels() { + if short, ok := e.localIds[id]; ok { + return short + } + } + return id +} + +// blockToJSON renders one block at the given depth (its indent, written +// first per the §4 canonical key order). The returned bool reports whether +// the caller should descend into the block's children. +func (e *exporter) blockToJSON(b *model.Block, depth int) (*omap, bool, error) { + // a snapshot's block graph is untrusted: without this, a ChildrenIds + // cycle recurses to an unrecoverable stack overflow, and a block shared + // by two parents is emitted twice (duplicate ids fail validation) + if b.Id != "" { + if e.visited[b.Id] { + return nil, false, nil + } + e.visited[b.Id] = true + } + m := &omap{} + m.setNonEmpty("indent", depth) + if !e.opts.OmitIds { + m.setNonEmpty("id", e.blockLabel(b.Id)) + } + liftedFields := map[string]bool{} + withChildren := true + + // nil inner messages are proto-equivalent to empty ones — never panic + switch c := b.Content.(type) { + case nil: + // legacy content-less blocks exist in old accounts: relation objects + // carry a bare wrapper around their "used in" dataview, and pages can + // hold orphaned empty leaves. They are transparent containers (§7a) + // and both callers handle them before this: appendBlocksFlat lifts, + // cellToJSON renders an empty cell. Kept as the drop it would have + // been, so a future caller cannot mint a block out of no content. + return nil, false, nil + case *model.BlockContentOfText: + if err := e.textToJSON(m, b, orEmpty(c.Text), liftedFields); err != nil { + return nil, false, err + } + case *model.BlockContentOfFile: + // file blocks are leaves in the editor, but legacy data holds real + // text children under them — dropping those would be silent loss + e.fileToJSON(m, orEmpty(c.File)) + case *model.BlockContentOfBookmark: + bm := orEmpty(c.Bookmark) + m.set("type", "bookmark") + m.setNonEmpty("url", bm.Url) + // a singular reference slot: a target the space does not hold is + // written as the sentinel, never as if it existed (§9) + m.setNonEmpty("object_id", e.singularObjectRef("/blocks", "bookmark object_id", bm.TargetObjectId)) + withChildren = false + case *model.BlockContentOfLink: + l := orEmpty(c.Link) + m.set("type", "link") + m.setNonEmpty("object_id", e.singularObjectRef("/blocks", "link object_id", l.TargetBlockId)) + if l.CardStyle != model.BlockContentLink_Text { + m.setNonEmpty("card_style", cardStyleNames.name(l.CardStyle)) + } + if l.IconSize != model.BlockContentLink_SizeNone { + m.setNonEmpty("icon_size", iconSizeNames.name(l.IconSize)) + } + if l.Description != model.BlockContentLink_None { + m.setNonEmpty("description", linkDescriptionNames.name(l.Description)) + } + m.setNonEmpty("properties", stringsToAny( + e.slotPropertySlugs(l.Relations, "a link block's `properties` entry"))) + withChildren = false + case *model.BlockContentOfDiv: + m.set("type", "divider") + if style := orEmpty(c.Div).Style; style != model.BlockContentDiv_Line { + m.setNonEmpty("style", divStyleNames.name(style)) + } + withChildren = false + case *model.BlockContentOfLayout: + switch orEmpty(c.Layout).Style { + case model.BlockContentLayout_Row: + m.set("type", "row") + case model.BlockContentLayout_Column: + m.set("type", "column") + default: + // header and stray table wrappers are structural (§7); a Div is + // a transparent container (§7a), lifted by appendBlocksFlat and + // turned into an empty cell by cellToJSON, so it never arrives + // here — and if it ever did, dropping it is the same answer + return nil, false, nil + } + case *model.BlockContentOfTable: + if err := e.tableToJSON(m, b); err != nil { + return nil, false, err + } + withChildren = false + case *model.BlockContentOfLatex: + lx := orEmpty(c.Latex) + m.set("type", "embed") + if lx.Processor != model.BlockContentLatex_Latex { + m.setNonEmpty("processor", processorNames.name(lx.Processor)) + } + m.setNonEmpty("text", lx.Text) + withChildren = false + case *model.BlockContentOfTableOfContents: + m.set("type", "table_of_contents") + withChildren = false + case *model.BlockContentOfRelation: + // a property block IS a reference to a property, so one with no key + // refers to nothing. It used to be emitted as `{"type": "property"}` + // — a block the schema accepted, import stored with the empty key, + // and the next export wrote again, forever. Dropped, like the + // nameless sort and the nameless column (§6). + if orEmpty(c.Relation).Key == "" { + e.warn("", "a property block names no property and is dropped; "+ + "a key slot has to name something") + return nil, false, nil + } + // an unwritable stored key drops the block the same way — warned by + // slotPropertySlug — instead of being emitted verbatim (§3) + relSlug := e.slotPropertySlug(orEmpty(c.Relation).Key, "a property block") + if relSlug == "" { + return nil, false, nil + } + m.set("type", "property") + m.setNonEmpty(memberProperty, relSlug) + withChildren = false + case *model.BlockContentOfDataview: + if err := e.dataviewToJSON(m, orEmpty(c.Dataview)); err != nil { + return nil, false, err + } + withChildren = false + case *model.BlockContentOfWidget: + w := orEmpty(c.Widget) + m.set("type", "widget") + if w.Layout != model.BlockContentWidget_Link { + m.setNonEmpty("layout", widgetLayoutNames.name(w.Layout)) + } + m.setNonEmpty("limit", w.Limit) + m.setNonEmpty("view_id", w.ViewId) + m.setNonEmpty("auto_added", w.AutoAdded) + case *model.BlockContentOfChat: + m.set("type", "chat") + withChildren = false + case *model.BlockContentOfFeaturedRelations: + m.set("type", "featured_properties") + withChildren = false + case *model.BlockContentOfIcon: + m.set("type", "icon") + m.setNonEmpty("name", orEmpty(c.Icon).Name) + withChildren = false + case *model.BlockContentOfSmartblock: + return nil, false, nil + default: + if e.opts.OnWarning != nil { + // read path (C11): drop the unrepresentable block with a warning + // instead of failing the whole read. + e.opts.OnWarning(Issue{Path: "/blocks", Message: fmt.Sprintf("block %s: content type %T has no JSON mapping — dropped", b.Id, b.Content)}) + return nil, false, nil + } + return nil, false, fmt.Errorf("block %s: content type %T has no JSON mapping", b.Id, b.Content) + } + + e.finishBlockJSON(m, b, liftedFields) + return m, withChildren, nil +} + +// finishBlockJSON writes the common block tail: align, verticalAlign, +// backgroundColor, fields — in the §4 canonical order. +func (e *exporter) finishBlockJSON(m *omap, b *model.Block, liftedFields map[string]bool) { + if b.Align != model.Block_AlignLeft { + m.setNonEmpty("align", alignNames.name(b.Align)) + } + if b.VerticalAlign != model.Block_VerticalAlignTop { + m.setNonEmpty("vertical_align", verticalAlignNames.name(b.VerticalAlign)) + } + m.setNonEmpty("background_color", b.BackgroundColor) + m.setNonEmpty("fields", e.fieldsToJSON(b.Fields, liftedFields)) +} + +func (e *exporter) fieldsToJSON(fields *types.Struct, lifted map[string]bool) *omap { + if fields == nil || len(fields.Fields) == 0 { + return nil + } + m := &omap{} + keys := make([]string, 0, len(fields.Fields)) + for k := range fields.Fields { + if !lifted[k] { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, k := range keys { + m.set(k, protoValueToJSON(fields.Fields[k])) + } + return m +} + +func (e *exporter) textToJSON(m *omap, b *model.Block, t *model.BlockContentText, liftedFields map[string]bool) error { + style := t.Style + // deprecated Header4 exports as heading3 (§5) + if style == model.BlockContentText_Header4 { + style = model.BlockContentText_Header3 + } + typ := textStyleNames.name(style) + if typ == "" { + return fmt.Errorf("block %s: text style %v has no JSON mapping", b.Id, t.Style) + } + m.set("type", typ) + + if style == model.BlockContentText_Checkbox { + m.setNonEmpty("checked", t.Checked) + } + if style == model.BlockContentText_Callout { + // the same typed shape as the object icon (§2b), restricted to the + // two kinds a block can hold. Shipping the envelope field without + // this would leave two icon conventions inside one document, which is + // the defect being removed. + m.setNonEmpty("icon", e.calloutIcon(t)) + } + if style == model.BlockContentText_Code { + if b.Fields != nil { + if lang := b.Fields.Fields[codeLangField].GetStringValue(); lang != "" { + m.set("language", lang) + liftedFields[codeLangField] = true + } + } + // literal text; stored marks and color dropped (§8.4, §11) + m.setNonEmpty("text", t.Text) + return nil + } + m.setNonEmpty("color", t.Color) + // exportMarks applies the missing-reference rule to mention targets + // before the codec renders them (§8, §9); with no existence capability + // wired it returns the marks untouched + m.setNonEmpty("text", renderInline(t.Text, e.exportMarks("/blocks", t.Marks.GetMarks()))) + return nil +} + +func (e *exporter) fileToJSON(m *omap, f *model.BlockContentFile) { + typ := fileTypeNames.name(f.Type) + if typ == "" { + typ = "file" // Type_None (§5) + } + m.set("type", typ) + objectId := f.TargetObjectId + if objectId == "" { + objectId = f.Hash // legacy content address migrates to objectId + } + // a singular reference slot: a target the space does not hold is written + // as the sentinel, never as if it existed (§9) — the legacy hash arm + // included, because in the un-migrated spaces that still store one the + // hash IS the file object's index row id + m.setNonEmpty("object_id", e.singularObjectRef("/blocks", typ+" object_id", objectId)) + m.setNonEmpty("name", f.Name) + m.setNonEmpty("mime_type", f.Mime) + m.setNonEmpty("size", f.Size_) + if f.Style != model.BlockContentFile_Auto { + m.setNonEmpty("style", fileStyleNames.name(f.Style)) + } + if f.AddedAt != 0 { + // addedAt is a string in the schema, so there is no number form to + // fall back to (§5): an unrepresentable timestamp is dropped rather + // than written as a string no reader can parse back + if s, ok := formatDate(f.AddedAt); ok { + m.set("added_at", s) + } else { + e.warn("", "file block: added_at %d has no RFC 3339 form (outside years 0000-9999), so it is omitted", f.AddedAt) + } + } +} + +func stringsToAny(ss []string) []any { + var out []any + for _, s := range ss { + if s != "" { + out = append(out, s) + } + } + return out +} + +// +// ---- compact ids (§9a) ---- +// + +// emittedLocalIds is the id census's population: the doc-local ids this +// export actually SERVES — every block it writes, plus the table rows, +// columns and dataview views inside them. It runs the block emit a second +// time on a throwaway exporter (own visited map, own ledgers, warnings +// swallowed so nothing is reported twice) rather than re-deriving the drop +// rules, because a second statement of "what export emits" would be a second +// thing to keep in step with blockToJSON, and the census is only correct +// while the two agree exactly (TestExport_CensusPopulationIsWhatExportEmits). +func (e *exporter) emittedLocalIds() map[string]bool { + opts := e.opts + if opts.OnWarning != nil { + // keep the nil-ness — several drop-vs-error decisions read it — but + // silence the probe: the real run reports the same issues. + opts.OnWarning = func(Issue) {} + } + probe := &exporter{ + opts: opts, + snapshot: e.snapshot, + sbType: e.sbType, + blocks: e.blocks, + rootId: e.rootId, + visited: map[string]bool{}, + emitted: map[string]bool{}, + } + // an error here is the real run's error too, and it fails there with the + // message the caller should see; the partial census costs nothing. + _, _ = probe.buildBlocks() + // A derived cell id is not SPELLED — a cell carries no id in the flat form + // — but unlike every other unspelled block it is not gone from the snapshot + // the round trip rebuilds: import re-derives `rowId-colId` from row and + // column ids that ARE spelled, so the same cell ids come back, and + // reserving them is stable across generations. Leaving them out is what + // makes a compacted COLUMN label an ambiguous suffix of its own cells in + // the live object (a cell id ends with the whole column id, so its last + // five characters ARE the column's label), and §9a promises a served label + // is neither equal to nor an ambiguous suffix of another served id. + // Measured: without this, 899 corpus documents serve a column label that a + // suffix resolver could match against a cell block instead. + for _, id := range probe.derivedCellIds(probe.emitted) { + probe.emitted[id] = true + } + return probe.emitted +} + +// buildLabelPlan works out which doc-local block/row/column/view ids may be +// relabeled to a short suffix (§9a). It walks BOTH id populations to do it: +// the doc-local ids that are the relabeling candidates, and every OBJECT id +// the document references — not because an object id is ever compacted (none +// is, §9a), but because every one of them is spelled verbatim in the output, +// so a label equal to one would make two different things answer to one name. +// mintedSuffixLabels' own census counts local ids only, so that avoid-set is +// the sole guard against it (TestExport_CompactLabelCannotTakeAServedId). +// +// **The local population is what export EMITS, not what the snapshot holds** +// (§9a, mirroring §3's term census). A block the document does not spell — +// a transparent container (§7a), a structural block (§7), a content-less +// leaf, anything unreachable — is gone from the +// snapshot the round trip rebuilds, so reserving its suffix slot makes +// `Export(S)` and `Export(Import(Export(S)))` disagree: the first read keeps +// a paragraph's id full because an invisible block shares its 5-char tail, +// the second compacts it. That is guarantee 3 (§11), broken on the API's +// default read shape. The protection lost is illusory anyway — a container +// the editor re-creates gets a FRESH id no census could have reserved +// against. +func (e *exporter) buildLabelPlan() { + objects := map[string]bool{} + locals := e.emittedLocalIds() + addObject := func(id string) { + if id != "" { + objects[id] = true + // the document spells the FOLDED form of a participant ref + // (§9), so the avoid-set carries that spelling too — the raw + // composite stays as well, since a suffix-trimming reader + // recovers it + if folded := e.opts.foldParticipantRef(id); folded != id { + objects[folded] = true + } + } + } + + for _, b := range e.snapshot.Blocks { + if b == nil { + continue + } + switch c := b.Content.(type) { + case *model.BlockContentOfText: + t := orEmpty(c.Text) + for _, mk := range t.Marks.GetMarks() { + if mk == nil { + continue + } + switch { + case mk.Type == model.BlockContentTextMark_Mention || mk.Type == model.BlockContentTextMark_Object: + addObject(mk.Param) + case mk.Type == model.BlockContentTextMark_Link && isObjectLink(mk.Param): + // normalizes to an Object mark on render (§8.3) + id, _ := parseObjectLink(mk.Param) + addObject(id) + } + } + addObject(t.IconImage) + case *model.BlockContentOfFile: + f := orEmpty(c.File) + if f.TargetObjectId != "" { + addObject(f.TargetObjectId) + } else { + addObject(f.Hash) + } + case *model.BlockContentOfBookmark: + addObject(orEmpty(c.Bookmark).TargetObjectId) + case *model.BlockContentOfLink: + addObject(orEmpty(c.Link).TargetBlockId) + case *model.BlockContentOfDataview: + dv := orEmpty(c.Dataview) + addObject(dv.TargetObjectId) + for _, v := range dv.Views { + if v == nil { + continue + } + addObject(v.DefaultTemplateId) + addObject(v.DefaultObjectTypeId) + for _, f := range flattenFilters(v.Filters) { + if format, ok := e.dvFormat(dv, f.RelationKey); ok && + (format == model.RelationFormat_object || format == model.RelationFormat_file) { + for _, id := range valueStringList(f.Value) { + addObject(id) + } + } + } + for _, s := range v.Sorts { + if s == nil { + continue + } + if format, ok := e.dvFormat(dv, s.RelationKey); ok && + (format == model.RelationFormat_object || format == model.RelationFormat_file) { + for _, cv := range s.CustomOrder { + for _, id := range valueStringList(cv) { + addObject(id) + } + } + } + } + } + for _, oo := range dv.ObjectOrders { + if oo == nil { + continue + } + for _, id := range oo.ObjectIds { + addObject(id) + } + } + } + } + + if e.snapshot.Details != nil { + stripped := strippedDetailKeys() + lifted := e.envelopeLiftedKeys() + for key, v := range e.snapshot.Details.Fields { + if stripped[key] || lifted[key] { + // a stripped key is not written at all, so it names nothing. + // A LIFTED one is written somewhere else, and the ids it + // writes there are fed in explicitly below — this used to say + // "lifted properties never appear as ids", which was true + // only while the recommended lists were the sole lift. + // `relationFormatObjectTypes` (§2d) is the deliberate + // exception in the other direction: its entries leave this + // walk and are NOT fed back, because the envelope writes + // them as TYPE-KEY terms, not object references — the same + // slot type_properties[].object_types is, which has never + // had census duty. A term is never textually joined with a + // block id, so a compact label equal to one collides with + // nothing. + continue + } + format, ok := e.resolveFormat(key) + if ok && (format == model.RelationFormat_object || format == model.RelationFormat_file) { + for _, id := range valueStringList(v) { + addObject(id) + } + } + } + } + // the typed envelope fields write object ids of their own (§2b). `icon`'s + // used to reach this set through the property walk above, because + // iconImage is a `file` relation; `cover`'s never did, because coverId is + // declared `longtext` — so a compact label equal to a file-backed cover id + // was always possible and is closed here for the first time. + for _, id := range e.liftedObjectIds() { + addObject(id) + } + if e.snapshot.Collections != nil { + if v := e.snapshot.Collections.Fields[storeKeyItems]; v != nil { + for _, el := range v.GetListValue().GetValues() { + addObject(el.GetStringValue()) + } + } + } + // the envelope id is never compacted (§9a) + delete(objects, e.objectId()) + + // no label may equal a full id present in the document (§9a); the avoid + // set covers every id this export knows about, object ids included — + // every one of those is written verbatim now, which makes the object + // half of this set matter MORE than it did when they were compacted + fullIds := map[string]bool{e.objectId(): true} + for id := range objects { + fullIds[id] = true + } + for id := range locals { + fullIds[id] = true + } + // only machine-minted opaque ids relabel (isMintedLocalId); every id that + // keeps its full spelling is reserved through the fullIds avoid-set, so no + // label can alias a served id — and the census inside mintedSuffixLabels + // runs over ALL local ids, so a label cannot be an ambiguous suffix of one + // either. For the LOCAL population the census is the binding guard (it + // counts a short id as itself); the avoid-set carries the OBJECT + // population, which the census never sees. Labels stay dash-free as + // before: '-' is the derived-cell-id separator and forbidden in row/column + // ids (§6.1) — minted suffixes are hex, so the check is a backstop. + e.localIds = mintedSuffixLabels(setToSlice(locals), compactIdMinLen, func(candidate string) bool { + return fullIds[candidate] || isInvalidLocalLabel(candidate) + }) +} + +// isBlockIdLabel reports whether s matches the block-id pattern +// ^[A-Za-z0-9_-]{1,64}$ — the charset §4 puts on a block, row or column id. +// Named for the deleted `refs` legend, whose plain keys carried the same +// charset; the id sanitizer is the caller that remains. +func isBlockIdLabel(s string) bool { + if len(s) == 0 || len(s) > 64 { + return false + } + for _, r := range s { + if !isLabelRune(r) && r != '-' { + return false + } + } + return true +} + +// isInvalidLocalLabel rejects relabel candidates for blocks/rows/columns/ +// views: the row/column charset has no dash (§6.1), which also keeps labels +// clear of derived cell ids. +func isInvalidLocalLabel(s string) bool { + if len(s) == 0 || len(s) > 64 { + return true + } + for _, r := range s { + if !isLabelRune(r) { + return true + } + } + return false +} + +func isLabelRune(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' +} + +func flattenFilters(filters []*model.BlockContentDataviewFilter) []*model.BlockContentDataviewFilter { + var out []*model.BlockContentDataviewFilter + var walk func([]*model.BlockContentDataviewFilter) + walk = func(fs []*model.BlockContentDataviewFilter) { + for _, f := range fs { + if f == nil { + continue + } + out = append(out, f) + walk(f.NestedFilters) + } + } + walk(filters) + return out +} + +func setToSlice(set map[string]bool) []string { + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/pkg/lib/anyblockjson/filters.go b/pkg/lib/anyblockjson/filters.go new file mode 100644 index 0000000000..da15efa5b6 --- /dev/null +++ b/pkg/lib/anyblockjson/filters.go @@ -0,0 +1,269 @@ +package anyblockjson + +// filters.go exposes the §6.2 filter/sort codec at fragment granularity: +// a bare filters array or sorts array — the shapes the API v2 query surface +// carries in request bodies — converts to the model tree +// (`model.BlockContentDataviewFilter` / `Sort`) through the same importer +// the whole-document dataview path uses, so the structured `filters` request +// form and the parsed compact filter string land on ONE internal tree. +// +// Validation runs the same checks the document path applies to a view's +// filters: enum vocabulary (conditions, date presets, sort directions), +// the counting-preset operand rule, the dynamic-placeholder format rule, and +// the unguarded-date-comparison warning (SPEC §6.2 — surfaced through +// Options.OnWarning, the C11 channel). Issue paths are fragment-relative: +// `/filters/i/…` and `/sorts/i/…`. + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// filterConditionList renders the condition vocabulary for error messages. +var filterConditionList = strings.Join([]string{ + "equal", "not_equal", "greater", "less", "greater_or_equal", "less_or_equal", + "contains", "not_contains", "in", "not_in", "empty", "not_empty", + "all_in", "not_all_in", "exact_in", "not_exact_in", "exists", +}, ", ") + +// datePresetList renders the date-preset vocabulary for error messages. +var datePresetList = strings.Join([]string{ + "yesterday", "today", "tomorrow", "last_week", "current_week", "next_week", + "last_month", "current_month", "next_month", "number_of_days_ago", + "number_of_days_now", "last_year", "current_year", "next_year", +}, ", ") + +// UnmarshalFilters converts a §6.2 structured filters array (the top-level +// nodes combine with an implicit AND) into model filter nodes. Select values +// resolve through Options.ResolveOptions exactly as on document import — +// wire a read-only resolver on query paths and a creating resolver on write +// paths. Formats rehydrate through Options.ResolveFormat. Errors wrap +// *ValidationError with `/filters/i/…` paths; warning-grade findings (the +// §6.2 unguarded-date-comparison trap) ride Options.OnWarning. +func UnmarshalFilters(raw json.RawMessage, opts Options) ([]*model.BlockContentDataviewFilter, error) { + var nodes []jsonFilter + if err := jsonUnmarshal(raw, &nodes); err != nil { + // the schema states the shape with a path where it can — a payload + // that is not an array, a member of the wrong type — and the raw + // decode error remains only for what never reaches it + if fragErr := validateQueryFragment("filters", raw); isValidationError(fragErr) { + return nil, fragErr + } + return nil, fmt.Errorf("decode filters: %w", err) + } + var generic []any + if err := jsonUnmarshal(raw, &generic); err != nil { + return nil, fmt.Errorf("decode filters: %w", err) + } + + var issues []Issue + addIssue := func(path, format string, args ...any) { + issues = append(issues, Issue{Path: path, Message: fmt.Sprintf(format, args...)}) + } + warnIssue := func(path, format string, args ...any) { + if opts.OnWarning != nil { + opts.OnWarning(Issue{Path: path, Message: fmt.Sprintf(format, args...)}) + } + } + + validateFilterVocabulary(nodes, "/filters", addIssue) + + // the document path's per-view semantic checks (counting-preset operand, + // placeholder format rule, the unguarded date-less warning) run on the + // generic form, with formats resolved from the space instead of the + // dataview's properties list + formats := referencedFormats(nodes, opts) + checkDateFilters(map[string]any{"filters": generic}, formats, + func(prop string) bool { return formats[prop] == "date" }, + "", addIssue, warnIssue) + + if len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + // §12, the closed shape: the same $defs/filterNode fragment the + // whole-document path holds a view's filters to — the pass that refuses + // an unknown member, which the tolerant decode above silently drops. It + // runs after the vocabulary pass so the enum wording above keeps owning + // the faults it can name, and only for what that pass had no rule for. + if len(nodes) > 0 { + if err := validateQueryFragment("filters", raw); err != nil { + return nil, err + } + } + + imp := &importer{opts: opts, doc: opts.fragmentDoc()} + dv := &model.BlockContentDataview{} + out := make([]*model.BlockContentDataviewFilter, 0, len(nodes)) + for _, jf := range nodes { + out = append(out, imp.filterFromJSON(jf, dv)) + } + return out, nil +} + +// UnmarshalSorts converts a §6.2 sorts array into model sort nodes. Formats +// rehydrate through Options.ResolveFormat; custom-order select values +// resolve through Options.ResolveOptions. Errors wrap *ValidationError with +// `/sorts/i/…` paths. +func UnmarshalSorts(raw json.RawMessage, opts Options) ([]*model.BlockContentDataviewSort, error) { + var sorts []jsonSort + if err := jsonUnmarshal(raw, &sorts); err != nil { + // same split as UnmarshalFilters: the schema's path-addressed + // verdict where it has one, the raw decode error otherwise + if fragErr := validateQueryFragment("sorts", raw); isValidationError(fragErr) { + return nil, fragErr + } + return nil, fmt.Errorf("decode sorts: %w", err) + } + var issues []Issue + for i, js := range sorts { + if js.Property == "" { + issues = append(issues, Issue{ + Path: fmt.Sprintf("/sorts/%d/property", i), + Message: "a sort needs a property key", + }) + } + if js.Direction != "" && !sortDirectionNames.has(js.Direction) { + issues = append(issues, Issue{ + Path: fmt.Sprintf("/sorts/%d/direction", i), + Message: fmt.Sprintf("unknown direction %q — allowed: asc, desc, custom", js.Direction), + }) + } + if js.EmptyPlacement != "" && !emptyPlacementNames.has(js.EmptyPlacement) { + issues = append(issues, Issue{ + Path: fmt.Sprintf("/sorts/%d/empty_placement", i), + Message: fmt.Sprintf("unknown empty_placement %q — allowed: start, end", js.EmptyPlacement), + }) + } + } + if len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + // §12, the closed shape: the same $defs/sort fragment the whole-document + // path holds a view's sorts to — after the vocabulary pass, for the same + // wording reason as UnmarshalFilters + if len(sorts) > 0 { + if err := validateQueryFragment("sorts", raw); err != nil { + return nil, err + } + } + + imp := &importer{opts: opts, doc: opts.fragmentDoc()} + dv := &model.BlockContentDataview{} + out := make([]*model.BlockContentDataviewSort, 0, len(sorts)) + for _, js := range sorts { + out = append(out, imp.sortFromJSON(js, dv)) + } + return out, nil +} + +// validateFilterVocabulary checks condition/datePreset/operator names +// recursively, with node paths. +func validateFilterVocabulary(nodes []jsonFilter, path string, addIssue func(string, string, ...any)) { + for i, node := range nodes { + nodePath := fmt.Sprintf("%s/%d", path, i) + if node.Operator != "" || len(node.Filters) > 0 { + if node.Operator != "" && node.Operator != "and" && node.Operator != "or" { + addIssue(nodePath+"/operator", "unknown operator %q — allowed: and, or", node.Operator) + } + validateFilterVocabulary(node.Filters, nodePath+"/filters", addIssue) + continue + } + if node.Property == "" { + addIssue(nodePath+"/property", "a filter leaf needs a property key") + } + if node.Condition != "" && !conditionNames.has(node.Condition) { + addIssue(nodePath+"/condition", "unknown condition %q — allowed: %s", node.Condition, filterConditionList) + } + if node.DatePreset != "" && !datePresetNames.has(node.DatePreset) { + addIssue(nodePath+"/datePreset", "unknown datePreset %q — allowed: %s", node.DatePreset, datePresetList) + } + } +} + +// referencedFormats resolves the §3 format name of every property key the +// filter tree references — the formats input of checkDateFilters. The term +// travels through the reader's vocabulary before the format is looked up, +// exactly as importer.filterFromJSON resolves it (propertyKey, then +// impDvFormat): a request naming the documented "Due date" spelling would +// otherwise resolve no format at all, and the format is what says whether a +// date preset means anything here. +func referencedFormats(nodes []jsonFilter, opts Options) map[string]string { + formats := map[string]string{} + var walk func(nodes []jsonFilter) + walk = func(nodes []jsonFilter) { + for _, node := range nodes { + if len(node.Filters) > 0 { + walk(node.Filters) + } + if node.Property == "" { + continue + } + if _, seen := formats[node.Property]; seen { + continue + } + if f, ok := resolveFormatWith(opts, opts.propertyKey(node.Property)); ok { + if name := FormatName(f); name != "" { + formats[node.Property] = name + } + } + } + } + walk(nodes) + return formats +} + +// validateQueryFragment holds a bare §6.2 filters or sorts array to the same +// schema fragments the whole-document path holds a view to ($defs/filterNode, +// $defs/sort): the array is wrapped into a minimal synthetic document — the +// validateFragmentRun pattern (fragment.go) — and the document validation +// runs, so the two doors cannot disagree about the shape. Issue paths are +// remapped from the synthetic /blocks/0/views/0/… to the fragment-relative +// /filters/… and /sorts/… the §6.2 fragment entry points promise. Warnings +// are discarded here: the fragment entry points run their own semantic pass +// with the caller's format resolver, which the synthetic document lacks. +func validateQueryFragment(member string, raw json.RawMessage) error { + payload, err := json.Marshal(map[string]any{ + "version": FormatVersion, + "type": "page", + "blocks": []any{map[string]any{ + "type": "dataview", + "views": []any{map[string]json.RawMessage{member: raw}}, + }}, + }) + if err != nil { + return fmt.Errorf("build synthetic %s document: %w", member, err) + } + if _, err := validateToDoc(payload, false, nil); err != nil { + var ve *ValidationError + if errors.As(err, &ve) { + return &ValidationError{Issues: refragmentIssues(ve.Issues)} + } + return err + } + return nil +} + +// refragmentIssues rewrites synthetic-document paths back to the +// fragment-relative form: /blocks/0/views/0/filters/1/condition names +// /filters/1/condition of the array the caller handed over. +func refragmentIssues(issues []Issue) []Issue { + const prefix = "/blocks/0/views/0" + out := make([]Issue, len(issues)) + for i, iss := range issues { + iss.Path = strings.TrimPrefix(iss.Path, prefix) + out[i] = iss + } + return out +} + +// isValidationError reports whether err wraps a *ValidationError — the +// path-addressed kind the fragment entry points prefer over a bare decode +// error. +func isValidationError(err error) bool { + var ve *ValidationError + return errors.As(err, &ve) +} diff --git a/pkg/lib/anyblockjson/filters_test.go b/pkg/lib/anyblockjson/filters_test.go new file mode 100644 index 0000000000..0eceed09f5 --- /dev/null +++ b/pkg/lib/anyblockjson/filters_test.go @@ -0,0 +1,360 @@ +package anyblockjson + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// fragOptionResolver resolves option names to ids from a fixed table — +// read-only, the query-path wiring. +type fragOptionResolver map[string]string + +func (r fragOptionResolver) OptionId(key domain.RelationKey, name string) (string, bool) { + id, ok := r[name] + return id, ok +} + +func (r fragOptionResolver) OptionName(key domain.RelationKey, id string) (string, bool) { + for name, oid := range r { + if oid == id { + return name, true + } + } + return "", false +} + +func fragFilterOpts() Options { + return Options{ + ResolveFormat: func(key domain.RelationKey) (model.RelationFormat, bool) { + switch string(key) { + case "status": + return model.RelationFormat_status, true + case "dueDate": + return model.RelationFormat_date, true + case "done": + return model.RelationFormat_checkbox, true + } + return 0, false + }, + ResolveOptions: fragOptionResolver{"In progress": "opt-inprogress", "Done": "opt-done"}, + } +} + +func TestUnmarshalFilters(t *testing.T) { + t.Run("bare leaves with option-name resolution and format rehydration", func(t *testing.T) { + // given + raw := json.RawMessage(`[ + {"property":"done","condition":"equal","value":false}, + {"property":"status","condition":"in","value":["In progress","Done"]}]`) + + // when + got, err := UnmarshalFilters(raw, fragFilterOpts()) + + // then + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "done", got[0].RelationKey) + assert.Equal(t, model.BlockContentDataviewFilter_Equal, got[0].Condition) + assert.Equal(t, model.RelationFormat_checkbox, got[0].Format) + assert.False(t, got[0].Value.GetBoolValue()) + assert.Equal(t, model.BlockContentDataviewFilter_In, got[1].Condition) + values := got[1].Value.GetListValue().Values + require.Len(t, values, 2) + assert.Equal(t, "opt-inprogress", values[0].GetStringValue(), "option names resolve to ids") + assert.Equal(t, "opt-done", values[1].GetStringValue()) + }) + + t.Run("or group with date preset", func(t *testing.T) { + raw := json.RawMessage(`[{"operator":"or","filters":[ + {"property":"dueDate","condition":"less","date_preset":"current_week"}, + {"property":"dueDate","condition":"empty"}]}]`) + + got, err := UnmarshalFilters(raw, fragFilterOpts()) + + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, model.BlockContentDataviewFilter_Or, got[0].Operator) + require.Len(t, got[0].NestedFilters, 2) + assert.Equal(t, model.BlockContentDataviewFilter_CurrentWeek, got[0].NestedFilters[0].QuickOption) + assert.Equal(t, model.BlockContentDataviewFilter_Empty, got[0].NestedFilters[1].Condition) + }) + + t.Run("unknown condition is a path-addressed error naming the vocabulary", func(t *testing.T) { + raw := json.RawMessage(`[{"property":"done","condition":"equals","value":false}]`) + + _, err := UnmarshalFilters(raw, fragFilterOpts()) + + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/filters/0/condition", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, `unknown condition "equals"`) + assert.Contains(t, ve.Issues[0].Message, "equal, not_equal, greater") + }) + + t.Run("unknown datePreset and operator error", func(t *testing.T) { + raw := json.RawMessage(`[{"operator":"xor","filters":[ + {"property":"dueDate","condition":"less","date_preset":"thisWeek"}]}]`) + + _, err := UnmarshalFilters(raw, fragFilterOpts()) + + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + require.Len(t, ve.Issues, 2) + assert.Equal(t, "/filters/0/operator", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, `unknown operator "xor"`) + assert.Equal(t, "/filters/0/filters/0/datePreset", ve.Issues[1].Path) + assert.Contains(t, ve.Issues[1].Message, `unknown datePreset "thisWeek"`) + }) + + t.Run("counting preset without a value errors (the document rule)", func(t *testing.T) { + raw := json.RawMessage(`[{"property":"dueDate","condition":"greater","date_preset":"number_of_days_ago"}]`) + + _, err := UnmarshalFilters(raw, fragFilterOpts()) + + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/filters/0", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, "needs a day count") + }) + + t.Run("unguarded date less warns on the OnWarning channel", func(t *testing.T) { + raw := json.RawMessage(`[{"property":"dueDate","condition":"less","date_preset":"today"}]`) + opts := fragFilterOpts() + var warnings []Issue + opts.OnWarning = func(i Issue) { warnings = append(warnings, i) } + + got, err := UnmarshalFilters(raw, opts) + + require.NoError(t, err) + require.Len(t, got, 1) + require.Len(t, warnings, 1) + assert.Equal(t, "/filters/0", warnings[0].Path) + assert.Contains(t, warnings[0].Message, "also matches objects with no dueDate") + assert.Contains(t, warnings[0].Message, "not_empty") + }) + + t.Run("guarded date less is clean", func(t *testing.T) { + raw := json.RawMessage(`[ + {"property":"dueDate","condition":"not_empty"}, + {"property":"dueDate","condition":"less","date_preset":"today"}]`) + opts := fragFilterOpts() + var warnings []Issue + opts.OnWarning = func(i Issue) { warnings = append(warnings, i) } + + _, err := UnmarshalFilters(raw, opts) + + require.NoError(t, err) + assert.Empty(t, warnings) + }) + + t.Run("placeholder on a non-object property warns", func(t *testing.T) { + // a WARNING here too, matching the document door: the same rule at + // two severities would let a stored filter validate on one surface + // and refuse on the other, and the stored pair is real data (the + // document door's I1 arm pins that side) + raw := json.RawMessage(`[{"property":"status","condition":"in","value":["_filter_template_2_"]}]`) + opts := fragFilterOpts() + var warnings []Issue + opts.OnWarning = func(i Issue) { warnings = append(warnings, i) } + + _, err := UnmarshalFilters(raw, opts) + + require.NoError(t, err) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "resolves to an object id") + }) + + t.Run("leaf without property errors", func(t *testing.T) { + raw := json.RawMessage(`[{"condition":"equal","value":1}]`) + + _, err := UnmarshalFilters(raw, fragFilterOpts()) + + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + assert.Equal(t, "/filters/0/property", ve.Issues[0].Path) + }) + + t.Run("filterstring output feeds straight in (the one-tree contract)", func(t *testing.T) { + // the string form's emitted array is the same shape this codec takes + raw := json.RawMessage(`[{"property":"done","condition":"equal","value":false},` + + `{"operator":"or","filters":[` + + `{"property":"dueDate","condition":"less","date_preset":"current_week"},` + + `{"property":"dueDate","condition":"empty"}]}]`) + + got, err := UnmarshalFilters(raw, fragFilterOpts()) + + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, model.BlockContentDataviewFilter_Or, got[1].Operator) + }) +} + +func TestUnmarshalSorts(t *testing.T) { + t.Run("direction, emptyPlacement and format rehydrate", func(t *testing.T) { + raw := json.RawMessage(`[{"property":"dueDate","direction":"desc","empty_placement":"end","include_time":true}]`) + + got, err := UnmarshalSorts(raw, fragFilterOpts()) + + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "dueDate", got[0].RelationKey) + assert.Equal(t, model.BlockContentDataviewSort_Desc, got[0].Type) + assert.Equal(t, model.BlockContentDataviewSort_End, got[0].EmptyPlacement) + assert.Equal(t, model.RelationFormat_date, got[0].Format) + assert.True(t, got[0].IncludeTime) + }) + + t.Run("default direction is asc", func(t *testing.T) { + got, err := UnmarshalSorts(json.RawMessage(`[{"property":"name"}]`), Options{}) + + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, model.BlockContentDataviewSort_Asc, got[0].Type) + }) + + t.Run("unknown direction errors with allowed values", func(t *testing.T) { + _, err := UnmarshalSorts(json.RawMessage(`[{"property":"name","direction":"descending"}]`), Options{}) + + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/sorts/0/direction", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, `unknown direction "descending" — allowed: asc, desc, custom`) + }) + + t.Run("missing property errors", func(t *testing.T) { + _, err := UnmarshalSorts(json.RawMessage(`[{"direction":"asc"}]`), Options{}) + + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + assert.Equal(t, "/sorts/0/property", ve.Issues[0].Path) + }) +} + +// The fragment door is the API v2 query surface, and it validates against the +// same $defs/filterNode and $defs/sort fragments the whole-document path holds +// a view's filters and sorts to (§12). Before it did, an unknown member was +// silently DECODED AND DROPPED — `jsonUnmarshal` is a plain json.Unmarshal — +// so a misspelled member ("directions", "datePresets") turned a stated query +// into a different, quieter one with no error, where the identical shape +// inside a whole document was a hard, path-addressed refusal. +func TestUnmarshalFilters_UnknownMemberIsRefused(t *testing.T) { + t.Run("a filter leaf carrying an unrecognized member is refused with the member named", func(t *testing.T) { + // given + raw := json.RawMessage(`[{"property":"done","condition":"equal","frobnicate":true}]`) + + // when + _, err := UnmarshalFilters(raw, fragFilterOpts()) + + // then + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + assert.Contains(t, issuePaths(t, err), "/filters/0/frobnicate", + "the refusal has to name the slot it is about (§12): %v", err) + assert.Contains(t, err.Error(), `"frobnicate"`) + }) + + t.Run("a sort carrying an unrecognized member is refused with the member named", func(t *testing.T) { + // given + raw := json.RawMessage(`[{"property":"name","frobnicate":true}]`) + + // when + _, err := UnmarshalSorts(raw, Options{}) + + // then + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/sorts/0/frobnicate", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, `"frobnicate"`) + }) + + t.Run("a payload that is not an array is a path-addressed refusal, not a bare decode error", func(t *testing.T) { + // given + raw := json.RawMessage(`{"property":"done"}`) + + // when + _, err := UnmarshalFilters(raw, fragFilterOpts()) + + // then + var ve *ValidationError + require.True(t, errors.As(err, &ve), "want *ValidationError, got: %v", err) + assert.Contains(t, issuePaths(t, err), "/filters") + }) + + t.Run("a wrong-typed known member is a path-addressed refusal, not a bare decode error", func(t *testing.T) { + // given + raw := json.RawMessage(`[{"property":"name","include_time":"yes"}]`) + + // when + _, err := UnmarshalSorts(raw, Options{}) + + // then + var ve *ValidationError + require.True(t, errors.As(err, &ve), "want *ValidationError, got: %v", err) + assert.Contains(t, issuePaths(t, err), "/sorts/0/include_time") + }) +} + +// The two doors must agree: the fragment refusal for an unknown member is the +// document refusal for the identical shape, issue for issue, with only the +// path prefix differing — the fragment validates through the same schema +// fragments, so agreement is by construction and this pins it. +func TestUnmarshalFilters_FragmentAgreesWithDocument(t *testing.T) { + cases := map[string]struct { + member string // "filters" or "sorts" + raw string + }{ + "filter leaf with unknown member": {"filters", `[{"property":"done","condition":"equal","frobnicate":true}]`}, + "filter group with unknown member": {"filters", `[{"operator":"and","filters":[{"property":"done","condition":"equal"}],"frobnicate":true}]`}, + "sort with unknown member": {"sorts", `[{"property":"name","frobnicate":true}]`}, + "valid filters": {"filters", `[{"property":"done","condition":"equal","value":false}]`}, + "valid sorts": {"sorts", `[{"property":"name","direction":"desc"}]`}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + // given — the same array once through the fragment door, once + // inside a whole document + doc := `{"version":2,"type":"page","blocks":[{"type":"dataview","views":[{"` + + tc.member + `":` + tc.raw + `}]}]}` + + // when + var fragErr error + if tc.member == "filters" { + _, fragErr = UnmarshalFilters(json.RawMessage(tc.raw), fragFilterOpts()) + } else { + _, fragErr = UnmarshalSorts(json.RawMessage(tc.raw), Options{}) + } + docErr := Validate([]byte(doc)) + + // then + if docErr == nil { + assert.NoError(t, fragErr, "document accepts what the fragment refuses") + return + } + require.Error(t, fragErr, "fragment accepts what the document refuses: %v", docErr) + var fragVe, docVe *ValidationError + require.True(t, errors.As(fragErr, &fragVe)) + require.True(t, errors.As(docErr, &docVe)) + want := make([]Issue, 0, len(docVe.Issues)) + for _, iss := range docVe.Issues { + iss.Path = strings.TrimPrefix(iss.Path, "/blocks/0/views/0") + want = append(want, iss) + } + assert.Equal(t, want, fragVe.Issues) + }) + } +} diff --git a/pkg/lib/anyblockjson/filterstring/filterstring.go b/pkg/lib/anyblockjson/filterstring/filterstring.go new file mode 100644 index 0000000000..2d3fd05b15 --- /dev/null +++ b/pkg/lib/anyblockjson/filterstring/filterstring.go @@ -0,0 +1,998 @@ +// Package filterstring parses the compact filter syntax of AnyBlock JSON +// (SPEC §6.2.1) into the §6.2 structured filter tree. The grammar and this +// parser ship as a library consumed by the API v2 request surface +// Phase 4 — POST search and the POST sets `filter` field); the *document* +// view field `filter` stays reserved post-v1, so nothing in the parent +// package reads or writes the string form. +// +// Parse returns the structured filters ARRAY as canonical JSON — exactly the +// §6.2 shape the structured `filters` request field carries — so both request +// forms land on one internal tree through the same downstream codec +// (anyblockjson.UnmarshalFilters). Every parse error is offset-addressed +// (*Error: byte offset + offending token) and, where a reference set is +// wired in via Options, carries a did-you-mean hint — the agent repair loop. +// +// Deliberately absent, per SPEC §6.2.1: free-standing NOT(…) (the internal +// model has no NOT-group), joins, subqueries, arbitrary functions. +package filterstring + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +const ( + // maxFilterLength bounds the input in bytes. It matches the maxLength the + // discovery schemas advertise for the `filter` field (4096); anything + // longer is rejected before lexing so no attacker-sized input reaches the + // recursive parser. + maxFilterLength = 4096 + // maxGroupDepth bounds parenthesis nesting — the recursion depth of the + // parser. It matches the document side's nesting bound (SPEC §4: [0,32]). + // Without it a paren-bomb input overflows the goroutine stack, which is a + // runtime FATAL (not a recoverable panic) and kills the whole process. + maxGroupDepth = 32 + // maxDayCount bounds the counting presets' operand (~100 years); larger + // values wrap around inside time.AddDate and produce meaningless ranges. + maxDayCount = 36500 + // maxTokenRunes bounds how much of an offending token an error echoes + // back — an unterminated string would otherwise mirror the whole rest of + // the input into the response (and the agent's context window). + maxTokenRunes = 32 +) + +// Options wires the space's reference sets into the parser. All fields are +// optional; a zero Options parses purely syntactically. +type Options struct { + // KnownKeys, when non-nil, is the closed reference set for property keys: + // a key outside it is an offset-addressed parse error with did-you-mean. + KnownKeys []string + // ResolveFormat, when non-nil, resolves a property key to its §3 format + // name ("date", "select", "multi_select", …). It drives the RFC 3339 → + // unix conversion for date properties (the §6.2.1 mapping: the string + // form uses RFC 3339, the structured form unix numbers) and targets the + // option-name validation at select-shaped keys. + ResolveFormat func(key string) (format string, ok bool) + // KnownOptions, when non-nil, lists the existing option names of a + // select/multi_select property (ok=false: unknown property — no check). + // A name outside the list is an offset-addressed parse error with + // did-you-mean: the query path resolves option names READ-ONLY, and a + // silent no-match would be worse than an error. + KnownOptions func(key string) (names []string, ok bool) +} + +// Error is an offset-addressed parse error: the byte offset into the input, +// the offending token (empty at end of input), and a message naming allowed +// values, plus an optional repair hint (did-you-mean). +type Error struct { + Offset int + Token string + Message string + Hint string +} + +func (e *Error) Error() string { + where := fmt.Sprintf("near %q", e.Token) + if e.Token == "" { + where = "at end of input" + } + msg := fmt.Sprintf("parse error at offset %d %s: %s", e.Offset, where, e.Message) + if e.Hint != "" { + msg += " — " + e.Hint + } + return msg +} + +// datePresets maps the preset function names (SPEC §6.2.1, the compact +// syntax's own camelCase function-name vocabulary — unrelated to the +// snake_case §3 wire vocabulary) to the structured form's date_preset names +// (§6.2, snake_case — anyblockjson's datePresetNames). The two counting +// presets take an operand. +var datePresets = map[string]string{ + "yesterday": "yesterday", + "today": "today", + "tomorrow": "tomorrow", + "lastWeek": "last_week", + "currentWeek": "current_week", + "nextWeek": "next_week", + "lastMonth": "last_month", + "currentMonth": "current_month", + "nextMonth": "next_month", + "lastYear": "last_year", + "currentYear": "current_year", + "nextYear": "next_year", + "daysAgo": "number_of_days_ago", + "daysFromNow": "number_of_days_now", +} + +// countingPresets are the preset functions that require a day-count operand. +var countingPresets = map[string]bool{"daysAgo": true, "daysFromNow": true} + +// presetFunctionList renders the function vocabulary for error messages, in +// SPEC order. +const presetFunctionList = "yesterday() · today() · tomorrow() · lastWeek() · currentWeek() · nextWeek() · lastMonth() · currentMonth() · nextMonth() · lastYear() · currentYear() · nextYear() · daysAgo(n) · daysFromNow(n)" + +// reservedWords are the keywords of the grammar; a property key cannot be +// one of them (matched case-insensitively). +var reservedWords = map[string]bool{ + "and": true, "or": true, "not": true, "is": true, "in": true, + "contains": true, "has": true, "all": true, "empty": true, + "exists": true, "true": true, "false": true, +} + +// +// ---- lexer ---- +// + +type tokenKind int + +const ( + tokEOF tokenKind = iota + tokIdent + tokString + tokNumber + tokOp // = != > < >= <= + tokLParen // ( + tokRParen // ) + tokComma // , +) + +type token struct { + kind tokenKind + text string // the raw token text (for strings: the decoded value) + raw string // the raw source text (for error reporting) + offset int // byte offset of the token's first character +} + +type lexer struct { + input string + pos int + toks []token +} + +func lex(input string) ([]token, *Error) { + lx := &lexer{input: input} + for { + lx.skipSpace() + if lx.pos >= len(lx.input) { + lx.toks = append(lx.toks, token{kind: tokEOF, offset: lx.pos}) + return lx.toks, nil + } + start := lx.pos + c := lx.input[lx.pos] + switch { + case c == '(': + lx.emit(tokLParen, "(", start) + case c == ')': + lx.emit(tokRParen, ")", start) + case c == ',': + lx.emit(tokComma, ",", start) + case c == '"': + if err := lx.lexString(start); err != nil { + return nil, err + } + case c == '=': + lx.emit(tokOp, "=", start) + case c == '!': + if lx.pos+1 < len(lx.input) && lx.input[lx.pos+1] == '=' { + lx.pos++ + lx.emit(tokOp, "!=", start) + } else { + return nil, &Error{Offset: start, Token: "!", Message: "unexpected character '!'; the negated conditions are !=, NOT CONTAINS, NOT IN, NOT HAS ALL, IS NOT EMPTY"} + } + case c == '>' || c == '<': + op := string(c) + if lx.pos+1 < len(lx.input) && lx.input[lx.pos+1] == '=' { + lx.pos++ + op += "=" + } + lx.emit(tokOp, op, start) + case c == '-' || (c >= '0' && c <= '9'): + lx.lexNumber(start) + case c == '\'' || c == '`': + return nil, &Error{Offset: start, Token: string(c), + Message: fmt.Sprintf("unexpected character %q", string(c)), + Hint: `string values use double quotes, e.g. severity = "High"`} + default: + // decode a full rune so multi-byte input is classified (and + // reported) as the character the caller wrote, never a stray byte + r, size := utf8.DecodeRuneInString(lx.input[lx.pos:]) + if isIdentStart(r) { + lx.lexIdent(start) + continue + } + return nil, &Error{Offset: start, Token: lx.input[start : start+size], Message: fmt.Sprintf("unexpected character %q", string(r))} + } + } +} + +func (lx *lexer) skipSpace() { + for lx.pos < len(lx.input) { + c := lx.input[lx.pos] + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + lx.pos++ + continue + } + return + } +} + +func (lx *lexer) emit(kind tokenKind, text string, start int) { + lx.pos = start + len(text) + lx.toks = append(lx.toks, token{kind: kind, text: text, raw: text, offset: start}) +} + +func (lx *lexer) lexString(start int) *Error { + var sb strings.Builder + i := start + 1 + for i < len(lx.input) { + c := lx.input[i] + switch c { + case '\\': + if i+1 >= len(lx.input) { + return &Error{Offset: start, Token: truncateToken(lx.input[start:]), Message: "unterminated string literal"} + } + next := lx.input[i+1] + switch next { + case '"', '\\': + sb.WriteByte(next) + case 'n': + sb.WriteByte('\n') + case 't': + sb.WriteByte('\t') + default: + return &Error{Offset: i, Token: lx.input[i : i+2], Message: fmt.Sprintf(`unknown escape \%c in string literal; allowed: \" \\ \n \t`, next)} + } + i += 2 + case '"': + lx.toks = append(lx.toks, token{kind: tokString, text: sb.String(), raw: lx.input[start : i+1], offset: start}) + lx.pos = i + 1 + return nil + default: + sb.WriteByte(c) + i++ + } + } + return &Error{Offset: start, Token: truncateToken(lx.input[start:]), Message: `unterminated string literal — close it with "`} +} + +// truncateToken caps an offending token at maxTokenRunes for error echoing — +// errors name the problem, they do not mirror the input back. +func truncateToken(s string) string { + runes := []rune(s) + if len(runes) <= maxTokenRunes { + return s + } + return string(runes[:maxTokenRunes]) + "…" +} + +func (lx *lexer) lexNumber(start int) { + i := start + if lx.input[i] == '-' { + i++ + } + for i < len(lx.input) && (lx.input[i] >= '0' && lx.input[i] <= '9' || lx.input[i] == '.') { + i++ + } + text := lx.input[start:i] + lx.toks = append(lx.toks, token{kind: tokNumber, text: text, raw: text, offset: start}) + lx.pos = i +} + +func isIdentStart(r rune) bool { + return r == '_' || unicode.IsLetter(r) +} + +// isIdentPart follows UAX #31's ID_Continue in the part that matters here: +// letters, digits, `_`, AND the combining marks (Mn, Mc) that carry the vowels +// of every Indic and South-East Asian script. Excluding marks is not a +// restriction on those scripts, it is a corruption of them — without Mn/Mc, +// मिल/मूल/मल/मैल (mil, mūl, mal, mail — four different words) all reduce to +// मल, while हिन्दी and हिंदी, two legal spellings of ONE word, reduce to two +// DIFFERENT tokens. NFC rescues Latin, Greek, Cyrillic and Vietnamese because +// precomposed forms exist for them; Devanagari, Thai, Bengali, Tamil, Khmer +// and Myanmar have none. +// +// identStart deliberately does NOT admit marks: a combining mark cannot begin +// an identifier because it has nothing to combine with, and UAX #31 agrees. +func isIdentPart(r rune) bool { + return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) || + unicode.Is(unicode.Mn, r) || unicode.Is(unicode.Mc, r) +} + +func (lx *lexer) lexIdent(start int) { + i := start + for i < len(lx.input) { + r, size := utf8.DecodeRuneInString(lx.input[i:]) + if !isIdentPart(r) { + break + } + i += size + } + text := lx.input[start:i] + lx.toks = append(lx.toks, token{kind: tokIdent, text: text, raw: text, offset: start}) + lx.pos = i +} + +// +// ---- parse tree ---- +// + +// node is the parsed tree before emission: either a group (op + children) or +// a leaf. +type node struct { + op string // "and" | "or"; "" = leaf + children []*node + + property string + condition string + value any // nil = no value (presence conditions, valueless presets) + hasValue bool + datePreset string +} + +// +// ---- parser ---- +// + +type parser struct { + toks []token + pos int + depth int // current parenthesis nesting (bounded by maxGroupDepth) + opts Options + // presetTok is the name token of the most recently parsed date-preset + // function — preset-misuse errors address it, not the closing ')' + presetTok token +} + +// Parse parses a compact filter string (SPEC §6.2.1) into the §6.2 +// structured filters array, returned as canonical compact JSON: top-level +// nodes combine with an implicit AND, groups exist only for OR and nesting. +// Errors are always *Error (offset-addressed). +func Parse(input string, opts Options) (json.RawMessage, error) { + if strings.TrimSpace(input) == "" { + return nil, &Error{Offset: 0, Message: "empty filter", Hint: `a filter is one or more conditions, e.g. done = false AND due_date < currentWeek()`} + } + if len(input) > maxFilterLength { + return nil, &Error{Offset: maxFilterLength, Token: "", + Message: fmt.Sprintf("filter string is %d bytes — the maximum is %d", len(input), maxFilterLength), + Hint: "split the query: narrow with type or run several searches"} + } + toks, lexErr := lex(input) + if lexErr != nil { + return nil, lexErr + } + p := &parser{toks: toks, opts: opts} + root, err := p.parseOr() + if err != nil { + return nil, err + } + if tok := p.peek(); tok.kind != tokEOF { + return nil, &Error{Offset: tok.offset, Token: tok.raw, + Message: "expected AND, OR or end of input", + Hint: "conditions combine with AND and OR; parentheses group"} + } + return emit(root), nil +} + +func (p *parser) peek() token { return p.toks[p.pos] } +func (p *parser) next() token { t := p.toks[p.pos]; p.pos++; return t } + +// keyword reports whether tok is the given keyword (case-insensitive). +func keyword(tok token, kw string) bool { + return tok.kind == tokIdent && strings.EqualFold(tok.text, kw) +} + +func (p *parser) parseOr() (*node, error) { + left, err := p.parseAnd() + if err != nil { + return nil, err + } + children := []*node{left} + for keyword(p.peek(), "or") { + p.next() + right, err := p.parseAnd() + if err != nil { + return nil, err + } + children = append(children, right) + } + if len(children) == 1 { + return left, nil + } + return &node{op: "or", children: flatten("or", children)}, nil +} + +func (p *parser) parseAnd() (*node, error) { + left, err := p.parsePrimary() + if err != nil { + return nil, err + } + children := []*node{left} + for keyword(p.peek(), "and") { + p.next() + right, err := p.parsePrimary() + if err != nil { + return nil, err + } + children = append(children, right) + } + if len(children) == 1 { + return left, nil + } + return &node{op: "and", children: flatten("and", children)}, nil +} + +// flatten merges same-operator children into their parent (canonical form: +// parentheses that do not change semantics leave no trace). +func flatten(op string, children []*node) []*node { + out := make([]*node, 0, len(children)) + for _, c := range children { + if c.op == op { + out = append(out, c.children...) + continue + } + out = append(out, c) + } + return out +} + +func (p *parser) parsePrimary() (*node, error) { + tok := p.peek() + if tok.kind == tokLParen { + p.next() + p.depth++ + if p.depth > maxGroupDepth { + return nil, &Error{Offset: tok.offset, Token: tok.raw, + Message: fmt.Sprintf("filter groups nest at most %d deep", maxGroupDepth), + Hint: "flatten the grouping — AND/OR chains do not need parentheses per condition"} + } + inner, err := p.parseOr() + p.depth-- + if err != nil { + return nil, err + } + if closing := p.next(); closing.kind != tokRParen { + return nil, &Error{Offset: closing.offset, Token: closing.raw, + Message: fmt.Sprintf("expected ) to close the group opened at offset %d", tok.offset)} + } + return inner, nil + } + return p.parseLeaf() +} + +func (p *parser) parseLeaf() (*node, error) { + tok := p.next() + if tok.kind != tokIdent { + return nil, &Error{Offset: tok.offset, Token: tok.raw, + Message: "expected a property key", + Hint: "a condition starts with a bare property key, e.g. status IN (\"Done\")"} + } + if reservedWords[strings.ToLower(tok.text)] { + return nil, &Error{Offset: tok.offset, Token: tok.raw, + Message: fmt.Sprintf("%q is a reserved word, not a property key", tok.text), + Hint: "a property key that collides with a keyword cannot be written here — express this condition with the structured filters array instead"} + } + key := tok.text + if err := p.checkKey(tok); err != nil { + return nil, err + } + + op := p.next() + switch { + case op.kind == tokOp: + return p.parseComparison(key, op) + case keyword(op, "contains"): + return p.parseValueLeaf(key, "contains") + case keyword(op, "in"): + return p.parseListLeaf(key, "in") + case keyword(op, "has"): + return p.parseHasAll(key, op, "all_in") + case keyword(op, "is"): + return p.parseIs(key) + case keyword(op, "exists"): + return &node{property: key, condition: "exists"}, nil + case keyword(op, "not"): + after := p.next() + switch { + case keyword(after, "contains"): + return p.parseValueLeaf(key, "not_contains") + case keyword(after, "in"): + return p.parseListLeaf(key, "not_in") + case keyword(after, "has"): + return p.parseHasAll(key, after, "not_all_in") + default: + return nil, &Error{Offset: after.offset, Token: after.raw, + Message: "expected CONTAINS, IN or HAS ALL after NOT", + Hint: "negations: != · NOT CONTAINS · NOT IN (…) · NOT HAS ALL (…) · IS NOT EMPTY"} + } + default: + return nil, &Error{Offset: op.offset, Token: op.raw, + Message: fmt.Sprintf("expected a condition after property %q", key), + Hint: "conditions: = != > < >= <= · CONTAINS · IN (…) · HAS ALL (…) · IS EMPTY · IS NOT EMPTY · EXISTS"} + } +} + +// parseComparison handles = != > < >= <=. An = / != followed by a +// parenthesized list is the set-literal form (exactIn / notExactIn). +func (p *parser) parseComparison(key string, op token) (*node, error) { + if p.peek().kind == tokLParen { + switch op.text { + case "=": + return p.parseListLeaf(key, "exact_in") + case "!=": + return p.parseListLeaf(key, "not_exact_in") + default: + return nil, &Error{Offset: p.peek().offset, Token: "(", + Message: fmt.Sprintf("a value list is only allowed after = or != (set literal), not after %s", op.text)} + } + } + condition := map[string]string{ + "=": "equal", "!=": "not_equal", + ">": "greater", "<": "less", ">=": "greater_or_equal", "<=": "less_or_equal", + }[op.text] + return p.parseValueLeaf(key, condition) +} + +// parseValueLeaf parses one value and builds the leaf, folding date-preset +// functions into datePreset. +func (p *parser) parseValueLeaf(key, condition string) (*node, error) { + value, preset, err := p.parseValue(key) + if err != nil { + return nil, err + } + n := &node{property: key, condition: condition, datePreset: preset} + if preset == "" || value != nil { + // counting presets keep their operand as value; plain presets are + // valueless (§6.2) + n.value, n.hasValue = value, true + } + // the engine transforms a preset into a day range only for these + // conditions (pkg/lib/database.transformDateFilter); anything else — + // including != — would silently drop the preset and answer a different + // question, so the parser rejects it up front + if preset != "" && condition != "equal" && + condition != "greater" && condition != "less" && + condition != "greater_or_equal" && condition != "less_or_equal" { + return nil, &Error{Offset: p.presetTok.offset, Token: p.presetTok.raw, + Message: fmt.Sprintf("a date preset cannot be used with %s", condition), + Hint: `presets work with = > < >= <=; negate by range instead, e.g. due_date < today() OR due_date > today()`} + } + return n, nil +} + +// parseListLeaf parses ( value, value, … ) and builds an in/notIn/allIn/ +// notAllIn/exactIn/notExactIn leaf. +func (p *parser) parseListLeaf(key, condition string) (*node, error) { + open := p.next() + if open.kind != tokLParen { + return nil, &Error{Offset: open.offset, Token: open.raw, + Message: fmt.Sprintf("expected ( to start the %s value list", condition)} + } + var values []any + for { + tok := p.peek() + if tok.kind == tokRParen && len(values) == 0 { + return nil, &Error{Offset: tok.offset, Token: tok.raw, + Message: "a value list needs at least one value"} + } + value, preset, err := p.parseValue(key) + if err != nil { + return nil, err + } + if preset != "" { + return nil, &Error{Offset: p.presetTok.offset, Token: p.presetTok.raw, + Message: "date presets cannot appear inside a value list"} + } + values = append(values, value) + sep := p.next() + if sep.kind == tokComma { + continue + } + if sep.kind == tokRParen { + return &node{property: key, condition: condition, value: values, hasValue: true}, nil + } + return nil, &Error{Offset: sep.offset, Token: sep.raw, + Message: "expected , or ) in the value list"} + } +} + +// parseHasAll parses HAS ALL ( … ) / NOT HAS ALL ( … ). +func (p *parser) parseHasAll(key string, hasTok token, condition string) (*node, error) { + all := p.next() + if !keyword(all, "all") { + return nil, &Error{Offset: all.offset, Token: all.raw, + Message: "expected ALL after HAS", + Hint: `the contains-all condition is HAS ALL ("a", "b")`} + } + return p.parseListLeaf(key, condition) +} + +// parseIs parses IS EMPTY / IS NOT EMPTY. +func (p *parser) parseIs(key string) (*node, error) { + tok := p.next() + if keyword(tok, "empty") { + return &node{property: key, condition: "empty"}, nil + } + if keyword(tok, "not") { + after := p.next() + if keyword(after, "empty") { + return &node{property: key, condition: "not_empty"}, nil + } + return nil, &Error{Offset: after.offset, Token: after.raw, + Message: "expected EMPTY after IS NOT"} + } + return nil, &Error{Offset: tok.offset, Token: tok.raw, + Message: "expected EMPTY or NOT EMPTY after IS"} +} + +// parseValue parses one value: string, number, true/false, or a date-preset +// function. It returns (value, presetName): a plain preset returns ("", +// preset) with a nil value; a counting preset returns its operand as value. +func (p *parser) parseValue(key string) (any, string, error) { + tok := p.next() + switch tok.kind { + case tokString: + return p.stringValue(key, tok) + case tokNumber: + f, err := strconv.ParseFloat(tok.text, 64) + if err != nil { + return nil, "", &Error{Offset: tok.offset, Token: tok.raw, Message: "invalid number"} + } + return f, "", nil + case tokIdent: + if strings.EqualFold(tok.text, "true") { + return true, "", nil + } + if strings.EqualFold(tok.text, "false") { + return false, "", nil + } + if p.peek().kind == tokLParen { + return p.parsePresetCall(key, tok) + } + return nil, "", &Error{Offset: tok.offset, Token: tok.raw, + Message: fmt.Sprintf("unexpected bare word %q in value position", tok.text), + Hint: `values are double-quoted strings, numbers, true/false, RFC 3339 dates in quotes, or date-preset functions like currentWeek()`} + default: + return nil, "", &Error{Offset: tok.offset, Token: tok.raw, + Message: "expected a value", + Hint: `values are double-quoted strings, numbers, true/false, RFC 3339 dates in quotes, or date-preset functions like currentWeek()`} + } +} + +// parsePresetCall parses name( [n] ) — the date-preset functions. +func (p *parser) parsePresetCall(key string, nameTok token) (any, string, error) { + p.presetTok = nameTok + preset, known := datePresets[nameTok.text] + if !known { + return nil, "", &Error{Offset: nameTok.offset, Token: nameTok.raw, + Message: fmt.Sprintf("unknown function %q", nameTok.text), + Hint: didYouMeanHint(nameTok.text, presetFunctionNames(), "the date-preset functions are "+presetFunctionList)} + } + if format, ok := p.resolveFormat(key); ok && format != "date" { + return nil, "", &Error{Offset: nameTok.offset, Token: nameTok.raw, + Message: fmt.Sprintf("%s() is a date preset, but property %q has format %q", nameTok.text, key, format)} + } + p.next() // consume ( + if countingPresets[nameTok.text] { + operand := p.next() + if operand.kind != tokNumber { + return nil, "", &Error{Offset: operand.offset, Token: operand.raw, + Message: fmt.Sprintf("%s takes a day count, e.g. %s(7)", nameTok.text, nameTok.text)} + } + count, err := strconv.ParseFloat(operand.text, 64) + if err != nil || count != float64(int64(count)) || count < 0 || count > maxDayCount { + return nil, "", &Error{Offset: operand.offset, Token: operand.raw, + Message: fmt.Sprintf("%s takes a whole day count between 0 and %d", nameTok.text, maxDayCount)} + } + if closing := p.next(); closing.kind != tokRParen { + return nil, "", &Error{Offset: closing.offset, Token: closing.raw, + Message: fmt.Sprintf("expected ) to close %s(", nameTok.text)} + } + return count, preset, nil + } + if closing := p.next(); closing.kind != tokRParen { + return nil, "", &Error{Offset: closing.offset, Token: closing.raw, + Message: fmt.Sprintf("%s takes no arguments", nameTok.text)} + } + return nil, preset, nil +} + +// stringValue applies the §6.2.1 value mapping to a quoted string: on a +// date-formatted property it must be an RFC 3339 date and converts to the +// structured form's unix number; on a select-shaped property it is an option +// NAME and is validated read-only against the space's options. +func (p *parser) stringValue(key string, tok token) (any, string, error) { + format, formatKnown := p.resolveFormat(key) + if formatKnown && format == "date" { + sec, ok := parseDate(tok.text) + if !ok { + return nil, "", &Error{Offset: tok.offset, Token: tok.raw, + Message: fmt.Sprintf("property %q is a date — %q is not an RFC 3339 date", key, tok.text), + Hint: `write dates as "2026-08-01" / "2026-08-01T15:00:00Z" or use a preset function like currentWeek()`} + } + return float64(sec), "", nil + } + if formatKnown && (format == "select" || format == "multi_select") && p.opts.KnownOptions != nil { + if names, ok := p.opts.KnownOptions(key); ok { + if !containsString(names, tok.text) { + return nil, "", &Error{Offset: tok.offset, Token: tok.raw, + Message: fmt.Sprintf("property %q has no option named %q — a query never creates options", key, tok.text), + Hint: didYouMeanHint(tok.text, names, fmt.Sprintf("list them with GET /v2/spaces/{spaceId}/properties/%s/options", key)), + } + } + } + } + return tok.text, "", nil +} + +func (p *parser) resolveFormat(key string) (string, bool) { + if p.opts.ResolveFormat == nil { + return "", false + } + return p.opts.ResolveFormat(key) +} + +// checkKey validates a property key against the reference set (when wired). +func (p *parser) checkKey(tok token) error { + if p.opts.KnownKeys == nil { + return nil + } + if containsString(p.opts.KnownKeys, tok.text) { + return nil + } + known := append([]string(nil), p.opts.KnownKeys...) + sort.Strings(known) + const maxListed = 15 + listed := known + suffix := "" + if len(listed) > maxListed { + suffix = fmt.Sprintf(", … (%d total)", len(known)) + listed = listed[:maxListed] + } + hint := didYouMeanHint(tok.text, known, "") + // a known key the compact syntax cannot spell (hyphen, space, keyword + // collision) needs explicit steering — repairing the string cannot work + for _, suggestion := range closest(tok.text, known, 3) { + if !bareWritable(suggestion) { + steer := fmt.Sprintf("property key %q cannot be written in the compact filter string — use the structured filters array", suggestion) + if hint == "" { + hint = steer + } else { + hint += " — " + steer + } + break + } + } + return &Error{Offset: tok.offset, Token: tok.raw, + Message: fmt.Sprintf("unknown property key %q — known property keys: %s%s", tok.text, strings.Join(listed, ", "), suffix), + Hint: hint, + } +} + +// IsBareKey reports whether a key can be written as a bare property key in a +// compact filter string: the `key = identifier` rule of the EBNF above, plus +// the reserved words the grammar keeps for itself. +// +// It is exported because the grammar is where the format's notion of an +// identifier LIVES, and the parent package needs to ask rather than restate +// it — a second copy of "letters, digits, `_`, not a keyword" is a copy that +// drifts, and the two packages already shipped one such drift +// (filterstring_agreement_test.go). +// +// It used to be a stronger statement than it is. When a document spelled a +// property with a normalized slug, that slug was minted THROUGH this +// predicate, so every spelling the format could write was one this grammar +// could parse. A spelling is a raw display name now, and names carry spaces: +// "Due date" is a perfectly good spelling and not a bare key. +// +// Nothing is unreachable that was reachable before, because resolution folds +// away case and separators — the bare `due_date` addresses "Due date", and +// `Дата_выполнения` addresses "Дата выполнения". What has no compact form is +// a name no identifier folds onto: "C++", "50% done", or a name that +// collides with a keyword. The parser refuses those by name and points at +// the structured filters array, which can carry any spelling; adding a +// quoted-key production to the grammar would be the alternative and is a +// grammar change, not a bug fix. +func IsBareKey(key string) bool { return bareWritable(key) } + +// bareWritable reports whether a property key can be written as a bare +// identifier in the compact syntax: identifier characters only and not a +// reserved word. +func bareWritable(key string) bool { + if key == "" || reservedWords[strings.ToLower(key)] { + return false + } + for i, r := range key { + if i == 0 && !isIdentStart(r) { + return false + } + if i > 0 && !isIdentPart(r) { + return false + } + } + return true +} + +// +// ---- emission ---- +// + +// emitNode is the §6.2 filter-node JSON shape, in canonical field order. +type emitNode struct { + Operator string `json:"operator,omitempty"` + Filters []emitNode `json:"filters,omitempty"` + Property string `json:"property,omitempty"` + Condition string `json:"condition,omitempty"` + Value *any `json:"value,omitempty"` + DatePreset string `json:"date_preset,omitempty"` +} + +func emitOne(n *node) emitNode { + if n.op != "" { + out := emitNode{Operator: n.op, Filters: make([]emitNode, 0, len(n.children))} + for _, c := range n.children { + out.Filters = append(out.Filters, emitOne(c)) + } + return out + } + out := emitNode{Property: n.property, Condition: n.condition, DatePreset: n.datePreset} + if n.hasValue { + v := n.value + out.Value = &v + } + return out +} + +// emit renders the root as the §6.2 top-level filters array: a top-level AND +// spreads into bare siblings (implicit AND); anything else is one node. +func emit(root *node) json.RawMessage { + var nodes []emitNode + if root.op == "and" { + for _, c := range root.children { + nodes = append(nodes, emitOne(c)) + } + } else { + nodes = []emitNode{emitOne(root)} + } + data, err := json.Marshal(nodes) + if err != nil { + // the emit structs are marshal-safe by construction + panic(fmt.Sprintf("filterstring: emit: %v", err)) + } + return data +} + +// +// ---- helpers ---- +// + +// ParseDate exposes the parser's date parsing: RFC 3339 (with offsets and +// fractional seconds) and date-only strings (UTC midnight) → unix seconds. +// Consumers validating the STRUCTURED filter form reuse it so both request +// forms agree on what a date string is. +func ParseDate(s string) (int64, bool) { return parseDate(s) } + +// parseDate mirrors the parent package's §3 date parsing: RFC 3339 (with +// offsets and fractional seconds) and date-only strings (UTC midnight). +func parseDate(s string) (int64, bool) { + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02"} { + if t, err := time.Parse(layout, s); err == nil { + return t.Unix(), true + } + } + return 0, false +} + +func containsString(list []string, s string) bool { + for _, e := range list { + if e == s { + return true + } + } + return false +} + +func presetFunctionNames() []string { + out := make([]string, 0, len(datePresets)) + for name := range datePresets { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// didYouMeanHint picks the closest known names; fallback (possibly empty) +// when nothing is close. +func didYouMeanHint(input string, known []string, fallback string) string { + suggestions := closest(input, known, 3) + if len(suggestions) == 0 { + return fallback + } + return "did you mean " + strings.Join(suggestions, ", ") + "?" +} + +// closest ranks known names by similarity to input: case-insensitive +// equality, then prefix, then containment, then edit distance ≤ 2. +// Deterministic (rank, then alphabetical). +func closest(input string, known []string, max int) []string { + in := strings.ToLower(input) + type scored struct { + name string + rank int + } + var out []scored + for _, k := range known { + lk := strings.ToLower(k) + switch { + case lk == in: + out = append(out, scored{k, 0}) + case strings.HasPrefix(lk, in) || strings.HasPrefix(in, lk): + out = append(out, scored{k, 1}) + case strings.Contains(lk, in) || strings.Contains(in, lk): + out = append(out, scored{k, 2}) + case editDistanceAtMost(lk, in, 2): + out = append(out, scored{k, 3}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].rank != out[j].rank { + return out[i].rank < out[j].rank + } + return out[i].name < out[j].name + }) + if len(out) > max { + out = out[:max] + } + names := make([]string, len(out)) + for i, sc := range out { + names[i] = sc.name + } + return names +} + +// editDistanceAtMost reports whether the Levenshtein distance between a and +// b is ≤ bound; long inputs never match (cost guard). +func editDistanceAtMost(a, b string, bound int) bool { + ra, rb := []rune(a), []rune(b) + if len(ra) > 64 || len(rb) > 64 { + return false + } + diff := len(ra) - len(rb) + if diff < 0 { + diff = -diff + } + if diff > bound { + return false + } + prev := make([]int, len(rb)+1) + curr := make([]int, len(rb)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(ra); i++ { + curr[0] = i + for j := 1; j <= len(rb); j++ { + cost := 1 + if ra[i-1] == rb[j-1] { + cost = 0 + } + m := prev[j] + 1 + if curr[j-1]+1 < m { + m = curr[j-1] + 1 + } + if prev[j-1]+cost < m { + m = prev[j-1] + cost + } + curr[j] = m + } + prev, curr = curr, prev + } + return prev[len(rb)] <= bound +} diff --git a/pkg/lib/anyblockjson/filterstring/filterstring_test.go b/pkg/lib/anyblockjson/filterstring/filterstring_test.go new file mode 100644 index 0000000000..0ac430cbe8 --- /dev/null +++ b/pkg/lib/anyblockjson/filterstring/filterstring_test.go @@ -0,0 +1,469 @@ +package filterstring + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// parse is the test harness: parse with opts and return the emitted JSON. +func parse(t *testing.T, input string, opts Options) string { + t.Helper() + out, err := Parse(input, opts) + require.NoError(t, err) + return string(out) +} + +// parseErr asserts the parse fails and returns the offset-addressed error. +func parseErr(t *testing.T, input string, opts Options) *Error { + t.Helper() + _, err := Parse(input, opts) + require.Error(t, err) + var pe *Error + require.True(t, errors.As(err, &pe), "every parse error is *Error, got %T", err) + return pe +} + +func TestParse_Grammar(t *testing.T) { + tests := []struct { + name string + input string + want string // the §6.2 structured filters array, compact JSON + }{ + { + name: "single equal leaf", + input: `done = false`, + want: `[{"property":"done","condition":"equal","value":false}]`, + }, + { + name: "not equal number", + input: `priority != 3`, + want: `[{"property":"priority","condition":"not_equal","value":3}]`, + }, + { + name: "all comparison operators", + input: `a > 1 AND b < 2 AND c >= 3 AND d <= 4`, + want: `[{"property":"a","condition":"greater","value":1},` + + `{"property":"b","condition":"less","value":2},` + + `{"property":"c","condition":"greater_or_equal","value":3},` + + `{"property":"d","condition":"less_or_equal","value":4}]`, + }, + { + name: "contains and not contains", + input: `name CONTAINS "report" AND name NOT CONTAINS "draft"`, + want: `[{"property":"name","condition":"contains","value":"report"},` + + `{"property":"name","condition":"not_contains","value":"draft"}]`, + }, + { + name: "in list", + input: `status IN ("In progress", "Blocked")`, + want: `[{"property":"status","condition":"in","value":["In progress","Blocked"]}]`, + }, + { + name: "not in list", + input: `status NOT IN ("Done")`, + want: `[{"property":"status","condition":"not_in","value":["Done"]}]`, + }, + { + name: "has all and not has all", + input: `tags HAS ALL ("urgent", "q3") OR tags NOT HAS ALL ("later")`, + want: `[{"operator":"or","filters":[` + + `{"property":"tags","condition":"all_in","value":["urgent","q3"]},` + + `{"property":"tags","condition":"not_all_in","value":["later"]}]}]`, + }, + { + name: "set literal is exactIn", + input: `tags = ("a", "b")`, + want: `[{"property":"tags","condition":"exact_in","value":["a","b"]}]`, + }, + { + name: "negated set literal is notExactIn", + input: `tags != ("a")`, + want: `[{"property":"tags","condition":"not_exact_in","value":["a"]}]`, + }, + { + name: "is empty and is not empty", + input: `assignee IS EMPTY OR assignee IS NOT EMPTY`, + want: `[{"operator":"or","filters":[` + + `{"property":"assignee","condition":"empty"},` + + `{"property":"assignee","condition":"not_empty"}]}]`, + }, + { + name: "exists", + input: `assignee EXISTS`, + want: `[{"property":"assignee","condition":"exists"}]`, + }, + { + name: "valueless date preset", + input: `dueDate < currentWeek()`, + want: `[{"property":"dueDate","condition":"less","date_preset":"current_week"}]`, + }, + { + name: "counting preset keeps its operand as value", + input: `lastModifiedDate > daysAgo(7)`, + want: `[{"property":"lastModifiedDate","condition":"greater","value":7,"date_preset":"number_of_days_ago"}]`, + }, + { + name: "daysFromNow maps to numberOfDaysNow", + input: `dueDate <= daysFromNow(0)`, + want: `[{"property":"dueDate","condition":"less_or_equal","value":0,"date_preset":"number_of_days_now"}]`, + }, + { + name: "keywords are case-insensitive", + input: `done = false and name contains "x" or assignee is empty`, + want: `[{"operator":"or","filters":[` + + `{"operator":"and","filters":[` + + `{"property":"done","condition":"equal","value":false},` + + `{"property":"name","condition":"contains","value":"x"}]},` + + `{"property":"assignee","condition":"empty"}]}]`, + }, + { + name: "string escapes", + input: `name = "say \"hi\"\n"`, + want: `[{"property":"name","condition":"equal","value":"say \"hi\"\n"}]`, + }, + { + name: "negative number", + input: `balance < -5`, + want: `[{"property":"balance","condition":"less","value":-5}]`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, parse(t, tt.input, Options{})) + }) + } +} + +func TestParse_Precedence(t *testing.T) { + t.Run("AND binds tighter than OR", func(t *testing.T) { + want := `[{"operator":"or","filters":[` + + `{"operator":"and","filters":[` + + `{"property":"a","condition":"equal","value":1},` + + `{"property":"b","condition":"equal","value":2}]},` + + `{"operator":"and","filters":[` + + `{"property":"c","condition":"equal","value":3},` + + `{"property":"d","condition":"equal","value":4}]}]}]` + assert.Equal(t, want, parse(t, `a = 1 AND b = 2 OR c = 3 AND d = 4`, Options{})) + }) + + t.Run("parentheses group OR under AND as bare top-level siblings", func(t *testing.T) { + // the worked SPEC example: top-level implicit AND, group only for OR + want := `[{"property":"done","condition":"equal","value":false},` + + `{"operator":"or","filters":[` + + `{"property":"dueDate","condition":"less","date_preset":"current_week"},` + + `{"property":"dueDate","condition":"empty"}]}]` + assert.Equal(t, want, parse(t, `done = false AND (dueDate < currentWeek() OR dueDate IS EMPTY)`, Options{})) + }) + + t.Run("redundant parentheses flatten (canonical form)", func(t *testing.T) { + want := `[{"property":"a","condition":"equal","value":1},` + + `{"property":"b","condition":"equal","value":2},` + + `{"property":"c","condition":"equal","value":3}]` + assert.Equal(t, want, parse(t, `(a = 1 AND b = 2) AND (c = 3)`, Options{})) + }) + + t.Run("nested OR in OR flattens", func(t *testing.T) { + want := `[{"operator":"or","filters":[` + + `{"property":"a","condition":"equal","value":1},` + + `{"property":"b","condition":"equal","value":2},` + + `{"property":"c","condition":"equal","value":3}]}]` + assert.Equal(t, want, parse(t, `a = 1 OR (b = 2 OR c = 3)`, Options{})) + }) + + t.Run("single leaf in parentheses", func(t *testing.T) { + assert.Equal(t, `[{"property":"a","condition":"equal","value":1}]`, + parse(t, `(a = 1)`, Options{})) + }) +} + +func TestParse_DateConversion(t *testing.T) { + dateFormat := Options{ResolveFormat: func(key string) (string, bool) { + if key == "dueDate" { + return "date", true + } + return "text", true + }} + + t.Run("RFC 3339 date-only converts to unix on a date property", func(t *testing.T) { + // 2026-08-01T00:00:00Z = 1785542400 + assert.Equal(t, `[{"property":"dueDate","condition":"less","value":1785542400}]`, + parse(t, `dueDate < "2026-08-01"`, dateFormat)) + }) + + t.Run("full RFC 3339 timestamp converts", func(t *testing.T) { + assert.Equal(t, `[{"property":"dueDate","condition":"greater","value":1785596400}]`, + parse(t, `dueDate > "2026-08-01T15:00:00Z"`, dateFormat)) + }) + + t.Run("a text property keeps date-looking strings verbatim", func(t *testing.T) { + assert.Equal(t, `[{"property":"name","condition":"equal","value":"2026-08-01"}]`, + parse(t, `name = "2026-08-01"`, dateFormat)) + }) + + t.Run("non-RFC-3339 string on a date property errors", func(t *testing.T) { + pe := parseErr(t, `dueDate < "next tuesday"`, dateFormat) + assert.Equal(t, 10, pe.Offset) + assert.Contains(t, pe.Message, `is not an RFC 3339 date`) + assert.Contains(t, pe.Hint, "currentWeek()") + }) + + t.Run("preset on a non-date property errors", func(t *testing.T) { + pe := parseErr(t, `name = currentWeek()`, dateFormat) + assert.Contains(t, pe.Message, `currentWeek() is a date preset, but property "name" has format "text"`) + }) + + t.Run("without a format resolver strings stay verbatim", func(t *testing.T) { + assert.Equal(t, `[{"property":"dueDate","condition":"less","value":"2026-08-01"}]`, + parse(t, `dueDate < "2026-08-01"`, Options{})) + }) +} + +func TestParse_ErrorPositions(t *testing.T) { + t.Run("unknown property key with did-you-mean", func(t *testing.T) { + pe := parseErr(t, `done = false AND dueDat < currentWeek()`, + Options{KnownKeys: []string{"done", "dueDate", "status", "name"}}) + assert.Equal(t, 17, pe.Offset) + assert.Equal(t, "dueDat", pe.Token) + assert.Contains(t, pe.Message, `unknown property key "dueDat"`) + assert.Contains(t, pe.Message, "known property keys: done, dueDate, name, status") + assert.Equal(t, "did you mean dueDate?", pe.Hint) + assert.Contains(t, pe.Error(), `parse error at offset 17 near "dueDat"`) + }) + + t.Run("unknown option name with did-you-mean", func(t *testing.T) { + opts := Options{ + ResolveFormat: func(key string) (string, bool) { return "select", true }, + KnownOptions: func(key string) ([]string, bool) { + return []string{"In progress", "Blocked", "Done"}, true + }, + } + pe := parseErr(t, `status IN ("In progres")`, opts) + assert.Equal(t, 11, pe.Offset) + assert.Equal(t, `"In progres"`, pe.Token) + assert.Contains(t, pe.Message, `property "status" has no option named "In progres"`) + assert.Contains(t, pe.Message, "a query never creates options") + assert.Equal(t, "did you mean In progress?", pe.Hint) + }) + + t.Run("unterminated string", func(t *testing.T) { + pe := parseErr(t, `name = "unclosed`, Options{}) + assert.Equal(t, 7, pe.Offset) + assert.Contains(t, pe.Message, "unterminated string literal") + }) + + t.Run("missing closing parenthesis", func(t *testing.T) { + pe := parseErr(t, `(a = 1 AND b = 2`, Options{}) + assert.Equal(t, 16, pe.Offset) + assert.Equal(t, "", pe.Token) // EOF + assert.Contains(t, pe.Message, "expected ) to close the group opened at offset 0") + assert.Contains(t, pe.Error(), "at end of input") + }) + + t.Run("missing condition after key", func(t *testing.T) { + pe := parseErr(t, `done`, Options{}) + assert.Contains(t, pe.Message, `expected a condition after property "done"`) + assert.Contains(t, pe.Hint, "IS EMPTY") + }) + + t.Run("trailing garbage after a complete filter", func(t *testing.T) { + pe := parseErr(t, `a = 1 b = 2`, Options{}) + assert.Equal(t, 6, pe.Offset) + assert.Equal(t, "b", pe.Token) + assert.Contains(t, pe.Message, "expected AND, OR or end of input") + }) + + t.Run("unknown function with did-you-mean", func(t *testing.T) { + pe := parseErr(t, `dueDate < currentWek()`, Options{}) + assert.Equal(t, 10, pe.Offset) + assert.Contains(t, pe.Message, `unknown function "currentWek"`) + assert.Equal(t, "did you mean currentWeek?", pe.Hint) + }) + + t.Run("counting preset without operand", func(t *testing.T) { + pe := parseErr(t, `dueDate > daysAgo()`, Options{}) + assert.Contains(t, pe.Message, "daysAgo takes a day count, e.g. daysAgo(7)") + }) + + t.Run("plain preset with an operand", func(t *testing.T) { + pe := parseErr(t, `dueDate < yesterday(5)`, Options{}) + assert.Contains(t, pe.Message, "yesterday takes no arguments") + }) + + t.Run("preset inside a value list", func(t *testing.T) { + pe := parseErr(t, `dueDate IN (today())`, Options{}) + assert.Contains(t, pe.Message, "date presets cannot appear inside a value list") + }) + + t.Run("reserved word as key", func(t *testing.T) { + pe := parseErr(t, `in = 1`, Options{}) + assert.Contains(t, pe.Message, `"in" is a reserved word, not a property key`) + }) + + t.Run("empty filter", func(t *testing.T) { + pe := parseErr(t, ` `, Options{}) + assert.Contains(t, pe.Message, "empty filter") + }) + + t.Run("empty value list", func(t *testing.T) { + pe := parseErr(t, `status IN ()`, Options{}) + assert.Contains(t, pe.Message, "a value list needs at least one value") + }) + + t.Run("bare word in value position", func(t *testing.T) { + pe := parseErr(t, `status = Done`, Options{}) + assert.Equal(t, 9, pe.Offset) + assert.Contains(t, pe.Message, `unexpected bare word "Done" in value position`) + assert.Contains(t, pe.Hint, "double-quoted strings") + }) + + t.Run("free-standing NOT is not in the grammar", func(t *testing.T) { + pe := parseErr(t, `NOT (done = true)`, Options{}) + assert.Contains(t, pe.Message, "reserved word") + }) + + t.Run("set literal after ordering operator", func(t *testing.T) { + pe := parseErr(t, `priority > (1, 2)`, Options{}) + assert.Contains(t, pe.Message, "a value list is only allowed after = or != (set literal), not after >") + }) + + t.Run("HAS without ALL", func(t *testing.T) { + pe := parseErr(t, `tags HAS ("a")`, Options{}) + assert.Contains(t, pe.Message, "expected ALL after HAS") + }) + + t.Run("NOT followed by nothing usable", func(t *testing.T) { + pe := parseErr(t, `tags NOT = 1`, Options{}) + assert.Contains(t, pe.Message, "expected CONTAINS, IN or HAS ALL after NOT") + assert.Contains(t, pe.Hint, "IS NOT EMPTY") + }) +} + +func TestParse_Bounds(t *testing.T) { + t.Run("a paren bomb is an ordinary parse error, never a crash", func(t *testing.T) { + // the historical failure mode: unbounded recursion → goroutine stack + // overflow → runtime FATAL that kills the whole process + pe := parseErr(t, strings.Repeat("(", 100_000), Options{}) + assert.Contains(t, pe.Message, "maximum is 4096") + }) + + t.Run("a balanced deep nest under the length cap hits the depth bound", func(t *testing.T) { + input := strings.Repeat("(", 1000) + "a = 1" + strings.Repeat(")", 1000) + pe := parseErr(t, input, Options{}) + assert.Contains(t, pe.Message, "filter groups nest at most 32 deep") + }) + + t.Run("nesting beyond 32 groups is rejected with an offset", func(t *testing.T) { + input := strings.Repeat("(", 40) + "a = 1" + strings.Repeat(")", 40) + pe := parseErr(t, input, Options{}) + assert.Equal(t, 32, pe.Offset, "the 33rd open paren is the offender") + assert.Contains(t, pe.Message, "filter groups nest at most 32 deep") + }) + + t.Run("nesting at exactly 32 groups parses", func(t *testing.T) { + input := strings.Repeat("(", 32) + "a = 1" + strings.Repeat(")", 32) + assert.Equal(t, `[{"property":"a","condition":"equal","value":1}]`, + parse(t, input, Options{})) + }) + + t.Run("input beyond 4096 bytes is rejected before lexing", func(t *testing.T) { + pe := parseErr(t, "name = \""+strings.Repeat("x", 5000)+"\"", Options{}) + assert.Contains(t, pe.Message, "the maximum is 4096") + }) + + t.Run("counting presets bound the day count", func(t *testing.T) { + pe := parseErr(t, `dueDate > daysAgo(999999999)`, Options{}) + assert.Contains(t, pe.Message, "daysAgo takes a whole day count between 0 and 36500") + }) + + t.Run("an unterminated string echoes at most 32 runes", func(t *testing.T) { + pe := parseErr(t, `a = "`+strings.Repeat("x", 1000), Options{}) + assert.Contains(t, pe.Message, "unterminated string literal") + assert.True(t, strings.HasSuffix(pe.Token, "…"), "long tokens are truncated, got %q", pe.Token) + assert.Less(t, len(pe.Error()), 200, "the error must not mirror the input back") + }) +} + +func TestParse_UnicodeKeys(t *testing.T) { + t.Run("non-ASCII property keys are identifiers", func(t *testing.T) { + assert.Equal(t, `[{"property":"café","condition":"equal","value":1}]`, + parse(t, `café = 1`, Options{})) + assert.Equal(t, `[{"property":"дата","condition":"empty"}]`, + parse(t, `дата IS EMPTY`, Options{})) + }) + + t.Run("an unknown non-ASCII key reports the full key, not a stray byte", func(t *testing.T) { + pe := parseErr(t, `café = 1`, Options{KnownKeys: []string{"status"}}) + assert.Equal(t, "café", pe.Token) + assert.Contains(t, pe.Message, `unknown property key "café"`) + }) + + t.Run("an unexpected rune is reported as the rune the caller wrote", func(t *testing.T) { + pe := parseErr(t, `a © 1`, Options{}) + assert.Equal(t, 2, pe.Offset) + assert.Equal(t, "©", pe.Token) + assert.Contains(t, pe.Message, `unexpected character "©"`) + }) +} + +func TestParse_Steering(t *testing.T) { + t.Run("single quotes get the double-quote hint", func(t *testing.T) { + pe := parseErr(t, `severity = 'High'`, Options{}) + assert.Equal(t, 11, pe.Offset) + assert.Contains(t, pe.Hint, `string values use double quotes`) + }) + + t.Run("a reserved-word key steers to the structured filters array", func(t *testing.T) { + pe := parseErr(t, `all = true`, Options{}) + assert.Contains(t, pe.Message, `"all" is a reserved word`) + assert.Contains(t, pe.Hint, "structured filters array") + }) + + t.Run("a known key the syntax cannot spell steers to the structured form", func(t *testing.T) { + pe := parseErr(t, `due IS EMPTY`, Options{KnownKeys: []string{"due-date"}}) + assert.Contains(t, pe.Hint, "did you mean due-date?") + assert.Contains(t, pe.Hint, `property key "due-date" cannot be written in the compact filter string`) + assert.Contains(t, pe.Hint, "structured filters array") + }) +} + +func TestParse_PresetConditions(t *testing.T) { + t.Run("a preset with != is rejected — the engine would drop it", func(t *testing.T) { + pe := parseErr(t, `dueDate != today()`, Options{}) + assert.Equal(t, 11, pe.Offset, "the error addresses the preset, not the closing paren") + assert.Equal(t, "today", pe.Token) + assert.Contains(t, pe.Message, "a date preset cannot be used with not_equal") + assert.Contains(t, pe.Hint, "negate by range") + }) + + t.Run("a preset with CONTAINS addresses the preset name", func(t *testing.T) { + pe := parseErr(t, `name CONTAINS today()`, Options{}) + assert.Equal(t, 14, pe.Offset) + assert.Equal(t, "today", pe.Token) + assert.Contains(t, pe.Message, "a date preset cannot be used with contains") + }) + + t.Run("a preset inside a value list addresses the preset name", func(t *testing.T) { + pe := parseErr(t, `dueDate IN ("x", today())`, Options{}) + assert.Equal(t, 17, pe.Offset) + assert.Equal(t, "today", pe.Token) + assert.Contains(t, pe.Message, "date presets cannot appear inside a value list") + }) +} + +func TestParse_EmitsValidJSON(t *testing.T) { + // every grammar example must emit a decodable §6.2 array + for _, example := range Examples { + t.Run(example, func(t *testing.T) { + out, err := Parse(example, Options{}) + require.NoError(t, err) + var nodes []map[string]any + require.NoError(t, json.Unmarshal(out, &nodes)) + require.NotEmpty(t, nodes) + }) + } +} diff --git a/pkg/lib/anyblockjson/filterstring/grammar.go b/pkg/lib/anyblockjson/filterstring/grammar.go new file mode 100644 index 0000000000..cd68048c28 --- /dev/null +++ b/pkg/lib/anyblockjson/filterstring/grammar.go @@ -0,0 +1,47 @@ +package filterstring + +// EBNF is the grammar this parser pins (SPEC §6.2.1 — the parser is the +// normative artifact; this text is what the API discovery surface serves, +// and the Phase-5 GBNF conversion consumes it). Keywords are matched +// case-insensitively; canonical rendering is uppercase. +const EBNF = `filter = orExpr ; +orExpr = andExpr , { "OR" , andExpr } ; +andExpr = primary , { "AND" , primary } ; +primary = "(" , orExpr , ")" | leaf ; +leaf = key , condition ; +condition = compare , value + | ( "=" | "!=" ) , valueList (* set literal: exact_in / not_exact_in *) + | [ "NOT" ] , "CONTAINS" , value + | [ "NOT" ] , "IN" , valueList + | [ "NOT" ] , "HAS" , "ALL" , valueList + | "IS" , [ "NOT" ] , "EMPTY" + | "EXISTS" ; +compare = "=" | "!=" | ">" | "<" | ">=" | "<=" ; +valueList = "(" , value , { "," , value } , ")" ; +value = string | number | "true" | "false" | preset ; +preset = presetName , "(" , ")" | countingName , "(" , number , ")" ; +presetName = "yesterday" | "today" | "tomorrow" | "lastWeek" | "currentWeek" + | "nextWeek" | "lastMonth" | "currentMonth" | "nextMonth" + | "lastYear" | "currentYear" | "nextYear" ; +countingName = "daysAgo" | "daysFromNow" ; +key = identifier ; (* a bare property key; not a keyword *) +identifier = identStart , { identPart } ; +identStart = letter | "_" ; (* letter = any Unicode letter *) +identPart = letter | digit | "_" | mark ; (* mark = Unicode Mn/Mc, UAX #31 ID_Continue: + an Indic or SE-Asian vowel is a combining + mark, and dropping it changes the word *) +number = [ "-" ] , digit , { digit } , [ "." , { digit } ] ; +string = '"' , { character } , '"' ; (* backslash escapes: \" \\ \n \t *) +(* every quoted keyword above matches case-insensitively: "AND" also + matches and/And; canonical rendering is uppercase *) +` + +// Examples are worked filter strings, served alongside the grammar (C12). +var Examples = []string{ + `done = false AND (due_date < currentWeek() OR due_date IS EMPTY)`, + `status IN ("In progress", "Blocked")`, + `tags HAS ALL ("urgent", "q3") AND assignee IS NOT EMPTY`, + `last_modified_date > daysAgo(7)`, + `type IN ("task", "bug") AND priority >= 3`, + `name CONTAINS "report" AND due_date < "2026-08-01"`, +} diff --git a/pkg/lib/anyblockjson/filterstring_agreement_test.go b/pkg/lib/anyblockjson/filterstring_agreement_test.go new file mode 100644 index 0000000000..589c3072d9 --- /dev/null +++ b/pkg/lib/anyblockjson/filterstring_agreement_test.go @@ -0,0 +1,69 @@ +package anyblockjson + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/filterstring" +) + +// The compact filter grammar's compiler lives in a SUBPACKAGE (§6.2.1, +// §13), so this package's vocabulary invariant — every identifier the format +// defines is snake_case (§1) — could not see the tokens it emits. It drifted: +// the compiler wrote "notEqual", "greaterOrEqual", "allIn", "notEmpty" and a +// `datePreset` member while the reader had already migrated to `not_equal`, +// `greater_or_equal`, `all_in`, `not_empty` and `date_preset`, so +// UnmarshalFilters rejected documents its own compiler produced. +// +// Nothing structural stops that happening again — the two vocabularies are +// declared in different packages, one as parser output and one as a reader +// table — so the guard has to be a test that crosses the boundary. It asserts +// the only thing that matters: whatever the compiler emits, this package +// accepts. +func TestFilterString_CompilerOutputIsAcceptedByThisPackage(t *testing.T) { + // one input per condition the grammar can emit, plus the date presets, + // so a token that drifts has nowhere to hide + // one input per condition the grammar can emit, plus the date presets, + // so a token that drifts has nowhere to hide. NOTE the asymmetry the + // migration deliberately kept: what a user TYPES stays camelCase + // (`daysAgo`, `currentWeek`) because it is the compact syntax's own + // vocabulary, served as an EBNF grammar; only the JSON it compiles TO is + // this format's snake_case (§1). + inputs := []string{ + `status = "Done"`, + `status != "Done"`, + `count > 3`, + `count < 3`, + `count >= 3`, + `count <= 3`, + `name CONTAINS "spec"`, + `name NOT CONTAINS "draft"`, + `status IN ("Done", "In progress")`, + `status NOT IN ("Done")`, + `tag HAS ALL ("a", "b")`, + `tag NOT HAS ALL ("a")`, + `name IS EMPTY`, + `name IS NOT EMPTY`, + `name EXISTS`, + `due_date = today()`, + `due_date > daysAgo(7)`, + `due_date < daysFromNow(3)`, + `due_date = currentWeek()`, + `due_date = lastMonth()`, + } + for _, in := range inputs { + t.Run(in, func(t *testing.T) { + raw, err := filterstring.Parse(in, filterstring.Options{}) + require.NoError(t, err, "the compiler must accept its own documented grammar (§6.2.1)") + require.NotEmpty(t, raw) + + got, err := UnmarshalFilters(raw, Options{}) + // the assertion that would have caught the drift: this package + // reads what that package writes + require.NoError(t, err, "this package must accept what its own filter compiler emits: %s", string(raw)) + assert.NotEmpty(t, got) + }) + } +} diff --git a/pkg/lib/anyblockjson/filtertemplate_test.go b/pkg/lib/anyblockjson/filtertemplate_test.go new file mode 100644 index 0000000000..30638e0e47 --- /dev/null +++ b/pkg/lib/anyblockjson/filtertemplate_test.go @@ -0,0 +1,128 @@ +package anyblockjson + +// Dynamic filter values (§6.2): the client substitutes these for a real +// object id before issuing the query (anytype-ts +// Dataview.valueTemplateMapper). They are stored verbatim, are opaque to the +// middleware, and are not object ids. + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func filterDoc(props, filters string) string { + return `{"version": 2, "id": "p1", "blocks": [{"type": "dataview", + "object_id": "someSet", "properties": [` + props + `], + "views": [{"name": "Mine", "filters": [` + filters + `]}]}]}` +} + +// the token is not an id: nothing in either direction rewrites it. The +// fixture used to carry a refs legend to prove the token could not be +// swallowed into one; there is no legend to be swallowed into now (§9a), so +// what is left to pin is that the token survives the object-valued path +// verbatim in both directions. +func TestRoundtrip_FilterTemplateSurvives(t *testing.T) { + for _, tok := range []string{"_filter_template_1_", "_filter_template_2_"} { + t.Run(tok, func(t *testing.T) { + doc := filterDoc(`{"property": "assignee", "format": "objects"}`, + `{"property": "assignee", "condition": "in", "value": ["`+tok+`"]}`) + + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + var got []string + for _, b := range snap.Blocks { + if dv := b.GetDataview(); dv != nil && len(dv.Views) > 0 { + for _, f := range dv.Views[0].Filters { + for _, v := range f.Value.GetListValue().GetValues() { + got = append(got, v.GetStringValue()) + } + } + } + } + assert.Equal(t, []string{tok}, got, "must reach the snapshot unrewritten") + + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(data), `"`+tok+`"`) + assert.NoError(t, Validate(data)) + }) + } +} + +// Resolving to an object id, a template token can only match an object/file +// property — and saying so is a WARNING, never a refusal: a stored dataview +// filter really carries this pair, so the rule as an error was an I1 break — +// export wrote the document with zero warnings and the package's own +// Validate refused it, making the object unexportable over one stored +// filter. The same tension was settled the same way for the date-preset +// rule beside it. +// +// How this can fail: turn the warnIssue back into addIssue and the Marshal +// arm below fails on its own output; drop the warning entirely and a filter +// that matches nothing ships with a clean bill of health. +func TestValidate_FilterTemplateOnWrongFormat(t *testing.T) { + for _, f := range []string{"select", "date", "text", "number"} { + t.Run(f, func(t *testing.T) { + doc := filterDoc( + `{"property": "stage", "format": "`+f+`"}`, + `{"property": "stage", "condition": "in", "value": ["_filter_template_2_"]}`) + var warns []Issue + require.NoError(t, ValidateWarn([]byte(doc), func(i Issue) { warns = append(warns, i) }), + "a stored filter must not make an object unexportable") + found := false + for _, w := range warns { + if strings.Contains(w.Message, "resolves to an object id") { + found = true + } + } + assert.True(t, found, "the mismatch is still named, as a warning") + }) + } + for _, f := range []string{"objects", "files"} { + t.Run(f+" is fine", func(t *testing.T) { + var warns []Issue + require.NoError(t, ValidateWarn([]byte(filterDoc( + `{"property": "assignee", "format": "`+f+`"}`, + `{"property": "assignee", "condition": "in", "value": ["_filter_template_2_"]}`)), + func(i Issue) { warns = append(warns, i) })) + for _, w := range warns { + assert.NotContains(t, w.Message, "resolves to an object id") + } + }) + } + + t.Run("the I1 arm: the stored pair exports, validates, and warns", func(t *testing.T) { + // the exact shape the invariant break was found on: a stored filter + // carrying the token on a property the same block declares as text + doc := filterDoc( + `{"property": "stage", "format": "text"}`, + `{"property": "stage", "condition": "in", "value": ["_filter_template_2_"]}`) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "the seam accepts what Validate accepts (I2)") + + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(data), `"_filter_template_2_"`) + require.NoError(t, Validate(data), + "Marshal never emits what its own Validate rejects (I1)") + }) +} + +func TestValidate_FilterTemplateNonTriggers(t *testing.T) { + t.Run("a real id on a select is not a token", func(t *testing.T) { + assert.NoError(t, Validate([]byte(filterDoc( + `{"property": "stage", "format": "select"}`, + `{"property": "stage", "condition": "in", "value": ["In progress"]}`)))) + }) + t.Run("undeclared format is not checked", func(t *testing.T) { + assert.NoError(t, Validate([]byte(filterDoc( + `{"property": "other", "format": "objects"}`, + `{"property": "notDeclared", "condition": "in", "value": ["_filter_template_2_"]}`)))) + }) +} diff --git a/pkg/lib/anyblockjson/flat_invariants_test.go b/pkg/lib/anyblockjson/flat_invariants_test.go new file mode 100644 index 0000000000..1b8049c2d9 --- /dev/null +++ b/pkg/lib/anyblockjson/flat_invariants_test.go @@ -0,0 +1,1394 @@ +package anyblockjson + +// Two invariants hold the format together, and the pre-freeze review found six +// violations of the first and four of the second — every one of them in a place +// the rule had never been applied, though it was written down. Instances get +// their own regression tests in prefreeze_review_test.go; these are the +// invariants themselves, so the next instance fails here without anyone having +// thought of it: +// +// I1. Marshal never emits a document its own Validate rejects (§11). +// I2. Validate and Unmarshal agree on every input (§12): if Validate accepts +// a document, Unmarshal must not fail to decode it, and vice versa. +// +// Both are driven by hostile inputs on purpose. A corpus generated from +// Marshal's own output cannot catch what Marshal gets wrong — it would agree +// with itself — and the goldens are exactly that corpus. So I1 runs over +// snapshots built from the id shapes real data and real generators produce +// (dots, slashes, non-ASCII, over-long, derived-cell-shaped, suffix-colliding), +// and I2 over hand-written documents. + +import ( + "encoding/json" + "fmt" + "math/rand" + "regexp" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// hostileIds are the id shapes that broke the id domain, plus the ones that +// make two surfaces compete: `a.b` and `dir/file` miss the schema's charset, +// `c-1` sanitizes onto `c_1`, `block_12345` suffixes onto `12345` under +// CompactIds, `r1-c1` is what a table derives for its own cell, `c_1-c1` is +// what one derives after a dashed row id has been sanitized, `dataview` is +// the id the importer pins, and the long one exceeds the 64-character bound. +var hostileIds = []string{ + "", "a.b", "a-b", "a_b", "dir/file", "блок", "c-1", "c_1", "c1", "r1", + "r1-c1", "r1-c2", "c_1-c1", "c_1-c_1", "r1-c_1", "12345", "block_12345", + "dataview", "-", "_", + strings.Repeat("x", 70), "R1-C1", "a b", "id\n2", "obj1", +} + +// sharedCellMarker is the text of the corpus's two-parent cell — the block +// that is both a table cell and a child of a plain block standing before the +// table. Nothing else in a hostile document spells it, so counting it in the +// output counts emissions of that one block. +const sharedCellMarker = "sharedcell" + +// hostileSnapshot builds a deterministic snapshot for seed n: a root, a handful +// of text blocks, and optionally a table and a dataview, with every id drawn +// from hostileIds — including duplicates, which the snapshot graph is allowed +// to contain because it is untrusted (§11). +func hostileSnapshot(n int) (model.SmartBlockType, *model.SmartBlockSnapshotBase) { + rnd := rand.New(rand.NewSource(int64(n))) + pick := func() string { return hostileIds[rnd.Intn(len(hostileIds))] } + + root := &model.Block{Id: "obj1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + blocks := []*model.Block{root} + add := func(b *model.Block) { + root.ChildrenIds = append(root.ChildrenIds, b.Id) + blocks = append(blocks, b) + } + + for i := 0; i < 1+rnd.Intn(4); i++ { + add(&model.Block{Id: pick(), Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: fmt.Sprintf("text %d x", i)}}}) + } + if rnd.Intn(2) == 0 { + colIds := []string{pick(), pick()} + rowIds := []string{pick(), pick()} + table := &model.Block{Id: pick(), + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}} + cols := &model.Block{Id: "cols" + fmt.Sprint(n), + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_TableColumns}}} + rows := &model.Block{Id: "rows" + fmt.Sprint(n), + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_TableRows}}} + table.ChildrenIds = []string{cols.Id, rows.Id} + // a block with TWO parents: the table's first cell, and a plain block + // standing before the table. The emit reaches it through the holder + // first, which is the arrival order the cell shorthand never handled — + // it set the emit-once mark without reading it, so the cell wrote the + // block a second time and one stored block imported back as two. Added + // before the table so the holder is walked first, and consuming no + // randomness so the corpus above is unchanged. + add(&model.Block{Id: "holder" + fmt.Sprint(n), + ChildrenIds: []string{rowIds[0] + "-" + colIds[0]}, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "holder"}}}) + add(table) + blocks = append(blocks, cols, rows) + for _, id := range colIds { + cols.ChildrenIds = append(cols.ChildrenIds, id) + blocks = append(blocks, &model.Block{Id: id, + Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}) + } + var unwritten []string + for _, rowId := range rowIds { + rows.ChildrenIds = append(rows.ChildrenIds, rowId) + row := &model.Block{Id: rowId, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}} + blocks = append(blocks, row) + for _, colId := range colIds { + cellId := rowId + "-" + colId + // a grid cell nobody has filled: no block exists for it, but + // the id is the table's all the same (§6.1) — the editor + // materializes the cell at exactly that id the first time it + // is filled, and validation claims the whole grid + if rnd.Intn(3) == 0 { + unwritten = append(unwritten, cellId) + continue + } + text := "cell " + cellId + if cellId == rowIds[0]+"-"+colIds[0] { + // the two-parent cell above, marked so the emit can be + // counted: whichever parent reaches it first, the phrase + // may appear in the document at most once (§11) + text = sharedCellMarker + } + row.ChildrenIds = append(row.ChildrenIds, cellId) + blocks = append(blocks, &model.Block{Id: cellId, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: text}}}) + } + } + // a plain block sitting on a derived cell id. While every cell was + // materialized the id was reserved by the cell block itself, so this + // slot — the one collision the emit side never made — had no coverage. + if len(unwritten) > 0 { + add(&model.Block{Id: unwritten[rnd.Intn(len(unwritten))], + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "sits on a cell id"}}}) + } + } + if rnd.Intn(3) == 0 { + // a date filter carrying a preset but no value: the value-bearing + // conditions and the presence-only ones make export write different + // things, and only one of those had coverage + cond := []model.BlockContentDataviewFilterCondition{ + model.BlockContentDataviewFilter_Empty, + model.BlockContentDataviewFilter_NotEmpty, + model.BlockContentDataviewFilter_Exists, + model.BlockContentDataviewFilter_Greater, + model.BlockContentDataviewFilter_Equal, + model.BlockContentDataviewFilter_NotEqual, + }[rnd.Intn(6)] + preset := []model.BlockContentDataviewFilterQuickOption{ + model.BlockContentDataviewFilter_NumberOfDaysAgo, + model.BlockContentDataviewFilter_NumberOfDaysNow, + model.BlockContentDataviewFilter_Today, + }[rnd.Intn(3)] + add(&model.Block{Id: pick(), Content: &model.BlockContentOfDataview{ + Dataview: &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{{Id: pick(), Name: "All", + Filters: []*model.BlockContentDataviewFilter{{ + RelationKey: "dueDate", Condition: cond, QuickOption: preset, + Format: model.RelationFormat_date, + }, { + // a stored relation key that cannot be a property + // SPELLING, named by a BLOCK slot. /properties drops + // such a key before anything slugs it, and so does the + // census, so the stored-key half of writableSlug's + // guard — the one that keeps an unwritable key out of a + // legend VALUE — was reachable from block slots alone, + // and the corpus named none. Appended after the picks + // so it consumes no randomness and the corpus above is + // unchanged. + RelationKey: "a\nb", Condition: model.BlockContentDataviewFilter_Equal, + Value: str("x"), + }, { + // a counting preset whose stored operand is not a day + // count. The engine reads it as 0 (domain.Value.Int64 + // answers 0 for every non-number kind) and the format + // admits only the count, so export has one honest + // rendering and the junk may not travel: written + // verbatim it is a document Validate refuses — I1. + // Appended after the picks so it consumes no + // randomness and the corpus above is unchanged. + RelationKey: "dueDate", Condition: model.BlockContentDataviewFilter_Greater, + QuickOption: model.BlockContentDataviewFilter_NumberOfDaysAgo, + Value: str("a week"), + }, { + // a filter naming NO property. Real data holds these + // (a relation deleted out from under a view), and the + // format now says a filter has to name the property it + // filters on (§6) — so an export that wrote the + // nameless node emitted a document its own Validate + // rejects. The sort and the column beside it have + // dropped their nameless form all along; this is the + // slot that did not. Appended after the picks so it + // consumes no randomness and the corpus above is + // unchanged. + RelationKey: "", Condition: model.BlockContentDataviewFilter_Equal, + Value: str("x"), + }}}}, + }}}) + } + // a property BLOCK naming no property — the second slot that used to be + // emitted nameless, and the second half of the same rule. Its id consumes + // no randomness, so the corpus above is unchanged. + add(&model.Block{Id: "nokey" + fmt.Sprint(n), + Content: &model.BlockContentOfRelation{Relation: &model.BlockContentRelation{}}}) + // details carry the keys the import surface must refuse: if export ever + // emitted one, Marshal's output would fail its own Validate, which is how + // this invariant proves the two surfaces are still each other's mirror. + // The two custom keys at the end exist to give a vocabulary something to + // spell: the hostileVocab variant maps them onto the spelling shapes a real + // space can mint (over-long, empty, shadowing a bundled spelling). + details := map[string]*types.Value{ + "id": str("obj1"), + "name": str("hostile"), + "spaceId": str("bafyspace"), + "uniqueKey": str("ot-page"), + "oldAnytypeID": str("legacy1"), + "sourceFilePath": str("/tmp/x.md"), + "restrictions": {Kind: &types.Value_NumberValue{NumberValue: 3}}, + "isArchived": {Kind: &types.Value_BoolValue{BoolValue: true}}, + "": str("empty key"), + "a\nb": str("newline key"), + "dueDate": str("next Friday"), + // the §15 #12 trim, both sides of it. `isArchived` above is TRUE, so + // the non-empty control is already here; these are the empty ones + // export omits — and `relationFormat`/`featuredRelations`, which the + // whitelist deliberately does not admit and which must therefore + // survive. I1 asks whether omitting them is a FIXPOINT: the second + // generation must not differ by a key the first one dropped. + "isHidden": {Kind: &types.Value_BoolValue{BoolValue: false}}, + "relationReadonlyValue": {Kind: &types.Value_BoolValue{BoolValue: false}}, + "revision": {Kind: &types.Value_NumberValue{NumberValue: 0}}, + "relationMaxCount": {Kind: &types.Value_NumberValue{NumberValue: 0}}, + "relationDefaultValue": str(""), + "relationFormat": {Kind: &types.Value_NumberValue{NumberValue: 0}}, + "featuredRelations": strList(), + "6a32d4856761631534b22f85": str("space-slugged"), + "artist": str("verbatim custom key"), + // the two shadow shapes of the verbatim-first family (§3): a custom + // stored key shaped like an INTERNAL bundled key's legacy slug, and + // one shaped like a WRITABLE bundled key's legacy slug beside that + // bundled key itself ("dueDate" above). Both are snake-shaped stored keys the + // details carried none of, which is exactly how "export writes it + // verbatim, the reader resolves it elsewhere" stayed invisible: the + // first made seed 0 fail I1's Validate leg outright, the second made + // every seed a valid-but-unimportable archive until the identity + // entry existed. + "unique_key": str("custom, not the resolution vector"), + "due_date": str("custom, beside dueDate"), + // a name-over-number key holding a stored STRING, both halves (§3). + // A string the vocabulary does not name has no written form: written + // verbatim it is a document Validate refuses (unknown layout), so + // every non-type seed failed I1 here until export learned to drop + // it. A string that IS a name survives — and reads back as the + // number, which the fixpoint leg checks. On TYPE seeds the same two + // values exercise the §2a lift's own string handling instead + // (typeSettingEnumValue) and the provenance drop. + "layout": str("garbage-name"), + "recommendedLayout": str("todo"), + // the third shadow shape, and the only one the bundled table cannot + // see: a stored key whose spelling the VOCABULARY IN FORCE binds to a + // different key (shadowVocab below spells the BSON key as it). Real + // spaces mint it by deleting a property — a UI-deleted entity vacates + // its stored key, so the key stops being reserved while + // objects still carry it, and the freed spelling is another + // property's. Both keys are written verbatim here, and + // without the identity entry the reader binds one of them twice. + "initiative": str("custom, whose spelling this space now gives away"), + // OBJECT REFERENCES, including the shapes that carry the very + // separator the informative suffix uses (§9). A snapshot is + // untrusted (§11) and the split is unconditional, so I1 asks whether + // a document Marshal writes from these is one its own Validate + // accepts and its own Unmarshal reads — and the refNames variant + // below asks it with the suffix and the participant fold both armed. + // `#name` is what a writer produces copying only the readable half + // of `id#name`; `obj1` is named by a resolver whose answer + // normalizes to nothing, which is the bare-id-never-a-dangling-# + // path; the composite is this space's own, so the fold fires on it. + // + // An id with a `#` INSIDE it — `a#b` — is deliberately absent: it is + // the format's one reference normalization (§11 N(S)), so it is not + // a fixpoint and belongs in the test that names it, + // TestRefs_AHashInsideAnIdIsNormalizedOnce, rather than here where + // every value must survive untouched. + "assignee": strList("#name", "#", "obj1", + domain.NewParticipantId(hostileSpaceId, hostileIdentity)), + // a SELECT property, whose values are spelled by name with the option + // id carried in `option_ids` (§3, §9a). The pool holds exactly the + // shapes that made a legend key hard under the deleted flat spelling + // — a name carrying the old separator, a name carrying a space (both + // outside the plain label charset), two options sharing one name, a + // name past the bound a joined key could carry, and an id no resolver + // knows — so I1 asks whether a document Marshal writes with those + // keys is one its own Validate accepts and its own Unmarshal reads. + "tag": strList(hostileOptionValues...), + } + // the envelope key is a STORED identity key written verbatim (§2), and a + // closed charset over it was falsified by a 36 808-object sweep: relation + // options carry their option *name* in the key, spaces and all. I1 never + // covered this slot, which is exactly why the bad rule shipped. + storedKeys := []string{ + "", "page", "task", "completion_status_Not Started", + "69bbfc78877a91b1d12d1a7c_C/C++", "69a56205ccba0a47d8d8eb71_тогглы", + "69bbfc78877a91b1d12d1a84_$addToSet", "opt-" + strings.Repeat("x", 80), + } + snap := &model.SmartBlockSnapshotBase{ + Blocks: blocks, + Details: fields(details), + Key: storedKeys[rnd.Intn(len(storedKeys))], + } + // the TYPE key slots (§3): the envelope `type`/`template_for` pair and — + // on type-document seeds — `type_properties[].object_types`. Drawn after + // every other pick so adding them did not reshuffle the corpus above. + snap.ObjectTypes = hostileTypePools[rnd.Intn(len(hostileTypePools))] + sbType := model.SmartBlockType_Page + // one draw, three arms — a switch rather than a second Intn so the corpus + // above is unchanged by the template arm's arrival. + switch rnd.Intn(4) { + case 0: + // a type document: the recommended lists resolve through + // hostileTypePropResolver, whose definitions carry the object_types + // shapes (a custom key the vocabulary spells `task`, an unwritable + // spelling, the `template` spelling) + sbType = model.SmartBlockType_STType + snap.Details.Fields["recommendedFeaturedRelations"] = strList("hp1") + snap.Details.Fields["recommendedRelations"] = strList("hp2") + snap.Details.Fields["recommendedHiddenRelations"] = strList("hp3") + case 1: + // a TEMPLATE, which is the only kind of document with a second type + // slot (§2). Without this arm the corpus never produced one, so the + // two-slot half of invertedTypes — and of export's own + // modelledTypeKeys — was unreachable from the sweep: `template_for` + // could have stopped being written at all and every seed stayed + // green. It also puts the type pools whose first key is NOT + // `template` behind a template, which is the shape v0.22 stopped + // losing. + sbType = model.SmartBlockType_Template + } + return sbType, snap +} + +// The space and member the hostile corpus's participant reference names: a +// real space id shape (`.`) and a checksummed account identity, +// so the fold's classifier and its round-trip recheck both engage. +const ( + hostileSpaceId = "bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq.30afw2fe3tvff" + hostileIdentity = "AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA" +) + +// hostileObjectNames names EVERY id, including the ones no caption can +// survive — an empty answer, a name that normalizes to nothing, and the +// hostile reference shapes above. A resolver this eager is the adversarial +// case: it is the export side's own guards, not the resolver's restraint, +// that have to keep the document readable back. +type hostileObjectNames struct{} + +func (hostileObjectNames) ObjectName(id string) (string, bool) { + switch id { + case "": + return "", true // an empty name is no name at the seam + case "obj1": + return "🎉", true // normalizes to nothing: a bare id, never a dangling # + } + return "Имя / Name #1 " + id, true +} + +// hostileOptions is the option pool the hostile corpus's select property +// resolves against, scanned first-match exactly as storeresolver does. Two +// entries deliberately share a name: within one value that is the collapse +// §11 documents, and the corpus asks that the collapse be a FIXPOINT rather +// than a coin flip on the pool's order. +var hostileOptions = spaceOptions{"tag": { + {id: "opt-hash", name: "C#"}, + {id: "opt-space", name: "import issue"}, + {id: "opt-dup1", name: "books"}, + {id: "opt-dup2", name: "books"}, + {id: "opt-long", name: strings.Repeat("n", maxPropertyKeyLen+1)}, +}} + +// hostileOptionValues is what the corpus's object carries: every option in +// the pool, both same-named ones, plus an id nothing resolves — which export +// writes verbatim and owes no entry for. +var hostileOptionValues = []string{ + "opt-hash", "opt-space", "opt-dup1", "opt-dup2", "opt-long", "opt-unknown", +} + +// hostileTypePools are the ObjectTypes shapes that make the type namespace +// compete with its readers: a custom key a hostile vocabulary spells as the +// bundled `task` key, a stored key shaped like the bundled objectType's +// legacy slug (the §3 shadow shape — it owes an identity entry), template +// pairs whose second entry only survives if the `template` spelling stays +// put, keys whose vocabulary spelling is unwritable or reserved, a +// prefix-less entry, and an +// entry with no key at all, and — the collateral-damage shapes — a keyless +// entry STANDING BESIDE a good one. Stored `ot-` has no spelling, so a +// positional write emitted no `type`, which made `template_for` inexpressible +// too and took the good sibling down with it; a store that ran an older build +// holds exactly these. +// overLongTypeKey is a stored type key past the 128-character bound the +// schema puts on a LEGEND value (§3). The primary slots are unbounded, so it +// round-trips verbatim through `type` — but no legend line may name it, and a +// vocabulary that spells it as something writable is what tries to. +var overLongTypeKey = strings.Repeat("y", 140) + +// overLongPropertyKey is its property-namespace twin: a stored relation key +// no key slot can carry, member name or value (§3). +var overLongPropertyKey = strings.Repeat("k", maxPropertyKeyLen+12) + +var hostileTypePools = [][]string{ + nil, + {"ot-page"}, + {"ot-69bbfc78877a91b1d12d1a7c"}, + {"ot-object_type"}, + {"ot-template", "ot-69bbfc78877a91b1d12d1a7c"}, + {"ot-template", "ot-task"}, + {"ot-" + overLongTypeKey}, + {"ot-longslugged"}, + {"ot-blankslugged"}, + {"ot-squatter"}, + {"page"}, + {"ot-"}, + {"ot-", "ot-task"}, + {"", "ot-69bbfc78877a91b1d12d1a7c"}, + {"ot-template", "ot-"}, + {"ot-template", "ot-", "ot-task"}, + {"ot-", "ot-template", "ot-task"}, + // a NON-template with a second type, and one with a third: the format + // models neither position, so both are truncated away — and the entries + // the census reserved for them are keys no slot ends up spelling. The + // pools above are all templates or single types, so the truncating + // branch had no shape at all. The second entry here is deliberately the + // FIRST one's spelling under hostileVocab, which is what makes the + // reservation observable. + {"ot-69bbfc78877a91b1d12d1a7c", "ot-task"}, + {"ot-page", "ot-task", "ot-squatter"}, + // every entry keyless: the whole type list is lost, and the document + // must still be a document. `{"ot-"}` covers the single-entry case; this + // is the one where closing ranks has nothing to close onto. + {"ot-", ""}, +} + +// hostileTypePropResolver serves the two property definitions the type-seed +// recommended lists name. PropertyId answers false so import-side wiring is +// exercised without it. +type hostileTypePropResolver struct{} + +func (hostileTypePropResolver) PropertyById(id string) (PropertyDefinition, bool) { + switch id { + case "hp1": + return PropertyDefinition{Key: "owner", Name: "Owner", Format: model.RelationFormat_object, + ObjectTypes: []string{"69bbfc78877a91b1d12d1a7c", "task", "squatter"}}, true + case "hp2": + return PropertyDefinition{Key: "genre", Format: model.RelationFormat_object, + ObjectTypes: []string{"longslugged", overLongTypeKey}}, true + case "hp3": + // a stored property key past the legend bound. The §2a `property` slot is + // a JSON value, so the schema bounded only its minLength for a while + // and export wrote such a key straight through — a document the seam + // then refused, which is I1. + return PropertyDefinition{Key: domain.RelationKey(overLongPropertyKey), Name: "Ledger"}, true + } + return PropertyDefinition{}, false +} + +// PropertyId maps a definition back to the id that serves it, which is what a +// real wiring does: a re-export resolves a recommended list that came back as +// bare KEYS (no resolver on the reader) through this reverse lookup, so a +// second generation carries the same definitions as the first. +func (hostileTypePropResolver) PropertyId(def PropertyDefinition) (string, bool) { + switch def.Key { + case "owner": + return "hp1", true + case "genre": + return "hp2", true + case domain.RelationKey(overLongPropertyKey): + return "hp3", true + } + return "", false +} + +// typePropTargets is one type_properties entry reduced to its two KEY slots: +// the resolved property key and the resolved target type keys of +// `object_types` (§2a). Both are what a reader ends up storing, so both must +// invert. +type typePropTargets struct { + Key string + Targets []string +} + +// wantTypePropTargets is what a faithful reader owes back for a type-document +// seed, read off hostileTypePropResolver itself so the expectation cannot +// drift from the fixture — the definitions in §2a section order (featured +// first, then the regular list), each with its target types intact. hp3 is +// deliberately absent: its stored key cannot be written, so export drops the +// entry with a warning rather than emit one the seam refuses (§2a). +func wantTypePropTargets() []typePropTargets { + out := make([]typePropTargets, 0, 2) + for _, id := range []string{"hp1", "hp2"} { + def, ok := hostileTypePropResolver{}.PropertyById(id) + if !ok { + panic("hostileTypePropResolver no longer serves " + id) + } + out = append(out, typePropTargets{Key: string(def.Key), Targets: def.ObjectTypes}) + } + return out +} + +// capturedTypeProps is a reader-side PropertyResolver that records what the +// type-property seam hands it. It is the ONLY way to see the `object_types` +// slot from outside: applyTypeProperties resolves those terms and passes them +// in the definition, while the recommended-relation lists it writes into the +// snapshot carry property ids, not targets. Answering false keeps the +// snapshot identical to the resolver-less read (the key passes through in +// place of an id), so the capture observes without steering. +type capturedTypeProps struct{ got []typePropTargets } + +func (c *capturedTypeProps) PropertyById(string) (PropertyDefinition, bool) { + return PropertyDefinition{}, false +} + +func (c *capturedTypeProps) PropertyId(def PropertyDefinition) (string, bool) { + c.got = append(c.got, typePropTargets{Key: string(def.Key), Targets: def.ObjectTypes}) + return "", false +} + +// invertedTypes is what a faithful reader owes back for a snapshot's +// ObjectTypes: the format writes object_types[0] as `type` — and [1] as +// `template_for` on a TEMPLATE — each spelled through the vocabulary in force +// and inverted through the document's own legend, so the stored KEYS must +// survive whatever the vocabulary did to their spellings. Entries normalize +// to the `ot-` URL form; a non-template's types past the first are not +// modeled by the format (§2). +// +// The second slot is keyed off the smartblock TYPE, not off keys[0] being the +// template key. This model said the latter until v0.22, faithfully, because +// export did — and both were wrong for a template whose object types do not +// begin with the template key, which is a shape the model permits and which +// lost its target type in silence. +// +// An entry with no key ("ot-", "") has no spelling and is dropped — and the +// survivors CLOSE RANKS, which is the load-bearing half. Dropping in place +// made a keyless entry contagious: it silenced the slot it sat in, and a +// silent `type` slot makes `template_for` inexpressible, so a template stored +// as ["ot-", "ot-task"] came back as no types at all rather than as ot-task. +func invertedTypes(objectTypes []string, sbType model.SmartBlockType) []string { + keys := make([]string, 0, len(objectTypes)) + for _, t := range objectTypes { + if key := strings.TrimPrefix(t, "ot-"); key != "" { + keys = append(keys, key) + } + } + if len(keys) == 0 { + return nil + } + out := []string{"ot-" + keys[0]} + if sbType == model.SmartBlockType_Template && len(keys) > 1 { + out = append(out, "ot-"+keys[1]) + } + return out +} + +// hostileVocab deliberately breaks the KeyVocabulary contract the way a real +// space can: a spelling comes from a display NAME — user-typed text, with no +// length bound and no charset audit — so nothing upstream guarantees a +// spelling is writable, and a space may name a property so that its spelling +// shadows a bundled one. This slot had no invariant coverage, which is +// exactly how "check the stored key, emit the spelling" shipped. +type hostileVocab struct{ BundledKeyVocabulary } + +func (hostileVocab) PropertySlug(key string) string { + switch key { + case "name": + return strings.Repeat("s", maxPropertyKeyLen+64) // apiObjectKey has no length bound + case "dueDate": + return "due\ndate" // a control character is not a spelling + case "artist": + return "" // a vocabulary with no answer at all + case "a\nb": + // a writable spelling for a stored key that is not writable — the + // mirror of the over-long slug above, and the shape that makes the + // guard's stored-key half load-bearing: the legend entry this + // spelling owes would carry a control character in its VALUE. + return "ab" + case "6a32d4856761631534b22f85": + // a space-minted spelling shadowing a bundled internal key's legacy + // slug — which is ALSO a stored key on the hostile details, so the term + // ledger must refuse the claim outright (§3: a stored key always + // keeps its own term, and this one owes an identity entry). The + // corpus's legend-rebind documents cover the honored-entry case. + return "unique_key" + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +// TypeSlug breaks the contract the same ways the property half does, plus +// the two shapes only the type namespace has: a spelling that collides with a +// bundled TYPE key (the confirmed defect — `69bbfc…` spelled `task` read +// back as the bundled Task type by a package-only reader), and answers that +// move the reserved `template` spelling in either direction, which silently +// drops a template's target type. +func (hostileVocab) TypeSlug(key string) string { + switch key { + case "69bbfc78877a91b1d12d1a7c": + return "task" + case "longslugged": + return strings.Repeat("t", maxPropertyKeyLen+64) + case "blankslugged": + return "" + case "squatter": + return "template" + case "template": + return "tmpl" + case overLongTypeKey: + // a perfectly good spelling for a stored key that is NOT one. The + // spelling half of the guard is what an over-long *spelling* trips; + // this is the mirror, and it was unreachable while the vocabulary had + // no answer for this key — `slug == key` short-circuits before the + // guard runs. Emitting the spelling regardless writes + // `type_internal_keys: {"diary": "yyy…(140)"}`, a legend value 12 characters + // past the 128 the schema bounds it to: Marshal producing a document + // its own Validate rejects, which is I1. + return "diary" + } + return BundledKeyVocabulary{}.TypeSlug(key) +} + +// roundTripVocab is hostileVocab's CONFORMING twin, and the only vocabulary +// in this file that is a genuine inverse pair: whatever a `…Slug` emits, the +// matching `…Key` inverts, and no answer binds a spelling the bundled table +// binds to a different key — the precondition KeyVocabulary states (§3, +// keyvocab.go). hostileVocab satisfies neither half; it is a WRITER +// vocabulary, aimed at what export must refuse. +// +// It exists because §11.1 states the round-trip guarantee for export and +// import "wired with equivalent resolvers", and nothing exercised that: every +// reader in this sweep is package-only, so a stored key whose spelling only +// the writer's vocabulary knows was never read back through it. +// +// The pathological shapes are kept — an over-long spelling, a spelling for a +// stored key that is over-long — because export BACKS THOSE OFF, so the document +// never carries them and the reader inverts the stored key verbatim. That is +// the conforming half of the same hostility. +type roundTripVocab struct{} + +var roundTripTypeSlugs = map[string]string{ + customTypeKey: "tsk7", + "squatter": "sqtr", + "longslugged": strings.Repeat("t", maxPropertyKeyLen+64), + overLongTypeKey: "diary", +} + +var roundTripPropertySlugs = map[string]string{ + "6a32d4856761631534b22f85": "prio", + "artist": "artist_name", +} + +func (roundTripVocab) PropertySlug(key string) string { + if slug, ok := roundTripPropertySlugs[key]; ok { + return slug + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (roundTripVocab) PropertyKey(slug string) (string, bool) { + for key, s := range roundTripPropertySlugs { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (roundTripVocab) TypeSlug(key string) string { + if slug, ok := roundTripTypeSlugs[key]; ok { + return slug + } + return BundledKeyVocabulary{}.TypeSlug(key) +} + +func (roundTripVocab) TypeKey(slug string) (string, bool) { + for key, s := range roundTripTypeSlugs { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// shadowVocab is roundTripVocab's third twin, and the one that says the +// legend is not only about the bundled table. It CONFORMS — a strict inverse +// pair, and no answer touches a spelling the bundled table binds (`initiative` +// and `squatter` are not in it, in either namespace) — and it still binds two +// spellings the corpus carries as STORED keys: the BSON property key is +// spelled `initiative`, which the details hold verbatim, and the BSON +// type key `squatter`, which the type pools and hp1's `object_types` +// hold verbatim. +// +// That is the shape a real space grows on its own: deleting a type or a +// property vacates its stored key (storeresolver's corpse policy) while +// the objects that used it keep the stored key, and the freed spelling +// becomes somebody else's. Export backs the spelling off — the census +// reserved the stored key — and then wrote the stored key with no legend +// entry, so this very vocabulary bound it to the other holder on the way +// back: a re-pointed type in silence, and two spellings of one property +// loudly (Unmarshal refuses a document Marshal just wrote — I1). +type shadowVocab struct{} + +var shadowPropertySlugs = map[string]string{"6a32d4856761631534b22f85": "initiative"} +var shadowTypeSlugs = map[string]string{customTypeKey: "squatter"} + +func (shadowVocab) PropertySlug(key string) string { + if slug, ok := shadowPropertySlugs[key]; ok { + return slug + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (shadowVocab) PropertyKey(slug string) (string, bool) { + for key, s := range shadowPropertySlugs { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (shadowVocab) TypeSlug(key string) string { + if slug, ok := shadowTypeSlugs[key]; ok { + return slug + } + return BundledKeyVocabulary{}.TypeSlug(key) +} + +func (shadowVocab) TypeKey(slug string) (string, bool) { + for key, s := range shadowTypeSlugs { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// I1: Marshal either fails loudly — §11 allows that for an over-deep tree or a +// table inside a cell — or produces a document its own Validate accepts AND +// its own Unmarshal imports. What it may never do is succeed and hand back an +// unimportable archive, which is a failure nobody sees until the archive is +// needed. The Unmarshal leg runs as a package-only reader, because that is +// what an archive's consumer is: it checks that the document plus its legend +// resolve without the vocabulary that wrote them — this leg, gated on +// Validate alone for a while, is where a valid-but-unimportable document (a +// shadow twin pair; a backed-off spelling a block slot recorded anyway) hid. +func TestInvariant_MarshalOutputValidates(t *testing.T) { + variants := map[string]struct { + write Options + // read is the vocabulary the READER is wired with. Empty is the + // default and the archive case — a package-only reader, which is what + // an archive's consumer is. roundTripVocab is the other precondition + // §11.1 offers, "equivalent resolvers", and the two are different + // promises: one says the document plus its legend stand alone, the + // other says the document plus the writer's own vocabulary do. + read KeyVocabulary + // readSpaceId is the space the READER is reading into — not a + // vocabulary, and not something the writer has to hand over: an + // import lands in a space, so every importer has one. The folded + // variant needs it because the fold trades the space half of a + // participant id for the reader's own (§9). Empty stays the default + // everywhere else, and a reader that leaves it empty on a folded + // document is warned rather than left to store a bare identity. + readSpaceId string + }{ + "plain": {}, + "compact": {write: Options{CompactIds: true}}, + "omitIds": {write: Options{OmitIds: true}}, + "hostileVocab": {write: Options{Keys: hostileVocab{}}}, + // the read shape (§9): every reference captioned and every + // participant folded. Without this variant no invariant run ever + // sees a `#name` suffix or a folded identity — the corpus would stop + // reaching the code under test, which is how a green invariant lies. + "refNames": {write: Options{ + RefNames: true, + ResolveObjectNames: hostileObjectNames{}, + SpaceId: hostileSpaceId, + }, readSpaceId: hostileSpaceId}, + "roundTripVocab": {write: Options{Keys: roundTripVocab{}}, read: roundTripVocab{}}, + "shadowVocab": {write: Options{Keys: shadowVocab{}}, read: shadowVocab{}}, + } + for name, variant := range variants { + t.Run(name, func(t *testing.T) { + for n := 0; n < 300; n++ { + sbType, snap := hostileSnapshot(n) + o := variant.write + o.ResolveOptions = hostileOptions + var wantProps []typePropTargets + if sbType == model.SmartBlockType_STType { + o.ResolveProperties = hostileTypePropResolver{} + wantProps = wantTypePropTargets() + } + data, err := Marshal(sbType, snap, o) + if err != nil { + continue // a loud failure is allowed; silent invalidity is not + } + var warned []Issue + require.NoError(t, ValidateWarn(data, func(i Issue) { warned = append(warned, i) }), + "seed %d produced:\n%s", n, data) + // I1's option-legend half: an `option_ids` outer key is a + // property SPELLING, and export groups under the spelling it + // just used — so a key it emits is in the document's own + // property census by construction, and Validate has nothing to + // report about the legend. If it ever does, the legend export + // wrote is one import will step over in silence + // (optionrefs.go). + assert.Empty(t, optionIdsWarnings(warned), "seed %d produced:\n%s", n, data) + // a block reached twice is written once (§11). The mark that + // says so is set in blockToJSON, and every emit path has to + // consult it — including the table cell's string shorthand, + // which does not go through blockToJSON at all. The corpus's + // two-parent cell is reached from a plain block and from its + // row, so a path that writes without checking writes it twice. + assert.LessOrEqual(t, strings.Count(string(data), sharedCellMarker), 1, + "seed %d emitted one block twice:\n%s", n, data) + // the option legend must actually be IN the document, or this + // sweep asks nothing about it — a corpus that stops reaching + // the code under test is the way a green invariant lies. Every + // name the deleted flat spelling could not carry is asserted + // here, and the same-named pair by its first writing + // (optionrefs.go). + names := docOptionIds(t, data)["Tag"] + if o.OmitIds { + // an id-less shape ships no legend of ids (§9), so here + // the assertion is that it is GONE — which is as much a + // statement about the corpus reaching the code as the + // positive one below + assert.Empty(t, names, "seed %d kept an option legend under OmitIds:\n%s", n, data) + } else { + for name, id := range map[string]string{ + "C#": "opt-hash", + "import issue": "opt-space", + "books": "opt-dup1", + strings.Repeat("n", maxPropertyKeyLen+1): "opt-long", + } { + assert.Equal(t, id, names[name], + "seed %d owes the legend an entry for %q:\n%s", n, name, data) + } + } + capture := &capturedTypeProps{} + _, back, err := Unmarshal(data, Options{ + GenerateId: seqIds(fmt.Sprintf("g%d_", n)), + Keys: variant.read, + SpaceId: variant.readSpaceId, + ResolveProperties: capture, + // the option resolver is wired on BOTH ends, which is what + // §11.1's "equivalent resolvers" means for select values — + // a different axis from the key vocabulary above, and the + // only wiring under which the option legend is an answer + // at all (a reader with no space cannot check an id) + ResolveOptions: hostileOptions, + }) + require.NoError(t, err, + "seed %d produced a valid document its own Unmarshal refuses:\n%s", n, data) + // the type slots must INVERT, not merely import: binding the + // spelling to a different stored type is exactly the silent + // failure the type_internal_keys legend exists to close (§3), and no + // error marks it + assert.Equal(t, invertedTypes(snap.ObjectTypes, sbType), back.ObjectTypes, + "seed %d: the archive must bind back to the types it came from:\n%s", n, data) + // and so must the OTHER type slot. `type_properties[].object_types` + // shares the envelope's term ledger and its legend, but nothing + // asserted it: the envelope truncates to one type, so the census's + // whole stated purpose — a type document naming many types — lived + // entirely in a slot no assertion watched. Dropping the ledger + // back-off left this corpus green while a bundled target was + // silently replaced by a custom type sharing its spelling. + assert.Equal(t, wantProps, capture.got, + "seed %d: the type properties must bind back to the types they came from:\n%s", n, data) + // §11.2 states byte-stability from a DOCUMENT, and + // TestInvariant_ImportedDocumentReExportsValid checks it + // there. This is the snapshot-side half — Export(S) == + // Export(Import(Export(S))) — which is what §9's "re-exports + // diff cleanly" means for an object exported twice, once + // before a round trip through the format and once after. + // + // Both generations are compared with ids OMITTED, because + // import mints an id wherever the snapshot had none (§9): a + // snapshot carrying an id-less block or view exports a + // document that is not canonical, and a second generation + // then differs by exactly those minted ids (§11.2 says so). + // Ids have their own assertions above and in + // TestExport_ValidIdsAreNeverRenamed; what this one asks is + // whether everything else — the terms, the legends, and the + // census that chose them — is a fixpoint. + stripped := o + stripped.OmitIds = true + stripped.SpaceId = variant.readSpaceId + gen1, err := Marshal(sbType, snap, stripped) + require.NoError(t, err, "seed %d", n) + gen2, err := Marshal(sbType, back, stripped) + require.NoError(t, err, "seed %d", n) + assert.Equal(t, string(gen1), string(gen2), + "seed %d: exporting the snapshot that came back must reproduce the document", n) + if variant.read == nil { + continue + } + // With equivalent resolvers on both ends (§11.1), the + // vocabulary is a spelling choice and nothing else: the + // snapshot that comes back must be the one a package-only + // round trip produces. Anything else means a term bound to a + // different stored key on the way home — which is where the + // PROPERTY namespace's half of this hole lives, since the + // assertions above watch only the type slots. + plainOpts := Options{ResolveOptions: hostileOptions} + if sbType == model.SmartBlockType_STType { + plainOpts.ResolveProperties = hostileTypePropResolver{} + } + plainData, err := Marshal(sbType, snap, plainOpts) + require.NoError(t, err, "seed %d", n) + _, plainBack, err := Unmarshal(plainData, Options{ + GenerateId: seqIds(fmt.Sprintf("g%d_", n)), + ResolveOptions: hostileOptions, + }) + require.NoError(t, err, "seed %d", n) + assert.Equal(t, plainBack, back, + "seed %d: a vocabulary spells the same snapshot differently; it may not mean a different one:\n%s", n, data) + } + }) + } +} + +// The same invariant on the goldens' own fixture, which is the case the +// existing corpus covers — kept so a regression there is not mistaken for a +// hostile-input-only problem. +// +// The legend CONTENT is asserted, not just the document's validity: "validates +// and warns about nothing" stays green if the legend vanishes entirely, which +// is the way this fixture would stop asking the question. On the id-less shape +// the assertion is the other way round — the legend must be gone (§9). +func TestInvariant_MarshalOutputValidates_RichFixture(t *testing.T) { + for name, tc := range map[string]struct { + opts Options + wantLegend map[string]map[string]string + }{ + "plain": {testOptions(), + legend("customStatus", map[string]string{"In progress": "opt1", "Done": "opt2"})}, + "compact": {Options{CompactIds: true, ResolveFormat: testFormatResolver, ResolveOptions: testResolver}, + legend("customStatus", map[string]string{"In progress": "opt1", "Done": "opt2"})}, + "omitIds": {Options{OmitIds: true, ResolveFormat: testFormatResolver, ResolveOptions: testResolver}, + nil}, + } { + t.Run(name, func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, richSnapshot(), tc.opts) + require.NoError(t, err) + var warned []Issue + require.NoError(t, ValidateWarn(data, func(i Issue) { warned = append(warned, i) })) + assert.Empty(t, optionIdsWarnings(warned), "%s", data) + assert.Equal(t, tc.wantLegend, docOptionIds(t, data), "%s", data) + }) + } +} + +// hostileDocs are hand-written documents aimed at the seam between the schema's +// idea of a value and Go's: the schema says "integer", the decoder says int64, +// and JSON Schema counts 2048.0 and 1e1 as integers. Every one of these is a +// document a generator can plausibly emit. +var hostileDocs = []string{ + `{"version": 2}`, + `{"version": 2.0}`, + `{"version": 2e0}`, + `{"version": 2.5}`, + `{"version": 3}`, + `{"version": 0}`, + `{"version": 2, "blocks": [{"type": "file", "size": 2048}]}`, + `{"version": 2, "blocks": [{"type": "file", "size": 2048.0}]}`, + `{"version": 2, "blocks": [{"type": "file", "size": 1e3}]}`, + `{"version": 2, "blocks": [{"type": "file", "size": 1e30}]}`, + `{"version": 2, "blocks": [{"type": "file", "size": -1}]}`, + `{"version": 2, "blocks": [{"type": "file", "size": 2048.5}]}`, + `{"version": 2, "blocks": [{"type": "widget", "limit": 10}]}`, + `{"version": 2, "blocks": [{"type": "widget", "limit": 1e1}]}`, + `{"version": 2, "blocks": [{"type": "widget", "limit": 1e20}]}`, + `{"version": 2, "blocks": [{"type": "widget", "limit": -3}]}`, + `{"version": 2, "blocks": [{"type": "dataview", "views": [{"id": "v1", "page_size": 50.0}]}]}`, + `{"version": 2, "blocks": [{"type": "dataview", "views": [{"id": "v1", "page_size": 1e19}]}]}`, + `{"version": 2, "blocks": [{"type": "dataview", "views": [{"id": "v1", "page_size": 0}]}]}`, + `{"version": 2, "blocks": [{"type": "table", "columns": [{"id": "c1", "width": 120.7}], "rows": []}]}`, + `{"version": 2, "blocks": [{"type": "table", "columns": [{"id": "c1", "width": 1e30}], "rows": []}]}`, + `{"version": 2, "blocks": [{"type": "table", "columns": [{"id": "c1", "width": -5}], "rows": []}]}`, + `{"version": 2, "blocks": [{"indent": 0.0, "type": "paragraph", "text": "x"}]}`, + `{"version": 2, "blocks": [{"indent": 1e1, "type": "paragraph", "text": "x"}]}`, + `{"version": 2, "properties": {"name": "x", "size": 9007199254740993}}`, + `{"version": 2, "blocks": [{"type": "paragraph", "text": "x"}]}`, + // a JSON number larger than float64 can hold. The loose surfaces have no + // schema bound to catch it by construction (§3 accepts any number), and the + // snapshot they decode into is a proto Struct, whose numbers are float64 — + // so there is nowhere to put such a value, and the answer has to be a + // path-addressed rejection rather than a decode error + `{"version": 2, "properties": {"num": 1e400}}`, + `{"version": 2, "properties": {"num": 1e309}}`, + `{"version": 2, "properties": {"num": 1e308}}`, + `{"version": 2, "store": {"k": 1e400}}`, + `{"version": 2, "blocks": [{"type": "paragraph", "text": "x", "fields": {"w": 1e400}}]}`, + `{"version": 2, "blocks": [{"type": "table", "columns": [{"id": "c1", "width": 1e400}], "rows": []}]}`, + `{"version": 2, "blocks": [{"type": "dataview", "views": [{"id": "v1", + "filters": [{"property": "p", "condition": "equal", "value": 1e400}]}]}]}`, + `{"version": 2, "blocks": [{"type": "table", "columns": [{"id": "c1"}], + "rows": [{"id": "r1", "cells": [["nested", {"indent": 1, "type": "paragraph", "text": "y"}]]}]}]}`, + // admission runs on the RESOLVED stored key (§3): the legacy slug + // spelling of a denied key (the fold still resolves it), a property_internal_keys legend rebinding a harmless + // spelling onto one, and the layout-name check behind the same resolution. + // These pin WHERE the rule lives as much as that it exists — a "fix" that + // moves the deny rule into import alone makes Validate accept what + // Unmarshal rejects, and this corpus is what catches that. + `{"version": 2, "properties": {"unique_key": "ot-page"}}`, + `{"version": 2, "properties": {"space_id": "other"}}`, + `{"version": 2, "properties": {"old_anytype_id": "legacy-1"}}`, + `{"version": 2, "properties": {"source_file_path": "/x/y"}}`, + `{"version": 2, "properties": {"resolved_layout": "nonsense"}}`, + `{"version": 2, "properties": {"resolved_layout": "todo"}}`, + `{"version": 2, "property_internal_keys": {"prio": "uniqueKey"}, "properties": {"prio": "ot-page"}}`, + `{"version": 2, "property_internal_keys": {"myid": "id"}, "properties": {"myid": "boom"}}`, + `{"version": 2, "property_internal_keys": {"s": "spaceId"}, "properties": {"s": "other"}}`, + // a benign rebind is the legend working as specified, and flows through + `{"version": 2, "property_internal_keys": {"prio": "6a32d4856761631534b22f85"}, "properties": {"prio": "high"}}`, + // a counting date preset with no count: an error where the preset's day + // range is applied, and nothing at all where it is inert (§6.2). Both + // halves of transformDateFilter's gate make it inert — the condition, and + // the property's format, which here comes from the bundled table because + // the block declares no properties list at all + `{"version": 2, "blocks": [{"type": "dataview", "views": [{"id": "v1", + "filters": [{"property": "due_date", "condition": "empty", "date_preset": "number_of_days_ago"}]}]}]}`, + `{"version": 2, "blocks": [{"type": "dataview", "views": [{"id": "v1", + "filters": [{"property": "due_date", "condition": "greater", "date_preset": "number_of_days_ago"}]}]}]}`, + `{"version": 2, "blocks": [{"type": "dataview", + "properties": [{"property": "due_date", "format": "text"}], "views": [{"id": "v1", + "filters": [{"property": "due_date", "condition": "greater", "date_preset": "number_of_days_ago"}]}]}]}`, + `{"version": 2, "blocks": [{"type": "dataview", "views": [{"id": "v1", + "filters": [{"property": "not_a_property", "condition": "greater", "date_preset": "number_of_days_ago"}]}]}]}`, + // a legend value is a stored key and obeys the writable-key rule (§3) + `{"version": 2, "property_internal_keys": {"p": ""}}`, + // a key holding a JSON-pointer metacharacter: legal in a stored key and + // in a spelling (§3 bounds length and control characters, nothing else), + // so both surfaces have to address it escaped — the accepted member as + // much as the refused legend value + `{"version": 2, "properties": {"a/b": "x"}}`, + `{"version": 2, "properties": {"a~b": "x"}}`, + `{"version": 2, "property_internal_keys": {"a/b": ""}}`, + `{"version": 2, "property_internal_keys": {"p": "a\nb"}}`, + `{"version": 2, "property_internal_keys": {"p": "` + strings.Repeat("k", 129) + `"}}`, + // the verbatim-first family (§3): twin spellings binding one stored key + // are refused by BOTH halves with default Options; an identity entry + // makes a shadow spelling a stored key in every reader; a legend VALUE is + // admitted like the stored key it is, member or no member spelling it + `{"version": 2, "properties": {"iconEmoji": "a", "icon_emoji": "b"}}`, + `{"version": 2, "properties": {"dueDate": "2025-01-01T00:00:00Z", "due_date": "x"}}`, + `{"version": 2, "property_internal_keys": {"unique_key": "unique_key"}, "properties": {"unique_key": "custom"}}`, + `{"version": 2, "property_internal_keys": {"unique_key": "6a32d4856761631534b22f85"}, "properties": {"unique_key": "high"}}`, + `{"version": 2, "property_internal_keys": {"sneaky": "uniqueKey"}}`, + `{"version": 2, "property_internal_keys": {"p": "oldAnytypeID"}}`, + // two spellings the document's own chain accepts that a WIDER vocabulary + // resolves onto a denied / an unwritable key — the i2Vocabularies entries + // that widen resolution exercise the §3 seam through these + `{"version": 2, "properties": {"prio": "bare"}}`, + `{"version": 2, "properties": {"blank": "x"}}`, + // the TYPE namespace mirrors the legend rules (§3): a benign rebind and + // an identity entry flow through, a legend value obeys the writable-key + // rule, the template gate runs on the RESOLVED type key, and two + // spellings a wider vocabulary resolves further than the document's own + // chain (the type-axis i2Vocabularies entries widen through these) + `{"version": 2, "type": "task"}`, + `{"version": 2, "type": "tsk"}`, + `{"version": 2, "type": "blanktype"}`, + `{"version": 2, "kind": "template", "type": "template", "template_for": "blanktype"}`, + `{"version": 2, "type_internal_keys": {"task": "69bbfc78877a91b1d12d1a7c"}, "type": "task"}`, + `{"version": 2, "type_internal_keys": {"object_type": "object_type"}, "type": "object_type"}`, + `{"version": 2, "type_internal_keys": {"t": ""}}`, + `{"version": 2, "type_internal_keys": {"t": "a\nb"}}`, + `{"version": 2, "type_internal_keys": {"t": "` + strings.Repeat("k", 129) + `"}}`, + `{"version": 2, "kind": "template", "type_internal_keys": {"template": "custom1"}, "type": "template", "template_for": "page"}`, + `{"version": 2, "kind": "template", "type_internal_keys": {"tpl": "template"}, "type": "tpl", "template_for": "page"}`, + `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_internal_keys": {"task": "69bbfc78877a91b1d12d1a7c"}, + "type_settings": {"property_definitions": [{"property": "owner", "format": "objects", "object_types": ["task", "blanktype"]}]}}`, + // a property definition's `property` is a PROPERTY key slot and admits like one: the + // schema bounds the spelling, a wider vocabulary resolves past it — and + // the two shapes the seam refuses with the DEFAULT vocabulary, where no + // resolution widens anything and the schema's `minLength: 1` is the only + // bound the slot ever had + `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "blank", "format": "text"}]}}`, + `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "` + strings.Repeat("k", maxPropertyKeyLen+1) + `"}]}}`, + `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "a\nb"}]}}`, + // the `option_ids` legend (§9a) at both levels: an entry the document + // spells, an entry nothing spells (the warning), the shapes the deleted + // flat key could not represent at all (a spelling carrying `#`, an option + // name carrying one, an option name past the joined key's bound), an + // option name a plain charset would have rejected, and the malformed + // keys — empty at either level, past the writable-key bound, a control + // character — which the schema and the package restatement have to refuse + // identically. A former plain compaction label is here too, in an + // object-id slot, since nothing resolves one now. + `{"version": 2, "option_ids": {"tag": {"High": "bafyreiopt"}}, "properties": {"tag": ["High"]}}`, + `{"version": 2, "option_ids": {"tag": {"import issue": "bafyreiopt"}}, + "properties": {"tag": ["import issue"]}, + "blocks": [{"type": "link", "object_id": "miovm"}]}`, + `{"version": 2, "option_ids": {"c#_lang": {"C#": "bafyreiopt"}}}`, + `{"version": 2, "option_ids": {"tag": {"#": "bafyreiopt"}}}`, + `{"version": 2, "option_ids": {"tag": {"has space": "bafyreiopt"}}}`, + `{"version": 2, "option_ids": {"tag": {"a\nb": "bafyreiopt"}}, "properties": {"tag": ["a\nb"]}}`, + `{"version": 2, "option_ids": {"tag": {"": "bafyreiopt"}}}`, + `{"version": 2, "option_ids": {"": {"High": "bafyreiopt"}}}`, + `{"version": 2, "option_ids": {"tag": "bafyreiopt"}}`, + `{"version": 2, "option_ids": {"ta\ng": {"High": "bafyreiopt"}}}`, + `{"version": 2, "option_ids": {"` + strings.Repeat("p", maxPropertyKeyLen+1) + `": {"High": "bafyreiopt"}}}`, + `{"version": 2, "option_ids": {"tag": {"` + strings.Repeat("n", maxPropertyKeyLen+1) + `": "bafyreiopt"}}, + "properties": {"tag": ["` + strings.Repeat("n", maxPropertyKeyLen+1) + `"]}}`, + // the `template` spelling, straight through the envelope: the + // type-moves-template vocabulary answers a different stored key for it, + // and the kind must be unmoved by that — it is read off `kind` and the + // vocabulary cannot reach `kind` + `{"version": 2, "kind": "template", "type": "template", "template_for": "task"}`, + `{"version": 2, "kind": "template", "type": "tpl", "template_for": "task"}`, + // a PAGE whose object type is the template type: the one shape the + // pre-v0.22 rule could not tell apart from a template, and the reason + // export spells `kind` out here + `{"version": 2, "kind": "page", "type": "template"}`, + // and the four refusals that replace the reservation — Validate rejects + // each, so I2 asks that Unmarshal reject them too rather than decoding a + // page that meant to be a template + `{"version": 2, "type": "template", "template_for": "task"}`, + `{"version": 2, "type": "template"}`, + `{"version": 2, "kind": "page", "type": "template", "template_for": "task"}`, + `{"version": 2, "kind": "template", "template_for": "task"}`, +} + +// i2Vocabularies is the Options axis I2 runs over. A vocabulary can resolve +// spellings the document's own chain (legend → bundled table → verbatim) +// cannot, and §3 licenses import to refuse MORE than Validate then: admission +// re-runs at the details seam on the wider resolved key, which Validate — +// deliberately vocabulary-less (§13) — never sees. For those configurations +// the invariant is containment plus path-addressed refusals; where nothing +// widens resolution, it is exact agreement. +var i2Vocabularies = map[string]struct { + keys KeyVocabulary + widens bool +}{ + "default": {nil, false}, + "bundled": {BundledKeyVocabulary{}, false}, + // a symmetric node-backed vocabulary: both directions agree + "space": {spaceVocabulary{slugOf: map[string]string{"6a32d4856761631534b22f85": "priority"}}, true}, + // PropertySlug and PropertyKey are NOT inverses — the accept side answers + // for a spelling the emit side never writes, the way a stale or hand-rolled + // vocabulary really breaks; the target is an ordinary custom key, so only + // the binding moves, never admission + "asymmetric": {asymmetricVocab{}, true}, + // the two resolutions the seam exists to refuse + "resolves-denied": {rebindingVocabulary{}, true}, + "resolves-unwritable": {blankKeyVocab{}, true}, + // the TYPE axis of the same matrix: a symmetric node-backed vocabulary, + // an asymmetric one whose accept side answers for a spelling the emit side + // never writes, and the unwritable resolution the type seam refuses + // the space-minted spelling SHADOWS the bundled `task` spelling, which is + // the collision this axis exists for — with a spelling no document spells + // (`task2`) the axis was inert by construction, and the corpus's + // `{"type": "task"}` never met a vocabulary that answers for it + "type-space": {typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "task"}}, true}, + "type-asymmetric": {asymmetricTypeVocab{}, true}, + "type-resolves-unwritable": {blankTypeVocab{}, true}, + // a vocabulary that moves the `template` spelling in either direction. + // This used to be held to the document's own answer by a reservation + // (importer.typeKey) because the kind was derived from the same field; + // since v0.22 the kind comes from `kind`, which no vocabulary can reach, + // so the vocabulary's answer is simply taken. It still widens nothing + // that could make Unmarshal refuse — it never resolves onto the empty + // key — so exact agreement is still the assertion. + "type-moves-template": {templateMovingVocab{}, false}, +} + +type asymmetricTypeVocab struct{ BundledKeyVocabulary } + +func (asymmetricTypeVocab) TypeKey(slug string) (string, bool) { + if slug == "tsk" { + return "69bbfc78877a91b1d12d1a7c", true + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +type asymmetricVocab struct{ BundledKeyVocabulary } + +func (asymmetricVocab) PropertyKey(slug string) (string, bool) { + if slug == "prio" { + return "6a32d4856761631534b22f85", true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +// I2: whatever Validate accepts, Unmarshal must decode, and whatever Validate +// rejects, Unmarshal must reject too — exactly, for every Options +// configuration that does not widen resolution, and as containment (Unmarshal +// accepts a subset, refusing only through path-addressed admission) for the +// vocabularies that do. A disagreement means the guarantee Validate offers — +// "this document imports" — is not true, and the failure arrives as a bare Go +// decode error with no JSON pointer, outside the path-addressed error +// contract §13 promises. +func TestInvariant_ValidateAndUnmarshalAgree(t *testing.T) { + for vocabName, vocab := range i2Vocabularies { + t.Run(vocabName, func(t *testing.T) { + for i, doc := range hostileDocs { + t.Run(doc[:min(len(doc), 60)], func(t *testing.T) { + valErr := Validate([]byte(doc)) + _, _, unmErr := Unmarshal([]byte(doc), + Options{GenerateId: seqIds(fmt.Sprintf("g%d_", i)), Keys: vocab.keys}) + switch { + case valErr != nil: + assert.Error(t, unmErr, + "Validate rejects this document, so Unmarshal must too: %v", valErr) + case !vocab.widens: + assert.NoError(t, unmErr, + "Validate accepts and nothing widens resolution, so Unmarshal must accept") + case unmErr != nil: + var ve *ValidationError + assert.ErrorAs(t, unmErr, &ve, + "a wider vocabulary may refuse more, but only through path-addressed admission") + } + if unmErr != nil { + // every rejection is path-addressed — never a raw + // decode error escaping from the Go layer. Checked + // whenever Unmarshal fails: the old clause ran only + // under valErr != nil, where Unmarshal returns + // validateToDoc's error unchanged — so it could never + // fire — and nil-panicked when Unmarshal wrongly + // accepted what Validate refused. + assert.NotContains(t, unmErr.Error(), "decode document", + "the reason must come from validation, not from json.Unmarshal") + } + }) + } + }) + } +} + +// Whatever Unmarshal accepts must re-export to something Validate accepts too: +// this is I1 with the input side as the generator, which is how an agent's +// document actually travels. +func TestInvariant_ImportedDocumentReExportsValid(t *testing.T) { + for _, doc := range hostileDocs { + if Validate([]byte(doc)) != nil { + continue + } + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err, doc) + out, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err, doc) + require.NoError(t, Validate(out), "%s re-exported as:\n%s", doc, out) + + // and the canonical form is byte-stable through another round (§11.2) + _, snap2, err := Unmarshal(out, Options{GenerateId: seqIds("h")}) + require.NoError(t, err, doc) + again, err := Marshal(sbType, snap2, Options{}) + require.NoError(t, err, doc) + assert.Equal(t, string(out), string(again), "re-export must be byte-stable for %s", doc) + } +} + +// A document's ids must survive a round trip unchanged when they are already +// valid: sanitizing is for ids that need it, and renaming one that does not +// would break the "provided ids are preserved so re-exports diff cleanly" +// promise (§9). +func TestExport_ValidIdsAreNeverRenamed(t *testing.T) { + doc := `{"version": 2, "id": "obj1", "blocks": [ + {"type": "paragraph", "id": "a_b", "text": "first"}, + {"type": "paragraph", "id": "keep-me", "text": "second"}, + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": ["x"]}]}]}` + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + out, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err) + + var got struct { + Blocks []struct { + Id string `json:"id"` + Columns []struct { + Id string `json:"id"` + } `json:"columns"` + Rows []struct { + Id string `json:"id"` + } `json:"rows"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(out, &got)) + require.Len(t, got.Blocks, 3) + assert.Equal(t, "a_b", got.Blocks[0].Id) + assert.Equal(t, "keep-me", got.Blocks[1].Id) + assert.Equal(t, "c1", got.Blocks[2].Columns[0].Id) + assert.Equal(t, "r1", got.Blocks[2].Rows[0].Id) +} + +// I3: every identifier the format defines is snake_case (§1 Naming). The rule +// is worth a test rather than a review habit — the vocabulary grows one enum +// value at a time, and a camelCase addition would be invisible until a +// generating model tripped over it, which is the failure the rename fixed. +// +// It covers the Go name tables as well as the schema: some vocabulary (the +// layout names) exists only in Go, which is exactly where a stray name hides. +func TestInvariant_VocabularyIsSnakeCase(t *testing.T) { + snake := regexp.MustCompile(`^[a-z][a-z0-9_]*$`) + + check := func(t *testing.T, where, ident string) { + t.Helper() + if strings.HasPrefix(ident, "$") { + return + } + // A name in the platform's `_` namespace (§1) is exempt from the + // *prefix* and nothing else: the reserved index.json listings are + // still names this format defines, so `_all_objects` passes and + // `_allObjects` does not. This used to be a two-entry allow-list, + // which exempted the whole spelling and so pinned nothing. + ident = strings.TrimPrefix(ident, PlatformPrefix) + assert.Regexp(t, snake, ident, "%s: %q is not snake_case", where, ident) + } + + for _, schema := range [][]byte{schemaJSON, indexSchemaJSON} { + var doc any + require.NoError(t, json.Unmarshal(schema, &doc)) + var walk func(node any) + walk = func(node any) { + switch n := node.(type) { + case map[string]any: + for key, v := range n { + switch key { + case "properties": + if props, ok := v.(map[string]any); ok { + for name, sub := range props { + check(t, "schema property", name) + walk(sub) + } + continue + } + case "enum": + if list, ok := v.([]any); ok { + for _, e := range list { + if s, isStr := e.(string); isStr { + check(t, "schema enum", s) + } + } + continue + } + } + walk(v) + } + case []any: + for _, v := range n { + walk(v) + } + } + } + walk(doc) + } + + for name, values := range map[string][]string{ + "kind": namesOf(kindNames.toName), + "textStyle": namesOf(textStyleNames.toName), + "fileType": namesOf(fileTypeNames.toName), + "processor": namesOf(processorNames.toName), + "widgetLayout": namesOf(widgetLayoutNames.toName), + "viewType": namesOf(viewTypeNames.toName), + "condition": namesOf(conditionNames.toName), + "date_preset": namesOf(datePresetNames.toName), + "aggregation": namesOf(aggregationNames.toName), + "format": namesOf(formatNames.toName), + "layout": namesOf(layoutNames.toName), + "card_style": namesOf(cardStyleNames.toName), + "card_size": namesOf(cardSizeNames.toName), + "list_size": namesOf(listSizeNames.toName), + "empty_placement": namesOf(emptyPlacementNames.toName), + } { + for _, v := range values { + check(t, name+" name table", v) + } + } +} + +func namesOf[T comparable](m map[T]string) []string { + out := make([]string, 0, len(m)) + for _, v := range m { + out = append(out, v) + } + return out +} diff --git a/pkg/lib/anyblockjson/flat_rules_test.go b/pkg/lib/anyblockjson/flat_rules_test.go new file mode 100644 index 0000000000..632ea917d2 --- /dev/null +++ b/pkg/lib/anyblockjson/flat_rules_test.go @@ -0,0 +1,391 @@ +package anyblockjson + +// flat_rules_test.go pins the specific flat-encoding rules the two invariants +// in flat_invariants_test.go rest on: leaf containment, the depth bound, the +// table-cell rules, the float-form indent (`1.0` read as 0, which passed +// validation and then imported as something else), and the property-message +// wording. Each was violated at least once before it was pinned. +// +// The invariants themselves live next door and are driven by hostile inputs; +// these are instances, expressed as the documents that broke them. + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// tableSnapshot builds root → table (one column, one row) whose single cell +// is cellContent with cellChildren nested under it. +func tableSnapshot(cellContent model.IsBlockContent, cellChildren ...*model.Block) *model.SmartBlockSnapshotBase { + cellChildIds := make([]string, 0, len(cellChildren)) + for _, c := range cellChildren { + cellChildIds = append(cellChildIds, c.Id) + } + blocks := []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"table1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "table1", ChildrenIds: []string{"tcols", "trows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "tcols", ChildrenIds: []string{"c1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: "trows", ChildrenIds: []string{"r1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + {Id: "c1", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + {Id: "r1", ChildrenIds: []string{"r1-c1"}, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + {Id: "r1-c1", ChildrenIds: cellChildIds, Content: cellContent}, + } + return &model.SmartBlockSnapshotBase{ + Blocks: append(blocks, cellChildren...), + Details: fields(map[string]*types.Value{"id": str("obj1")}), + } +} + +// innerTable returns a minimal valid table subtree rooted at id. +func innerTable(id string) []*model.Block { + return []*model.Block{ + {Id: id, ChildrenIds: []string{id + "cols", id + "rows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: id + "cols", Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: id + "rows", Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + } +} + +// Finding 1: cells cannot contain tables (the schema's recursion cut) — a +// snapshot violating that must fail Marshal loudly rather than produce a +// document Validate rejects. +func TestMarshal_TableInCellErrors(t *testing.T) { + t.Run("table among cell descendants", func(t *testing.T) { + inner := innerTable("inner") + snap := tableSnapshot( + &model.BlockContentOfText{Text: &model.BlockContentText{Text: "cell"}}, + inner..., + ) + _, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cells cannot contain tables") + assert.Contains(t, err.Error(), "r1-c1") + }) + + t.Run("cell block is a table", func(t *testing.T) { + inner := innerTable("inner") + snap := tableSnapshot(inner[0].Content, nil...) + // graft the inner table's wrappers under the cell id + snap.Blocks[len(snap.Blocks)-1].ChildrenIds = inner[0].ChildrenIds + snap.Blocks = append(snap.Blocks, inner[1], inner[2]) + _, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cells cannot contain tables") + }) +} + +// Finding 2: the schema's integer type admits integer-valued floats +// (1.0, 1e0); Validate and Unmarshal must agree on every such input, and +// V1/V2 must fire on float-form violations. +func TestIndent_FloatForms(t *testing.T) { + agree := func(t *testing.T, doc string) (valErr, unmErr error) { + valErr = Validate([]byte(doc)) + _, _, unmErr = Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + assert.Equal(t, valErr == nil, unmErr == nil, + "Validate (%v) and Unmarshal (%v) must agree", valErr, unmErr) + return valErr, unmErr + } + + t.Run("V2 fires on float indent under a leaf", func(t *testing.T) { + doc := `{"version": 2, "blocks": [{"type": "divider"}, {"indent": 1.0, "type": "paragraph", "text": "x"}]}` + valErr, _ := agree(t, doc) + require.Error(t, valErr) + assert.Contains(t, valErr.Error(), "divider blocks cannot have children") + }) + + t.Run("V1 fires on float jump", func(t *testing.T) { + doc := `{"version": 2, "blocks": [{"type": "paragraph", "text": "a"}, {"indent": 5.0, "type": "paragraph", "text": "b"}]}` + valErr, _ := agree(t, doc) + require.Error(t, valErr) + assert.Contains(t, valErr.Error(), "indent 5 follows indent 0") + }) + + t.Run("valid float forms import with the right depth and canonicalize", func(t *testing.T) { + for _, form := range []string{"1.0", "1e0"} { + doc := fmt.Sprintf(`{"version": 2, "blocks": [ + {"id": "a", "type": "toggle", "text": "t"}, + {"indent": %s, "id": "b", "type": "paragraph", "text": "x"} + ]}`, form) + require.NoError(t, Validate([]byte(doc)), form) + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err, form) + byId := map[string]*model.Block{} + for _, b := range snap.Blocks { + byId[b.Id] = b + } + assert.Equal(t, []string{"b"}, byId["a"].ChildrenIds, form) + out, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err, form) + assert.Contains(t, string(out), `"indent": 1`, form) + } + }) +} + +// Finding 3: a snapshot nested deeper than the F4 bound must fail Marshal +// loudly; at the bound it must marshal to a document Validate accepts. +func TestMarshal_DepthBound(t *testing.T) { + chain := func(depth int) *model.SmartBlockSnapshotBase { + blocks := []*model.Block{{Id: "obj1", ChildrenIds: []string{"n0"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}} + for i := 0; i <= depth; i++ { + b := textBlock(fmt.Sprintf("n%d", i), model.BlockContentText_Toggle, fmt.Sprintf("level %d", i)) + if i < depth { + b.ChildrenIds = []string{fmt.Sprintf("n%d", i+1)} + } + blocks = append(blocks, b) + } + return &model.SmartBlockSnapshotBase{Blocks: blocks} + } + + t.Run("depth 40 errors", func(t *testing.T) { + _, err := Marshal(model.SmartBlockType_Page, chain(40), Options{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "exceeds the format bound 32") + assert.Contains(t, err.Error(), "n33") + }) + + t.Run("depth 32 marshals and validates", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, chain(32), Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + }) +} + +// M3 (C11): with an OnWarning sink the read path degrades content the format +// can't represent (here: over-deep nesting) to a warning instead of failing +// the whole document, and the degraded output still validates. +func TestMarshal_OnWarningDegradesOverDeep(t *testing.T) { + deepChain := func(depth int) *model.SmartBlockSnapshotBase { + blocks := []*model.Block{{Id: "obj1", ChildrenIds: []string{"n0"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}} + for i := 0; i <= depth; i++ { + b := textBlock(fmt.Sprintf("n%d", i), model.BlockContentText_Toggle, fmt.Sprintf("level %d", i)) + if i < depth { + b.ChildrenIds = []string{fmt.Sprintf("n%d", i+1)} + } + blocks = append(blocks, b) + } + return &model.SmartBlockSnapshotBase{Blocks: blocks} + } + + // without a sink the over-deep document still fails loudly (canonical export) + _, err := Marshal(model.SmartBlockType_Page, deepChain(40), Options{}) + require.Error(t, err) + + // with a sink it degrades: clamp + warn, and the result validates + var warnings []Issue + data, err := Marshal(model.SmartBlockType_Page, deepChain(40), + Options{OnWarning: func(i Issue) { warnings = append(warnings, i) }}) + require.NoError(t, err, "the read must succeed with a warning sink") + require.NotEmpty(t, warnings, "the clamp emits a warning") + assert.Contains(t, warnings[0].Message, "clamped") + require.NoError(t, Validate(data), "clamped indents stay within the format bound") +} + +// Finding 4: unknown properties are rejected with a message naming the key; +// `children` additionally gets the flat-migration hint. +func TestValidate_UnknownPropertyMessages(t *testing.T) { + err := Validate([]byte(`{"version": 2, "blocks": [{"type": "toggle", "children": [{"type": "paragraph"}]}]}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), `/blocks/0/children: property "children" is not allowed — the flat format has no children; nest with indent instead`) + + err = Validate([]byte(`{"version": 2, "blocks": [{"type": "paragraph", "banana": 1}]}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), `/blocks/0/banana: property "banana" is not allowed`) +} + +// Finding 5: the F10 array-form cell round-trips byte-stable. +func TestRoundTrip_CellArrayForm(t *testing.T) { + snap := tableSnapshot( + &model.BlockContentOfText{Text: &model.BlockContentText{Style: model.BlockContentText_Toggle, Text: "cell"}}, + textBlock("child1", model.BlockContentText_Paragraph, "nested"), + ) + first, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(first)) + assert.Contains(t, string(first), `"cells": [`) + assert.Contains(t, string(first), `"indent": 1`) + + sbType, snap2, err := Unmarshal(first, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + byId := map[string]*model.Block{} + for _, b := range snap2.Blocks { + byId[b.Id] = b + } + cell := byId["r1-c1"] + require.NotNil(t, cell) + require.Len(t, cell.ChildrenIds, 1) + child := byId[cell.ChildrenIds[0]] + require.NotNil(t, child) + assert.Equal(t, "nested", child.Content.(*model.BlockContentOfText).Text.Text) + + second, err := Marshal(sbType, snap2, Options{}) + require.NoError(t, err) + assert.Equal(t, string(first), string(second)) +} + +// Finding 6: every V2 leaf type actually drops children on export, and the +// validation leaf set matches the export behavior (drift alarm). +func TestLeafTypes_ExportAgreement(t *testing.T) { + // factories keyed by JSON type; each returns the leaf block (children + // attached by the test) plus any extra blocks its subtree needs + leafFactories := map[string]func() (*model.Block, []*model.Block){ + "embed": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfLatex{Latex: &model.BlockContentLatex{Text: "x"}}}, nil + }, + "bookmark": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfBookmark{Bookmark: &model.BlockContentBookmark{Url: "https://x.io"}}}, nil + }, + "link": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{TargetBlockId: "obj"}}}, nil + }, + "divider": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfDiv{Div: &model.BlockContentDiv{}}}, nil + }, + "table": func() (*model.Block, []*model.Block) { + blocks := innerTable("leaf") + return blocks[0], blocks[1:] + }, + "property": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfRelation{Relation: &model.BlockContentRelation{Key: "name"}}}, nil + }, + "dataview": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{}}}, nil + }, + "icon": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfIcon{Icon: &model.BlockContentIcon{Name: "smile"}}}, nil + }, + "table_of_contents": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfTableOfContents{TableOfContents: &model.BlockContentTableOfContents{}}}, nil + }, + "featured_properties": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfFeaturedRelations{FeaturedRelations: &model.BlockContentFeaturedRelations{}}}, nil + }, + "chat": func() (*model.Block, []*model.Block) { + return &model.Block{Id: "leaf", Content: &model.BlockContentOfChat{Chat: &model.BlockContentChat{}}}, nil + }, + } + + // the validation leaf set must equal the factory set plus the equation + // input alias (which exports as embed) + wantLeafSet := map[string]bool{"equation": true} + for typ := range leafFactories { + wantLeafSet[typ] = true + } + assert.Equal(t, wantLeafSet, leafBlockTypes, + "leafBlockTypes drifted from the export withChildren=false set — update both together") + + // maxIndentInBlocks reads the deepest indent in an exported document + maxIndent := func(t *testing.T, data []byte) int { + var doc struct { + Blocks []struct { + Indent int `json:"indent"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + out := 0 + for _, b := range doc.Blocks { + if b.Indent > out { + out = b.Indent + } + } + return out + } + + // wrap each leaf in a toggle so structural top-level dropping (§7, + // featuredProperties) does not interfere; a surviving grandchild would + // appear at indent 2 + build := func(leaf *model.Block, extra []*model.Block) *model.SmartBlockSnapshotBase { + child := textBlock("grandchild", model.BlockContentText_Paragraph, "under leaf") + leaf.ChildrenIds = append(leaf.ChildrenIds, child.Id) + wrapper := textBlock("w1", model.BlockContentText_Toggle, "wrap") + wrapper.ChildrenIds = []string{leaf.Id} + blocks := []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"w1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + wrapper, leaf, child, + } + return &model.SmartBlockSnapshotBase{Blocks: append(blocks, extra...)} + } + + for typ, factory := range leafFactories { + t.Run(typ, func(t *testing.T) { + leaf, extra := factory() + data, err := Marshal(model.SmartBlockType_Page, build(leaf, extra), Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.Equal(t, 1, maxIndent(t, data), "children of a %s block must be dropped on export", typ) + }) + } + + t.Run("control: paragraph keeps children", func(t *testing.T) { + leaf := textBlock("leaf", model.BlockContentText_Paragraph, "parent") + data, err := Marshal(model.SmartBlockType_Page, build(leaf, nil), Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.Equal(t, 2, maxIndent(t, data)) + }) +} + +// Finding 7: lenient clamps land on the containment checks — a clamped +// block still errors under a leaf parent or a row (V2/V3 evaluate on the +// clamped indents). +func TestNormalizeIndent_ContainmentStillErrors(t *testing.T) { + t.Run("clamped under a leaf", func(t *testing.T) { + doc := `{"version": 2, "blocks": [{"type": "divider"}, {"indent": 5, "type": "paragraph", "text": "x"}]}` + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), NormalizeIndent: true}) + require.Error(t, err) + assert.Contains(t, err.Error(), "divider blocks cannot have children") + }) + + t.Run("clamped non-column under a row", func(t *testing.T) { + doc := `{"version": 2, "blocks": [{"type": "row"}, {"indent": 7, "type": "paragraph", "text": "x"}]}` + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), NormalizeIndent: true}) + require.Error(t, err) + assert.Contains(t, err.Error(), "a row block can only contain column blocks") + }) +} + +// Finding 8: the prefix property holds at real-world depth (≥ 6), not just +// the depth-2 rich fixture. +func TestValidate_PrefixProperty_Deep(t *testing.T) { + var parts []string + parts = append(parts, `{"id": "d0", "type": "toggle", "text": "level 0"}`) + for d := 1; d <= 8; d++ { + parts = append(parts, fmt.Sprintf(`{"indent": %d, "id": "d%d", "type": "toggle", "text": "level %d"}`, d, d, d)) + } + parts = append(parts, `{"id": "top", "type": "paragraph", "text": "back to top"}`) + doc := `{"version": 2, "blocks": [` + strings.Join(parts, ",") + `]}` + + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + canonical, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err) + + var parsed struct { + Blocks []json.RawMessage `json:"blocks"` + } + require.NoError(t, json.Unmarshal(canonical, &parsed)) + require.Len(t, parsed.Blocks, 10) + for n := 0; n <= len(parsed.Blocks); n++ { + blockParts := make([]string, 0, n) + for _, b := range parsed.Blocks[:n] { + blockParts = append(blockParts, string(b)) + } + prefix := `{"version": 2, "blocks": [` + strings.Join(blockParts, ",") + `]}` + require.NoError(t, Validate([]byte(prefix)), "prefix of %d blocks", n) + } +} diff --git a/pkg/lib/anyblockjson/fold_test.go b/pkg/lib/anyblockjson/fold_test.go new file mode 100644 index 0000000000..1d9938cd98 --- /dev/null +++ b/pkg/lib/anyblockjson/fold_test.go @@ -0,0 +1,401 @@ +package anyblockjson + +// fold_test.go — the participant fold (§9): `_participant__` +// exports as the bare identity when Options.SpaceId names the space, and a +// bare identity imports back as this space's participant id. + +import ( + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The real shapes, from the account that produced the corpus: a checksummed +// 48-character identity and the 135-character composite built from it. +const ( + foldSpaceId = "bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq.30afw2fe3tvff" + foldIdentity = "AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA" + foldComposite = "_participant_bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq_30afw2fe3tvff_" + foldIdentity + + // the same member seen from another space: NOT this run's, so it must + // pass through whole + foreignComposite = "_participant_bafyreiforeignspaceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_2xyz_" + foldIdentity +) + +// foldSnapshot puts a participant reference in every §9 slot the census +// found them in: object-format property values, items, a block object_id, a +// filter value, an object order — plus a foreign-space composite as the +// control. +func foldSnapshot() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + { + Id: "bafyreifoldroot", + ChildrenIds: []string{"lnk", "dv1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }, + {Id: "lnk", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: foldComposite, + }}}, + {Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{{ + Id: "view1", Name: "All", + Filters: []*model.BlockContentDataviewFilter{{ + Id: "f1", + RelationKey: "assignee", + Condition: model.BlockContentDataviewFilter_In, + Value: strList(foldComposite), + }}, + }}, + ObjectOrders: []*model.BlockContentDataviewObjectOrder{{ + ViewId: "view1", + ObjectIds: []string{foldComposite}, + }}, + }}}, + }, + Details: fields(map[string]*types.Value{ + "id": str("bafyreifoldroot"), + "name": str("Fold host"), + "assignee": strList(foldComposite), + "owner": strList(foldComposite, foreignComposite), + }), + Collections: fields(map[string]*types.Value{ + storeKeyItems: strList(foldComposite), + }), + } +} + +// foldOptions resolves the space-minted `owner` key to the objects format — +// the corpus's heaviest participant slot is exactly such a custom property — +// and carries the space id that arms the fold. +func foldOptions() Options { + o := refOptions() + prev := o.ResolveFormat + o.ResolveFormat = func(key domain.RelationKey) (model.RelationFormat, bool) { + if key == "owner" { + return model.RelationFormat_object, true + } + return prev(key) + } + o.SpaceId = foldSpaceId + return o +} + +// Every slot folds, the foreign-space composite does not, and the output +// still validates (I1). The trigger is the VALUE's shape, never the property +// name: `owner` is a space-minted custom property the bundle knows nothing +// about. +// +// How this can fail: unhook the fold from a slot and the 135-character +// composite shows up there; fold on the property name instead of the value +// and the custom `owner` slot keeps the composite; drop the same-space gate +// and the foreign composite folds too (silent re-homing on import). +func TestFold_ParticipantRefsFoldOnEverySlot(t *testing.T) { + // when + data, err := Marshal(model.SmartBlockType_Page, foldSnapshot(), foldOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (§11 I1)") + doc := string(data) + + // then + assert.NotContains(t, doc, foldComposite, "no slot keeps this space's composite id") + assert.Contains(t, doc, `"`+foldIdentity+`"`, "the bare identity stands in") + assert.Contains(t, doc, foreignComposite, "a foreign space's composite passes through whole") +} + +// The participant document's own envelope id folds too — a reader must be +// able to textually join a folded reference to the participant document it +// points at (§9) — and import rebuilds the composite as the object id. +// +// How this can fail: skip the fold on the `id` slot and the envelope keeps +// 135 characters; skip the unfold and the imported snapshot's id detail (and +// root block id) hold a bare identity where every store write expects the +// composite — the silent corruption Options.SpaceId exists to prevent. +func TestFold_ParticipantOwnEnvelopeId(t *testing.T) { + // given a participant document + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: foldComposite, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(map[string]*types.Value{ + "id": str(foldComposite), + "name": str("Roma Kha"), + }), + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, foldOptions()) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"id": "`+foldIdentity+`"`) + assert.NotContains(t, string(data), foldComposite) + + // and back + _, imported, err := Unmarshal(data, foldOptions()) + require.NoError(t, err) + assert.Equal(t, foldComposite, + imported.GetDetails().GetFields()["id"].GetStringValue(), + "import rebuilds the composite as the object id") + assert.Equal(t, foldComposite, imported.Blocks[0].Id, + "and as the root block id") +} + +// With no SpaceId the fold is OFF in both directions: the composite passes +// through export whole, and a bare identity is left alone on import rather +// than guessed into some space. +// +// How this can fail: fold on export regardless of SpaceId and the first +// assertion finds the identity; unfold against an empty space and the second +// case builds `_participant__` garbage. +func TestFold_DisabledWithoutSpaceId(t *testing.T) { + // given + opts := foldOptions() + opts.SpaceId = "" + + // when + data, err := Marshal(model.SmartBlockType_Page, foldSnapshot(), opts) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), foldComposite, "the composite survives whole") + assert.NotContains(t, string(data), `"`+foldIdentity+`"`) + + // and a bare identity on import stays bare + doc := `{"version": 2, "properties": {"assignee": ["` + foldIdentity + `"]}}` + _, snap, err := Unmarshal([]byte(doc), opts) + require.NoError(t, err) + assert.Equal(t, []string{foldIdentity}, + valueStringList(snap.GetDetails().GetFields()["assignee"])) +} + +// Import rebuilds the composite from a bare identity — with or without the +// informative suffix — and leaves near-misses alone: the checksum is the +// classifier, so a 48-character base58 string that is not an identity does +// not unfold. +// +// How this can fail: skip the unfold and the bare identity lands in the +// snapshot (the corruption this change exists to fix); unfold by length or +// charset alone and the corrupted-checksum control rebuilds a participant id +// for a string that names nobody. +func TestFold_ImportRebuildsTheComposite(t *testing.T) { + // a plausible-looking non-identity: same charset and length, bad checksum + notAnIdentity := foldIdentity[:len(foldIdentity)-4] + "aaaa" + + // given + doc := `{"version": 2, "properties": { + "assignee": ["` + foldIdentity + `#roma_kha", "` + notAnIdentity + `"]}}` + + // when + _, snap, err := Unmarshal([]byte(doc), foldOptions()) + + // then + require.NoError(t, err) + assert.Equal(t, []string{foldComposite, notAnIdentity}, + valueStringList(snap.GetDetails().GetFields()["assignee"]), + "the identity unfolds (suffix trimmed first); the near-miss passes verbatim") +} + +// The round trip is byte-stable and snapshot-lossless: fold on export, +// rebuild on import, fold again identically. +// +// How this can fail: any asymmetry between the two halves — a slot that +// folds but does not unfold (the details stop matching), or unfolds into a +// different spelling (the bytes stop matching). +func TestFold_RoundTripLossless(t *testing.T) { + // given + opts := foldOptions() + snap := foldSnapshot() + + // when + first, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + sbType, imported, err := Unmarshal(first, opts) + require.NoError(t, err) + second, err := Marshal(sbType, imported, opts) + require.NoError(t, err) + + // then + assert.Equal(t, string(first), string(second), "byte-stable (§11)") + for _, key := range []string{"assignee", "owner"} { + assert.Equal(t, + valueStringList(snap.Details.Fields[key]), + valueStringList(imported.GetDetails().GetFields()[key]), + "detail %q holds the original composites again", key) + } + assert.Equal(t, + valueStringList(snap.Collections.Fields[storeKeyItems]), + valueStringList(imported.GetCollections().GetFields()[storeKeyItems]), + "items too") +} + +// Fold and suffix compose: with RefNames on and a resolver that knows the +// COMPOSITE id (the id the space indexes), the document spells +// `#`. +// +// How this can fail: ask the resolver about the folded identity instead of +// the stored composite and no name resolves, so the suffix vanishes. +func TestFold_ComposesWithTheNameSuffix(t *testing.T) { + // given + opts := foldOptions() + opts.RefNames = true + opts.ResolveObjectNames = testObjectNames{foldComposite: "Roma Kha"} + + // when + data, err := Marshal(model.SmartBlockType_Page, foldSnapshot(), opts) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"`+foldIdentity+`#roma_kha"`, + "resolvable AND readable: the folded identity plus the informative name") + assert.True(t, strings.Contains(string(data), foldIdentity)) +} + +// A composite built from a BLANK identity addresses nobody, and 9,103 of the +// corpus's 37,429 objects carry one. It is not an identity, so it does not +// fold — and the guard that says so is the ONLY one that does: the +// round-trip recheck passes, because NewParticipantId(space, "") rebuilds +// that exact string. Without the classifier the value would fold to the +// empty string and the reference would be deleted outright. +// +// Attribution has its own guard for this shape (attribution_test.go), which +// is why it went uncovered here: the ordinary reference slots share none of +// that path. +// +// How this can fail: drop !isAccountIdentity from foldParticipantRef and +// both slots below lose their value entirely. +func TestFold_AnEmptyIdentityCompositeIsNotAnIdentity(t *testing.T) { + // given + empty := domain.NewParticipantId(foldSpaceId, "") + require.Equal(t, empty, domain.NewParticipantId(foldSpaceId, ""), + "the recheck cannot refuse this shape: it rebuilds byte-identically") + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: "bafyreifoldroot", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(map[string]*types.Value{ + "id": str("bafyreifoldroot"), + "assignee": strList(empty), + }), + Collections: fields(map[string]*types.Value{storeKeyItems: strList(empty)}), + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, foldOptions()) + require.NoError(t, err) + _, back, err := Unmarshal(data, foldOptions()) + require.NoError(t, err) + + // then + assert.Equal(t, []string{empty}, valueStringList(back.GetDetails().GetFields()["assignee"]), + "a value that addresses nobody still may not be deleted") + assert.Equal(t, []string{empty}, + valueStringList(back.GetCollections().GetFields()[storeKeyItems])) +} + +// The fold has two gates — the space embedded in the id must be this run's, +// and the composite must rebuild byte-identically — and they are NOT +// redundant, though on every real space id either alone would do. Each input +// below is refused by exactly one of them, so neither can be deleted as +// "already covered by the other". Two refactors, each green on its own, is +// how both would otherwise go. +// +// How this can fail: drop the spaceId != o.SpaceId gate and the first case +// folds; drop the NewParticipantId recheck and the second does. Either fold +// re-homes a member onto a space that is not theirs. +func TestFold_NeitherGateIsRedundant(t *testing.T) { + for name, tc := range map[string]struct{ spaceId, stored string }{ + // ParseParticipantId always answers parts[2] + "." + parts[3], so a + // space id spelled with `_` and no `.` parses back as a DIFFERENT + // space — while NewParticipantId, which replaces only the first `.`, + // rebuilds this id exactly. Only the same-space gate refuses. + "only the same-space gate refuses": { + spaceId: "a_b", stored: "_participant_a_b_" + foldIdentity, + }, + // Here the parse answers this run's own space id — the gate is + // satisfied — but NewParticipantId puts the `_` in a different place + // than the stored id has it. Only the recheck refuses. + "only the round-trip recheck refuses": { + spaceId: "a.b.c", stored: "_participant_a.b_c_" + foldIdentity, + }, + } { + t.Run(name, func(t *testing.T) { + // given + o := foldOptions() + o.SpaceId = tc.spaceId + + // then + assert.Equal(t, tc.stored, o.foldParticipantRef(tc.stored), + "folding this would re-home the member on import") + }) + } +} + +// A resolver that answers with a name the suffix grammar reduces to nothing +// — an emoji-only title, which real objects have — leaves the reference +// BARE. Never a dangling `#`: that value reads back as the id it came from +// only because splitRefName refuses to split at index 0, and a document full +// of them is unreadable besides. +// +// How this can fail: append the separator before checking the normalized +// label and every emoji-named reference gains a trailing `#`. +func TestRefNames_ANameThatNormalizesToNothingLeavesTheRefBare(t *testing.T) { + // given + opts := refOptions() + opts.RefNames = true + opts.ResolveObjectNames = testObjectNames{"bafyreiassigned": "🎉🎉🎉"} + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: "bafyreirefroot", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(map[string]*types.Value{ + "id": str("bafyreirefroot"), + "assignee": strList("bafyreiassigned"), + }), + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"bafyreiassigned"`) + assert.NotContains(t, string(data), "#", "an empty label is no label, not an empty suffix") +} + +// A reader that names no space cannot rebuild a folded participant id, and +// says so once for the document (§9). It may not refuse — Validate never +// sees Options, so a refusal here would leave the two surfaces disagreeing +// about one document (§12 I2) — so the warning is the whole of the defence, +// and the tool with no target space (cmd/anyblockconvert) is where it lands. +// +// How this can fail: drop the SpaceId check in importer.objectRef and a +// bare identity is stored where a composite belongs, in silence. +func TestFold_AReaderWithNoSpaceSaysSoInsteadOfCorrupting(t *testing.T) { + // given a document written by a folded export + data, err := Marshal(model.SmartBlockType_Page, foldSnapshot(), foldOptions()) + require.NoError(t, err) + + // when it is read by a reader that names no space + reader := refOptions() + var warned []Issue + reader.OnWarning = func(i Issue) { warned = append(warned, i) } + _, back, err := Unmarshal(data, reader) + require.NoError(t, err) + + // then + require.Len(t, warned, 1, "one line for the document, not one per reference") + assert.Contains(t, warned[0].Message, "Options.SpaceId names no space") + assert.Equal(t, []string{foldIdentity, foreignComposite}, + valueStringList(back.GetDetails().GetFields()["owner"]), + "the identity is stored as it stands — the warning is what makes that visible") +} diff --git a/pkg/lib/anyblockjson/format_test.go b/pkg/lib/anyblockjson/format_test.go new file mode 100644 index 0000000000..950b729a7f --- /dev/null +++ b/pkg/lib/anyblockjson/format_test.go @@ -0,0 +1,123 @@ +package anyblockjson + +// There is one text format on the wire, "text" (§3). The stored +// longtext/shorttext split survives it because import resolves "text" +// against the key's existing format. + +import ( + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func dataviewSnapshot(links ...*model.RelationLink) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "root", ChildrenIds: []string{"dataview"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dataview", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + RelationLinks: links, + Views: []*model.BlockContentDataviewView{{Id: "v1", Name: "All"}}, + }}}, + }, + Details: fields(map[string]*types.Value{"id": str("root")}), + } +} + +func linkFormats(t *testing.T, snap *model.SmartBlockSnapshotBase) map[string]model.RelationFormat { + t.Helper() + for _, b := range snap.Blocks { + if dv := b.GetDataview(); dv != nil { + out := map[string]model.RelationFormat{} + for _, rl := range dv.RelationLinks { + out[rl.Key] = rl.Format + } + return out + } + } + t.Fatal("no dataview block") + return nil +} + +// Both stored text formats serialize to the single name "text". +func TestExport_TextFormatsCollapse(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, dataviewSnapshot( + &model.RelationLink{Key: "name", Format: model.RelationFormat_shorttext}, + &model.RelationLink{Key: "description", Format: model.RelationFormat_longtext}, + ), testOptions()) + require.NoError(t, err) + + assert.NotContains(t, string(data), "shortText") + assert.Contains(t, string(data), `"property": "Name"`) + assert.Equal(t, 2, strings.Count(string(data), `"format": "text"`)) +} + +// The collapse is not lossy: a key already known to be shorttext gets its +// stored format back, so proto -> json -> proto is an identity for it. +func TestRoundtrip_ShortTextSurvivesCollapse(t *testing.T) { + t.Run("bundled key", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, dataviewSnapshot( + &model.RelationLink{Key: "name", Format: model.RelationFormat_shorttext}, + &model.RelationLink{Key: "description", Format: model.RelationFormat_longtext}, + ), testOptions()) + require.NoError(t, err) + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + got := linkFormats(t, back) + assert.Equal(t, model.RelationFormat_shorttext, got["name"], "bundled name is shorttext") + assert.Equal(t, model.RelationFormat_longtext, got["description"]) + }) + + // same rule via the wiring's resolver, for non-bundled keys + t.Run("resolver key", func(t *testing.T) { + doc := `{"version": 2, "id": "root", "blocks": [{"type": "dataview", + "properties": [{"property": "legacyShort", "format": "text"}, + {"property": "plainNote", "format": "text"}], + "views": [{"name": "All"}]}]}` + opts := Options{GenerateId: seqIds("g")} + opts.ResolveFormat = func(key domain.RelationKey) (model.RelationFormat, bool) { + if key == "legacyShort" { + return model.RelationFormat_shorttext, true + } + return 0, false + } + _, snap, err := Unmarshal([]byte(doc), opts) + require.NoError(t, err) + + got := linkFormats(t, snap) + assert.Equal(t, model.RelationFormat_shorttext, got["legacyShort"]) + assert.Equal(t, model.RelationFormat_longtext, got["plainNote"], "unknown key is a new text property") + }) + + // a resolver disagreeing about a *non*-text format must not win: the + // document stays authoritative for every name that is unambiguous. + t.Run("only text defers to the key", func(t *testing.T) { + doc := `{"version": 2, "id": "root", "blocks": [{"type": "dataview", + "properties": [{"property": "customDate", "format": "number"}], + "views": [{"name": "All"}]}]}` + _, snap, err := Unmarshal([]byte(doc), Options{ + GenerateId: seqIds("g"), + ResolveFormat: testFormatResolver, // says customDate is a date + }) + require.NoError(t, err) + assert.Equal(t, model.RelationFormat_number, linkFormats(t, snap)["customDate"]) + }) +} + +// shortText is gone from the vocabulary, not merely unused. +func TestValidate_ShortTextRejected(t *testing.T) { + doc := `{"version": 2, "id": "root", "blocks": [{"type": "dataview", + "properties": [{"property": "name", "format": "shortText"}], + "views": [{"name": "All"}]}]}` + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, err) + assert.Contains(t, err.Error(), "format") +} diff --git a/pkg/lib/anyblockjson/fragment.go b/pkg/lib/anyblockjson/fragment.go new file mode 100644 index 0000000000..af42ccead0 --- /dev/null +++ b/pkg/lib/anyblockjson/fragment.go @@ -0,0 +1,256 @@ +package anyblockjson + +// fragment.go exposes the conversion machinery at fragment granularity — +// single blocks, flat runs, one property value, the inline codec — for +// wiring that edits a live document op-by-op instead of round-tripping the +// whole document (the API's PATCH surface). +// +// Fragment validation reuses the document validation wholesale: a run is +// wrapped into a minimal synthetic document and validated there, so V1 +// monotonicity and the §5 per-type shape checks apply exactly as on a whole +// document. Two fragment-specific rules on top: +// - structural block types (title/description/featuredProperties, §7) are +// rejected explicitly — a fragment has no document to absorb them into, +// and the import path's silent top-level absorption must not fire; +// - no primary-dataview pinning (§7): a fragment never names the +// document's own dataview, so no block is renamed to the "dataview" id. + +import ( + "encoding/json" + "fmt" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// fragmentStructuralTypes are the §7 structural block types a fragment must +// not carry (the whole-document import absorbs or drops them silently; a +// fragment rejects them loudly instead). +var fragmentStructuralTypes = map[string]bool{ + "title": true, "description": true, "featured_properties": true, +} + +// validateFragmentRun wraps the run in a minimal synthetic page document and +// runs the document validation, then applies the fragment-specific +// structural-type rejection. Issue paths are /blocks/i/… — run-relative. +func validateFragmentRun(run []json.RawMessage, opts Options) ([]*jsonBlock, error) { + payload, err := json.Marshal(map[string]any{ + "version": FormatVersion, + "type": "page", + "blocks": run, + }) + if err != nil { + return nil, fmt.Errorf("build synthetic fragment document: %w", err) + } + if _, err := validateToDoc(payload, opts.NormalizeIndent, opts.OnWarning); err != nil { + return nil, err + } + jbs := make([]*jsonBlock, 0, len(run)) + var issues []Issue + for i, raw := range run { + var jb jsonBlock + if err := json.Unmarshal(raw, &jb); err != nil { + return nil, fmt.Errorf("decode block %d: %w", i, err) + } + if fragmentStructuralTypes[jb.Type] { + issues = append(issues, Issue{ + Path: fmt.Sprintf("/blocks/%d/type", i), + Message: fmt.Sprintf("%q is a structural block — the editor owns it; it cannot appear in a fragment", jb.Type), + }) + } + jbs = append(jbs, &jb) + } + if len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + return jbs, nil +} + +// UnmarshalBlocks converts a flat run of AnyBlock JSON blocks (§4) into +// model blocks. Indents are run-relative: 0 is the run's top level, and the +// run must obey V1 monotonicity internally (the first block at 0, each block +// at most one level deeper than its predecessor). The returned blocks slice +// holds every created block — for tables that includes the internal subtree +// (§6.1) — with the ChildrenIds graph already wired; topIds names the run's +// top-level blocks in order, ready for a state splice. Blocks without ids +// get generated ones (Options.GenerateId). Errors wrap *ValidationError +// with run-relative /blocks/i/… paths. +func UnmarshalBlocks(run []json.RawMessage, opts Options) (blocks []*model.Block, topIds []string, err error) { + jbs, err := validateFragmentRun(run, opts) + if err != nil { + return nil, nil, err + } + imp := &importer{opts: opts, doc: opts.fragmentDoc()} + root := &model.Block{} + // §7a: the write path lifts too, or an API caller that pasted a `group` + // mints a Layout_Div straight into a live object — one no read will ever + // show and normalization never removes while it has children + liftedJbs, liftedIndents := liftTransparentContainers(jbs, imp.blockIndents(jbs, -1)) + blocks, err = imp.flatSubtree(liftedJbs, liftedIndents, root, -1) + if err != nil { + return nil, nil, fmt.Errorf("build fragment blocks: %w", err) + } + return blocks, root.ChildrenIds, nil +} + +// UnmarshalBlock converts one AnyBlock JSON block object into its model +// block(s): the addressed block first, followed by any internal blocks it +// owns (the table subtree, §6.1). forcedId, when non-empty, overrides the +// payload id — the edit path uses it to keep a replaced block's identity. +// The block is validated like a one-element run (so §5 shape checks apply); +// structural types are rejected. An indent field, if present, must be 0 +// (V1 on the synthetic run). +func UnmarshalBlock(raw json.RawMessage, forcedId string, opts Options) ([]*model.Block, error) { + jbs, err := validateFragmentRun([]json.RawMessage{raw}, opts) + if err != nil { + return nil, err + } + // §7a: this entry point's contract is exactly one block, and a + // transparent container is not one. Returning zero blocks would leave + // the caller's edit silently unapplied — a replaceBlock that replaced + // nothing — so it is named as the error it is. + if transparentBlockTypes[jbs[0].Type] { + return nil, &ValidationError{Issues: []Issue{{ + Path: "/blocks/0/type", + Message: fmt.Sprintf("%q is a transparent container — it contributes no block of its own, "+ + "so it cannot be the one block this call addresses", jbs[0].Type), + }}} + } + imp := &importer{opts: opts, doc: opts.fragmentDoc()} + blocks, err := imp.blockFromJSON(jbs[0], forcedId) + if err != nil { + return nil, fmt.Errorf("build fragment block: %w", err) + } + return blocks, nil +} + +// UnmarshalPropertyValue decodes one property value per its resolved §3 +// format rules (dates parse, select option names resolve/create through +// Options.ResolveOptions, object/file ids pass through, scalars of +// list-shaped formats wrap into lists). It is the import twin of +// MarshalPropertyValue. A nil v yields an explicit null value (presence is +// preserved, §3). +// +// `key` is a STORED key here, not a spelling — a value-level caller holds +// the key it is writing — so the property legend has nothing to do. The +// OPTION legend does: hand this call the {option name: option id} map for +// this key through Options.Legend.OptionIds, and a select value resolves by +// id first (§3 step 1) instead of by name. Without it a name shared by two +// options lands on whichever answers first, and an option renamed since the +// value was written mints a second option under the stale name — the two +// losses §9a exists to close, which this entry point had no way to avoid. +func UnmarshalPropertyValue(key string, v any, opts Options) *types.Value { + // A key the whole-document import DROPS returns nothing here too, or the + // two doors disagree about the same key. It matters most for attribution + // (§3): `creator` and `lastModifiedBy` are DERIVED — the store sets them, + // a writer never does — so a caller round-tripping one value through this + // pair would hand back a value that can only be discarded on the way in. + // MarshalPropertyValue writes the member as `#` for a reader to + // display; this refuses to read it back, and the asymmetry is the point. + if isDroppedOnImport(key) { + return nil + } + imp := &importer{opts: opts, doc: opts.fragmentDoc()} + return imp.propertyValue(key, key, v) +} + +// MarshalBlockSubtree serializes one block subtree into a fragment envelope +// (§4): `blocks` is the flat run — subtree[0] is the root, emitted at indent +// 0, and the remaining entries back the root's ChildrenIds graph (entries not +// reachable from the root are ignored, ids the slice does not carry are +// skipped, the same leniency as a whole-document export; tables need their +// internal blocks in the slice to render, §6.1) — beside the legends those +// blocks owe, in the envelope's own member order: +// +// {"property_internal_keys": {…}, "type_internal_keys": {…}, "option_ids": {…}, "blocks": […]} +// +// **The legends are why this is an object rather than the bare array it used +// to be.** A block run names properties at seven slots and options at two, +// and it names them in the DOCUMENT's spelling — which is the writer's +// vocabulary, not the reader's. The exporter computed all three legends here +// all along and discarded them at the return, so a `property` block came back +// as `{"property": "priority"}` with nothing saying which relation `priority` is, +// where the same block inside a whole document carries +// `property_internal_keys: {"priority": "6a32d485…"}`. A reader resolved it through +// its own table, which is precisely the misresolution §3 wrote the legend to +// prevent. Feed the three maps to the reading side through Options.Legend. +// +// **OmitIds and the compaction flags are refused, not ignored.** This surface +// exists for wiring that edits a live document op-by-op, and both destroy the +// addresses that wiring runs on: OmitIds drops every block id, the view id +// and the filter id, so the run says what to write but not where; the block +// relabeling rewrites doc-local ids to short suffixes that are meaningful +// only inside the emitted run and are not the object's ids at all. Silently +// honouring either produced a fragment that reads correctly and cannot be +// applied. A caller that wants an id-less or relabeled rendering wants +// Marshal on the whole document. +func MarshalBlockSubtree(subtree []*model.Block, opts Options) (json.RawMessage, error) { + if len(subtree) == 0 || subtree[0] == nil || subtree[0].Id == "" { + return nil, fmt.Errorf("empty subtree") + } + if opts.OmitIds { + return nil, fmt.Errorf("OmitIds is not available on a block subtree: " + + "a fragment is addressed by the ids it carries") + } + if opts.compactBlockLabels() { + return nil, fmt.Errorf("block-label compaction is not available on a block subtree: " + + "the short labels are local to the emitted run, not the object's ids") + } + e := &exporter{ + opts: opts, + snapshot: &model.SmartBlockSnapshotBase{Blocks: subtree}, + blocks: map[string]*model.Block{}, + visited: map[string]bool{}, + } + e.indexBlocks() + // the fragment's root is the caller's, not the one indexBlocks infers from + // an id-less snapshot: the emit below starts at subtree[0], so that is the + // entry point the id reservations have to be reachable from (§4). + e.rootId = subtree[0].Id + var out []any + // topLevel=false: a fragment caller addresses the blocks explicitly, so + // even a structural root renders rather than silently vanishing + if err := e.appendBlocksFlat(&out, []string{subtree[0].Id}, 0, false); err != nil { + return nil, fmt.Errorf("marshal block subtree: %w", err) + } + // the legends AFTER the emit: every key slot has claimed its term by now, + // which is the same ordering buildDoc relies on (§9a) + env := &omap{} + if m := e.buildPropertyKeys(); m != nil { + env.set(memberPropertyInternalKeys, m) + } + // The type half is UNREACHABLE from every fragment slot today: typeSlug is + // called only from envelopeTypeTerms and buildTypeProperties, and neither + // is on this path, so no subtree can owe a type_internal_keys line. Kept anyway, + // because the cost is three lines and the failure mode of removing it is a + // fragment that silently omits a legend the day a block slot starts + // carrying a type term. A probe confirms the branch never fires. + if m := e.buildTypeKeys(); m != nil { + env.set(memberTypeInternalKeys, m) + } + if m := e.buildOptionIds(); m != nil { + env.set("option_ids", m) + } + env.set("blocks", out) + data, err := json.Marshal(env) + if err != nil { + return nil, fmt.Errorf("encode block subtree: %w", err) + } + return data, nil +} + +// ParseInlineText parses §8 inline Markdown into plain text and marks — the +// single-field import codec. Mention/object mark params stay as written — +// an object reference is never compacted, so there is nothing to resolve +// them through (§9a). +func ParseInlineText(md string) (string, []*model.BlockContentTextMark, error) { + return parseInline(md) +} + +// RenderInlineText renders plain text and marks back into §8 inline +// Markdown — the single-field export codec, the exact inverse used by +// Marshal for every text-bearing block. +func RenderInlineText(text string, marks []*model.BlockContentTextMark) string { + return renderInline(text, marks) +} diff --git a/pkg/lib/anyblockjson/fragment_test.go b/pkg/lib/anyblockjson/fragment_test.go new file mode 100644 index 0000000000..78313815a3 --- /dev/null +++ b/pkg/lib/anyblockjson/fragment_test.go @@ -0,0 +1,321 @@ +package anyblockjson + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func rawRun(blocks ...string) []json.RawMessage { + out := make([]json.RawMessage, len(blocks)) + for i, b := range blocks { + out[i] = json.RawMessage(b) + } + return out +} + +func TestUnmarshalBlocks(t *testing.T) { + t.Run("run with relative indents builds the subtree", func(t *testing.T) { + // given + run := rawRun( + `{"id":"a1","type":"paragraph","text":"parent"}`, + `{"indent":1,"id":"a2","type":"paragraph","text":"child"}`, + `{"id":"a3","type":"quote","text":"sibling"}`, + ) + + // when + blocks, topIds, err := UnmarshalBlocks(run, Options{}) + + // then + require.NoError(t, err) + assert.Equal(t, []string{"a1", "a3"}, topIds) + require.Len(t, blocks, 3) + assert.Equal(t, []string{"a2"}, blocks[0].ChildrenIds, "indent 1 nests under the predecessor") + }) + + t.Run("missing ids are generated", func(t *testing.T) { + n := 0 + opts := Options{GenerateId: func() string { n++; return "gen" + string(rune('0'+n)) }} + + blocks, topIds, err := UnmarshalBlocks(rawRun(`{"type":"paragraph","text":"x"}`), opts) + + require.NoError(t, err) + require.Len(t, blocks, 1) + assert.Equal(t, "gen1", blocks[0].Id) + assert.Equal(t, []string{"gen1"}, topIds) + }) + + t.Run("V1 monotonicity applies to the run", func(t *testing.T) { + _, _, err := UnmarshalBlocks(rawRun( + `{"type":"paragraph","text":"a"}`, + `{"indent":2,"type":"paragraph","text":"b"}`, + ), Options{}) + + var verr *ValidationError + require.ErrorAs(t, err, &verr) + }) + + t.Run("structural block types are rejected explicitly", func(t *testing.T) { + for typ, body := range map[string]string{ + "title": `{"type":"title","text":"x"}`, + "description": `{"type":"description","text":"x"}`, + "featured_properties": `{"type":"featured_properties"}`, + } { + _, _, err := UnmarshalBlocks(rawRun(body), Options{}) + + var verr *ValidationError + require.ErrorAs(t, err, &verr, typ) + require.Len(t, verr.Issues, 1) + assert.Equal(t, "/blocks/0/type", verr.Issues[0].Path) + assert.Contains(t, verr.Issues[0].Message, "structural block") + } + }) + + t.Run("no primary-dataview pinning in fragments", func(t *testing.T) { + // a whole-document import would rename this block to the fixed + // "dataview" id (§7); a fragment must not + blocks, _, err := UnmarshalBlocks(rawRun(`{"type":"dataview","views":[{"name":"All"}]}`), + Options{GenerateId: func() string { return "genDataview1" }}) + + require.NoError(t, err) + require.NotEmpty(t, blocks) + assert.Equal(t, "genDataview1", blocks[0].Id, "the fragment dataview keeps a generated id") + }) + + t.Run("table run carries its internal subtree", func(t *testing.T) { + blocks, topIds, err := UnmarshalBlocks(rawRun( + `{"id":"tbl1","type":"table","columns":[{"id":"colA"}],"rows":[{"id":"rowA","cells":["hi"]}]}`, + ), Options{}) + + require.NoError(t, err) + assert.Equal(t, []string{"tbl1"}, topIds) + ids := make(map[string]bool, len(blocks)) + for _, b := range blocks { + ids[b.Id] = true + } + assert.True(t, ids["colA"], "column block present") + assert.True(t, ids["rowA"], "row block present") + assert.True(t, ids["rowA-colA"], "derived cell id present") + }) +} + +func TestUnmarshalBlock(t *testing.T) { + t.Run("forcedId overrides the payload id", func(t *testing.T) { + blocks, err := UnmarshalBlock(json.RawMessage(`{"type":"checkbox","checked":true,"text":"todo"}`), "keepMe1", Options{}) + + require.NoError(t, err) + require.Len(t, blocks, 1) + assert.Equal(t, "keepMe1", blocks[0].Id) + assert.True(t, blocks[0].GetText().Checked) + }) + + t.Run("round-trips through MarshalBlockSubtree", func(t *testing.T) { + // given + src := json.RawMessage(`{"id":"b1","type":"quote","text":"a **bold** word","color":"red"}`) + blocks, err := UnmarshalBlock(src, "", Options{}) + require.NoError(t, err) + + // when + out, err := MarshalBlockSubtree(blocks, Options{}) + + // then + require.NoError(t, err) + arr := fragmentBlocks(t, out) + require.Len(t, arr, 1) + assert.Equal(t, "quote", arr[0]["type"]) + assert.Equal(t, "a **bold** word", arr[0]["text"], "inline markup survives the round-trip") + assert.Equal(t, "red", arr[0]["color"]) + }) + + t.Run("invalid shape fails the §5 checks", func(t *testing.T) { + _, err := UnmarshalBlock(json.RawMessage(`{"type":"nonsense"}`), "", Options{}) + + var verr *ValidationError + require.ErrorAs(t, err, &verr) + }) +} + +func TestMarshalBlockSubtree(t *testing.T) { + t.Run("subtree renders as a flat run with depths", func(t *testing.T) { + // given + subtree := []*model.Block{ + {Id: "p1", ChildrenIds: []string{"p2"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "parent"}}}, + {Id: "p2", + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "child"}}}, + } + + // when + out, err := MarshalBlockSubtree(subtree, Options{}) + + // then + require.NoError(t, err) + arr := fragmentBlocks(t, out) + require.Len(t, arr, 2) + assert.Equal(t, "p1", arr[0]["id"]) + assert.Nil(t, arr[0]["indent"]) + assert.Equal(t, float64(1), arr[1]["indent"]) + }) + + t.Run("empty subtree errors", func(t *testing.T) { + _, err := MarshalBlockSubtree(nil, Options{}) + require.Error(t, err) + }) +} + +// fragmentLabelSubtree is two minted block ids, parent and child: the only +// shape doc-local relabeling touches (isMintedLocalId). +func fragmentLabelSubtree() []*model.Block { + return []*model.Block{ + {Id: "64b2c1d2e3f4a5b6c7d8e9f0", ChildrenIds: []string{"1111111111111111111a1b2c"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "parent"}}}, + {Id: "1111111111111111111a1b2c", + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "child"}}}, + } +} + +// fragmentBlocks digs the `blocks` run out of a fragment envelope. +func fragmentBlocks(t *testing.T, out json.RawMessage) []map[string]any { + t.Helper() + var env struct { + Blocks []map[string]any `json:"blocks"` + } + require.NoError(t, json.Unmarshal(out, &env)) + return env.Blocks +} + +// fragmentLegend digs the three legends out of a fragment envelope. +func fragmentLegend(t *testing.T, out json.RawMessage) Legend { + t.Helper() + var env struct { + PropertyKeys map[string]string `json:"property_internal_keys"` + TypeKeys map[string]string `json:"type_internal_keys"` + OptionIds map[string]map[string]string `json:"option_ids"` + } + require.NoError(t, json.Unmarshal(out, &env)) + return Legend{PropertyKeys: env.PropertyKeys, TypeKeys: env.TypeKeys, OptionIds: env.OptionIds} +} + +func fragmentIds(t *testing.T, out json.RawMessage) []string { + t.Helper() + blocks := fragmentBlocks(t, out) + ids := make([]string, len(blocks)) + for i, b := range blocks { + ids[i], _ = b["id"].(string) + } + return ids +} + +// A fragment is addressed by the ids it carries, so the two options that take +// its addresses away are REFUSED rather than honoured. +// +// This surface exists for wiring that edits a live document op-by-op. OmitIds +// drops every block id, the view id and the filter id, so the run says what +// to write and not where. Block-label compaction rewrites doc-local ids to +// short suffixes that are local to the emitted run and are not the object's +// ids at all — the fragment used to hand back `8e9f0` for the block stored as +// `64b2c1d2e3f4a5b6c7d8e9f0`. Both produced a fragment that reads correctly +// and cannot be applied, which is the failure this format refuses to make +// silently. +func TestMarshalBlockSubtree_RefusesTheOptionsThatTakeAwayItsAddresses(t *testing.T) { + for name, tc := range map[string]struct { + opts Options + want string + }{ + "OmitIds": {Options{OmitIds: true}, "a fragment is addressed by the ids it carries"}, + "CompactBlockLabels": {Options{CompactBlockLabels: true}, "not the object's ids"}, + "CompactIds": {Options{CompactIds: true}, "not the object's ids"}, + } { + t.Run(name, func(t *testing.T) { + out, err := MarshalBlockSubtree(fragmentLabelSubtree(), tc.opts) + require.Error(t, err, "emitted:\n%s", out) + assert.Contains(t, err.Error(), tc.want) + assert.Nil(t, out, "a refused fragment hands back nothing") + }) + } + + t.Run("without them every id is the object's own", func(t *testing.T) { + out, err := MarshalBlockSubtree(fragmentLabelSubtree(), Options{}) + require.NoError(t, err) + assert.Equal(t, []string{"64b2c1d2e3f4a5b6c7d8e9f0", "1111111111111111111a1b2c"}, + fragmentIds(t, out)) + }) +} + +// A fragment has no envelope, so it can carry no legend — which is why the +// only compaction it may do is the legend-less one. Object references are +// therefore written in full here under EVERY option, exactly as in a whole +// document (§9a). +// +// This is the surviving, testable half of what 42396b448 fixed in passing: +// the fragment used to build its plan under the object-ref flag too, and +// emitted short object labels into an array that had nowhere to define them. +// That flag is deleted, so the bug is gone by construction and cannot be +// reproduced; what can be pinned is the property that made it a bug, and this +// fails the moment any legend-backed compaction reaches the fragment path +// again. +func TestMarshalBlockSubtree_ObjectRefsAreNeverCompacted(t *testing.T) { + const target = "bafyreimentiontargetidxxx" + subtree := []*model.Block{{Id: "64b2c1d2e3f4a5b6c7d8e9f0", + Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Text: "ping Roman", Marks: &model.BlockContentTextMarks{ + Marks: []*model.BlockContentTextMark{{ + Range: &model.Range{From: 5, To: 10}, + Type: model.BlockContentTextMark_Mention, Param: target}}}}}}} + + out, err := MarshalBlockSubtree(subtree, Options{}) + require.NoError(t, err) + assert.Contains(t, string(out), `object_id=\"`+target+`\"`, + "the mention target must be spelled in full — a fragment defines no object labels") + assert.NotContains(t, string(out), `object_id=\"idxxx\"`) +} + +func TestUnmarshalPropertyValue(t *testing.T) { + t.Run("date strings parse per §3", func(t *testing.T) { + v := UnmarshalPropertyValue("dueDate", "2026-07-30", Options{}) + + require.NotNil(t, v) + assert.NotZero(t, v.GetNumberValue(), "a date resolves to epoch seconds") + }) + + t.Run("round-trips with MarshalPropertyValue", func(t *testing.T) { + v := UnmarshalPropertyValue("dueDate", "2026-07-30T00:00:00Z", Options{}) + back, _ := MarshalPropertyValue("dueDate", v, Options{}) + assert.Equal(t, "2026-07-30T00:00:00Z", back) + }) + + t.Run("nil keeps presence as an explicit null", func(t *testing.T) { + v := UnmarshalPropertyValue("anything", nil, Options{}) + require.NotNil(t, v) + assert.IsType(t, &types.Value_NullValue{}, v.GetKind()) + }) +} + +func TestInlineTextCodec(t *testing.T) { + t.Run("parse and render invert each other", func(t *testing.T) { + // given + md := "plain **bold** and *italic* text" + + // when + text, marks, err := ParseInlineText(md) + + // then + require.NoError(t, err) + assert.Equal(t, "plain bold and italic text", text) + require.Len(t, marks, 2) + assert.Equal(t, md, RenderInlineText(text, marks)) + }) + + t.Run("render escapes markup characters", func(t *testing.T) { + rendered := RenderInlineText("2*3*4", nil) + text, marks, err := ParseInlineText(rendered) + require.NoError(t, err) + assert.Equal(t, "2*3*4", text) + assert.Empty(t, marks) + }) +} diff --git a/pkg/lib/anyblockjson/fragmentlegend_test.go b/pkg/lib/anyblockjson/fragmentlegend_test.go new file mode 100644 index 0000000000..1b1ccfd5ee --- /dev/null +++ b/pkg/lib/anyblockjson/fragmentlegend_test.go @@ -0,0 +1,273 @@ +package anyblockjson + +// fragmentlegend_test.go — a fragment carries what its blocks mean. +// +// The fragment surface (fragment.go, filters.go, BuildRecommendedLists) is +// the seam that edits live objects op-by-op, and it ran the §3 resolution +// chain from step 2 in both directions. Export computed all three legends and +// discarded them at the return; import built `&jsonDoc{}`, an empty legend, so +// step 1 was unconditionally silent. A block cut out of a document that said +// `{"priority": "6a32d485…"}` resolved `priority` through the READER's table +// — the exact misresolution the legend exists to prevent, reintroduced at the +// one seam where it lands on a live object. +// +// Every test here uses a DECOY: the reader's vocabulary binds the spelling to +// a different stored key, and the option pool holds a same-named option under +// a different id. Without a decoy the legend and the fallback agree and the +// test asks nothing. + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +const fragKey = "6a32d4856761631534b22f85" + +// fragSubtree is a property block and a dataview naming one custom key, with +// a filter carrying a select value — the three fragment slots that owe a +// legend between them. +func fragSubtree() []*model.Block { + return []*model.Block{ + {Id: "root1", ChildrenIds: []string{"rel1", "dv1"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "holder"}}}, + {Id: "rel1", Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{Key: fragKey}}}, + {Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + RelationLinks: []*model.RelationLink{{Key: fragKey, Format: model.RelationFormat_tag}}, + Views: []*model.BlockContentDataviewView{{Id: "v1", Name: "All", + Type: model.BlockContentDataviewView_Table, + Filters: []*model.BlockContentDataviewFilter{{ + Id: "f1", RelationKey: fragKey, Format: model.RelationFormat_tag, + Condition: model.BlockContentDataviewFilter_In, + Value: strList("bafylive"), + }}}}, + }}}, + } +} + +// fragSpace is the writer's space: it spells the custom key `priority` and +// serves one option called High. +func fragSpace() Options { + return Options{ + Keys: slugVocab{slugs: map[string]string{fragKey: "priority"}}, + ResolveFormat: selectFormats, + ResolveOptions: spaceOptions{fragKey: {{id: "bafylive", name: "High"}}}, + } +} + +// The export half: the legends the blocks owe come back with them. +func TestMarshalBlockSubtree_CarriesTheLegendsItsBlocksOwe(t *testing.T) { + // when + out, err := MarshalBlockSubtree(fragSubtree(), fragSpace()) + + // then — the run spells the space's slug, and the envelope says what it + // means; without the legend `priority` is just a word + require.NoError(t, err) + blocks := fragmentBlocks(t, out) + require.Len(t, blocks, 3) + assert.Equal(t, "priority", blocks[1]["property"], "the property block spells the slug") + + legend := fragmentLegend(t, out) + assert.Equal(t, map[string]string{"priority": fragKey}, legend.PropertyKeys) + assert.Equal(t, map[string]map[string]string{"priority": {"High": "bafylive"}}, legend.OptionIds) + + // and the member order is the envelope's (§2, §4): the legend that + // inverts a spelling precedes the legend keyed BY one + s := string(out) + assert.Less(t, indexOf(s, `"property_internal_keys"`), indexOf(s, `"option_ids"`)) + assert.Less(t, indexOf(s, `"option_ids"`), indexOf(s, `"blocks"`)) +} + +// The import half, with a decoy on both axes: the reader binds `priority` to +// another relation, and its option pool holds a SECOND option also called +// High. The legend has to beat both. +func TestUnmarshalBlocks_HonoursTheLegendItIsHandedOverItsOwnVocabulary(t *testing.T) { + // given — the fragment the writer produced, and a reader that disagrees + out, err := MarshalBlockSubtree(fragSubtree(), fragSpace()) + require.NoError(t, err) + var env struct { + Blocks []json.RawMessage `json:"blocks"` + } + require.NoError(t, json.Unmarshal(out, &env)) + legend := fragmentLegend(t, out) + + reader := Options{ + GenerateId: seqIds("g"), + Keys: slugVocab{slugs: map[string]string{"decoyKey": "priority"}}, + ResolveFormat: selectFormats, + ResolveOptions: spaceOptions{ + fragKey: { + {id: "bafydecoy", name: "High"}, // what a NAME resolves to + {id: "bafylive", name: "High"}, // what the LEGEND names + }, + "decoyKey": {{id: "bafydecoy", name: "High"}}, + }, + } + + t.Run("without the legend the reader's own answers win", func(t *testing.T) { + blocks, _, err := UnmarshalBlocks(env.Blocks, reader) + require.NoError(t, err) + assert.Equal(t, "decoyKey", fragBlockKey(t, blocks), + "`priority` is this reader's spelling for a different relation") + }) + + t.Run("with it the document's own statement is chain step 1", func(t *testing.T) { + withLegend := reader + withLegend.Legend = legend + + blocks, _, err := UnmarshalBlocks(env.Blocks, withLegend) + require.NoError(t, err) + assert.Equal(t, fragKey, fragBlockKey(t, blocks), + "the legend names the relation the fragment was cut from") + assert.Equal(t, []string{"bafylive"}, valueStringList(fragFilterValue(t, blocks)), + "and the option id beats the same-named decoy the pool answers first") + }) +} + +// The value-level pair. MarshalPropertyValue writes an option NAME and used +// to drop the id it stood for, so every row-level caller silently had the +// pre-§9a behaviour: a shared name lands on whichever option answers first. +func TestPropertyValue_TheOptionIdSurvivesTheValueLevelSurface(t *testing.T) { + // given — a space with two options called High, one of them the value's + writer := Options{ + ResolveFormat: selectFormats, + ResolveOptions: spaceOptions{fragKey: { + {id: "bafydecoy", name: "High"}, + {id: "bafylive", name: "High"}, + }}, + } + + // when + out, ids := MarshalPropertyValue(fragKey, strList("bafylive"), writer) + + // then + assert.Equal(t, []any{"High"}, out, "the value is written as the option NAME") + require.Equal(t, map[string]string{"High": "bafylive"}, ids, + "and the id it stood for comes back, or the name is all the reader gets") + + // the reader: name resolution alone answers the decoy, which is the loss + back := UnmarshalPropertyValue(fragKey, out, writer) + assert.Equal(t, []string{"bafydecoy"}, valueStringList(back), + "first match wins, and it is the wrong option") + + // handed the id, it answers the value that was written + withLegend := writer + withLegend.Legend = Legend{OptionIds: map[string]map[string]string{fragKey: ids}} + back = UnmarshalPropertyValue(fragKey, out, withLegend) + assert.Equal(t, []string{"bafylive"}, valueStringList(back)) +} + +// UnmarshalFilters is the query-side twin, and it resolves both a key slot and +// an option value. +func TestUnmarshalFilters_HonoursTheLegendItIsHandedOver(t *testing.T) { + raw := json.RawMessage(`[{"property":"priority","condition":"in","value":["High"]}]`) + reader := Options{ + Keys: slugVocab{slugs: map[string]string{"decoyKey": "priority"}}, + ResolveFormat: selectFormats, + ResolveOptions: spaceOptions{ + fragKey: {{id: "bafydecoy", name: "High"}, {id: "bafylive", name: "High"}}, + "decoyKey": {{id: "bafydecoy", name: "High"}}, + }, + } + + t.Run("without the legend", func(t *testing.T) { + got, err := UnmarshalFilters(raw, reader) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "decoyKey", got[0].RelationKey) + }) + + t.Run("with it", func(t *testing.T) { + withLegend := reader + withLegend.Legend = Legend{ + PropertyKeys: map[string]string{"priority": fragKey}, + OptionIds: map[string]map[string]string{"priority": {"High": "bafylive"}}, + } + + got, err := UnmarshalFilters(raw, withLegend) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, fragKey, got[0].RelationKey) + assert.Equal(t, []string{"bafylive"}, valueStringList(got[0].Value), + "the option legend is keyed by the SPELLING the slot wrote (§9a)") + }) +} + +// BuildRecommendedLists is the PATCH-type door into the same array +// applyTypeProperties reads out of a document, and its own doc comment used to +// say the caller "owes them that document's legend before calling, because +// nothing downstream of this signature can see it". Now it can. +func TestBuildRecommendedLists_HonoursTheLegendItIsHandedOver(t *testing.T) { + props := []TypeProperty{{Property: "priority", Section: "featured"}} + reader := Options{Keys: slugVocab{slugs: map[string]string{"decoyKey": "priority"}}} + + t.Run("without the legend", func(t *testing.T) { + lists, err := BuildRecommendedLists(props, reader) + require.NoError(t, err) + assert.Equal(t, []string{"decoyKey"}, listKeys(t, lists, "recommendedFeaturedRelations")) + }) + + t.Run("with it", func(t *testing.T) { + withLegend := reader + withLegend.Legend = Legend{PropertyKeys: map[string]string{"priority": fragKey}} + + lists, err := BuildRecommendedLists(props, withLegend) + require.NoError(t, err) + assert.Equal(t, []string{fragKey}, listKeys(t, lists, "recommendedFeaturedRelations"), + "the type's recommended list names the relation the document meant") + }) +} + +// fragBlockKey is the property block's resolved key in an imported run. +func fragBlockKey(t *testing.T, blocks []*model.Block) string { + t.Helper() + for _, b := range blocks { + if c, ok := b.Content.(*model.BlockContentOfRelation); ok { + return c.Relation.Key + } + } + t.Fatal("no property block in the run") + return "" +} + +// fragFilterValue is the single dataview filter's value in an imported run. +func fragFilterValue(t *testing.T, blocks []*model.Block) *types.Value { + t.Helper() + for _, b := range blocks { + if c, ok := b.Content.(*model.BlockContentOfDataview); ok { + require.Len(t, c.Dataview.Views, 1) + require.Len(t, c.Dataview.Views[0].Filters, 1) + return c.Dataview.Views[0].Filters[0].Value + } + } + t.Fatal("no dataview in the run") + return nil +} + +// listKeys is the resolved key list of one recommended section, by the detail +// key that section writes to. +func listKeys(t *testing.T, lists []RecommendedList, detailKey string) []string { + t.Helper() + for _, l := range lists { + if l.DetailKey == detailKey { + return l.Ids + } + } + t.Fatalf("no %q list in %v", detailKey, lists) + return nil +} + +func indexOf(s, sub string) int { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return i + } + } + return -1 +} diff --git a/pkg/lib/anyblockjson/golden_gen_test.go b/pkg/lib/anyblockjson/golden_gen_test.go new file mode 100644 index 0000000000..cc3420d191 --- /dev/null +++ b/pkg/lib/anyblockjson/golden_gen_test.go @@ -0,0 +1,47 @@ +package anyblockjson + +import ( + "flag" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +var updateGolden = flag.Bool("update", false, "rewrite golden files") + +func checkGolden(t *testing.T, name string, got []byte) { + t.Helper() + path := filepath.Join("testdata", name) + if *updateGolden { + require.NoError(t, os.WriteFile(path, got, 0o644)) + return + } + want, err := os.ReadFile(path) + require.NoError(t, err, "golden %s missing; run go test -update", name) + require.Equal(t, string(want), string(got)) +} + +// TestMarshal_GoldenFiles freezes the canonical bytes for the rich snapshot +// in all serialization modes (§11 canon). +func TestMarshal_GoldenFiles(t *testing.T) { + for _, tc := range []struct { + name string + opts func() Options + }{ + {"rich.json", testOptions}, + {"rich_omit_ids.json", func() Options { o := testOptions(); o.OmitIds = true; return o }}, + {"rich_compact_ids.json", func() Options { o := testOptions(); o.CompactIds = true; return o }}, + {"rich_compact_omit.json", func() Options { o := testOptions(); o.CompactIds = true; o.OmitIds = true; return o }}, + } { + t.Run(tc.name, func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, richSnapshot(), tc.opts()) + require.NoError(t, err) + require.NoError(t, Validate(data)) + checkGolden(t, tc.name, data) + }) + } +} diff --git a/pkg/lib/anyblockjson/groupby_test.go b/pkg/lib/anyblockjson/groupby_test.go new file mode 100644 index 0000000000..6194e9eab9 --- /dev/null +++ b/pkg/lib/anyblockjson/groupby_test.go @@ -0,0 +1,101 @@ +package anyblockjson + +// Only kanban (select/multiSelect/checkbox) and calendar (date) group. A +// groupBy anywhere else renders nothing at all, which authors reliably get +// wrong because the document looks entirely reasonable. + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func dataviewDoc(props, view string) string { + return `{"version": 2, "id": "p1", "blocks": [{"type": "dataview", + "object_id": "someSet", "properties": [` + props + `], + "views": [` + view + `]}]}` +} + +func TestValidate_GroupByImpossibleIsError(t *testing.T) { + for _, tc := range []struct { + name, props, view, wantMsg string + }{ + { + name: "kanban on an object relation", + props: `{"property": "category", "format": "objects"}`, + view: `{"type": "kanban", "name": "By category", "group_by": "category"}`, + wantMsg: `cannot group by "category"`, + }, + { + name: "kanban on a date", + props: `{"property": "due", "format": "date"}`, + view: `{"type": "kanban", "name": "By due", "group_by": "due"}`, + wantMsg: `cannot group by "due"`, + }, + { + name: "calendar on a select", + props: `{"property": "status", "format": "select"}`, + view: `{"type": "calendar", "name": "Cal", "group_by": "status"}`, + wantMsg: `cannot group by "status"`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := Validate([]byte(dataviewDoc(tc.props, tc.view))) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantMsg) + }) + } +} + +func TestValidate_GroupByValidCombinations(t *testing.T) { + for _, tc := range []struct{ name, props, view string }{ + {"kanban + select", `{"property": "status", "format": "select"}`, + `{"type": "kanban", "name": "K", "group_by": "status"}`}, + {"kanban + multi_select", `{"property": "tags", "format": "multi_select"}`, + `{"type": "kanban", "name": "K", "group_by": "tags"}`}, + {"kanban + checkbox", `{"property": "done", "format": "checkbox"}`, + `{"type": "kanban", "name": "K", "group_by": "done"}`}, + {"calendar + date", `{"property": "due", "format": "date"}`, + `{"type": "calendar", "name": "C", "group_by": "due"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + assert.NoError(t, Validate([]byte(dataviewDoc(tc.props, tc.view)))) + }) + } +} + +// A stale groupBy on a non-grouping view is real exported data: switching a +// kanban to a table in the editor leaves groupRelationKey behind +// (insertGroupRelationKey's default branch is a no-op). It must warn, never +// reject, or round-tripping an account would fail. +func TestValidate_GroupByOnNonGroupingViewOnlyWarns(t *testing.T) { + for _, viewType := range []string{"table", "list", "gallery", "graph"} { + t.Run(viewType, func(t *testing.T) { + view := `{"type": "` + viewType + `", "name": "V", "group_by": "status"}` + if viewType == "table" { + view = `{"name": "V", "group_by": "status"}` // table is the default type + } + doc := dataviewDoc(`{"property": "status", "format": "select"}`, view) + + require.NoError(t, Validate([]byte(doc)), "must not reject real data") + + var warnings []Issue + _, _, err := Unmarshal([]byte(doc), Options{ + GenerateId: seqIds("g"), + OnWarning: func(i Issue) { warnings = append(warnings, i) }, + }) + require.NoError(t, err) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "do not group") + assert.Contains(t, warnings[0].Path, "group_by") + }) + } +} + +// nothing to check against when the key carries no declared format +func TestValidate_GroupByUndeclaredKeyIsAccepted(t *testing.T) { + doc := dataviewDoc(`{"property": "other", "format": "select"}`, + `{"type": "kanban", "name": "K", "group_by": "notInProperties"}`) + assert.NoError(t, Validate([]byte(doc))) +} diff --git a/pkg/lib/anyblockjson/iconcover.go b/pkg/lib/anyblockjson/iconcover.go new file mode 100644 index 0000000000..c9cc0f0557 --- /dev/null +++ b/pkg/lib/anyblockjson/iconcover.go @@ -0,0 +1,762 @@ +package anyblockjson + +// iconcover.go implements §2b: the typed `icon` and `cover` envelope fields. +// +// Nine hidden stored keys — iconEmoji, iconImage, iconName, iconOption, +// coverId, coverType, coverScale, coverX, coverY — used to sit in +// `properties` as nine independent slots, none of which said which of the +// others it excluded. Over 36 966 real objects that produced 22 distinct flat +// key-sets for what is really one choice with eight shapes, and one +// undecodable pair: `"cover_id": "blue"` is a COLOUR under `cover_type: 2` +// and a GRADIENT under `cover_type: 3`, and both occur. +// +// The two fields collapse each family into one object whose `format` member +// selects the variant. They live in the ENVELOPE rather than in `properties` +// for four reasons that are forced rather than aesthetic: `cover` is already +// a stored property key in real data (30 documents, plus 66 spelling it +// `pageCover`), a `properties` member can be rebound by the `property_internal_keys` +// legend to point at an arbitrary relation, `properties` carries +// presence-is-meaningful (§3) while the envelope omits empties (§4), and an +// envelope member has a schema node of its own to annotate. + +import ( + "fmt" + "math" + "strings" + "unicode/utf8" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/constant" +) + +// The nine stored detail keys the typed envelope fields carry. Named off the +// bundle so a rename there is a compile error here rather than a silent +// un-lift. +var ( + detailKeyIconEmoji = bundle.RelationKeyIconEmoji.String() + detailKeyIconImage = bundle.RelationKeyIconImage.String() + detailKeyIconName = bundle.RelationKeyIconName.String() + detailKeyIconOption = bundle.RelationKeyIconOption.String() + detailKeyCoverId = bundle.RelationKeyCoverId.String() + detailKeyCoverType = bundle.RelationKeyCoverType.String() + detailKeyCoverScale = bundle.RelationKeyCoverScale.String() + detailKeyCoverX = bundle.RelationKeyCoverX.String() + detailKeyCoverY = bundle.RelationKeyCoverY.String() +) + +// liftedDetailKeys is the icon/cover lift list, and it is the single source of +// truth for both directions: export writes these keys nowhere but the typed +// envelope fields, and import refuses them in `properties` +// (deniedPropertyKey reads this same set, §2b, §3). The precedent is +// `type_properties` (§2a); the difference is that this list is ALWAYS on, +// because the fields it feeds are a pure function of the details bag and need +// no resolver. +// +// Deriving the refusal from the list is the point — a restated list is how +// the export and import surfaces drifted apart the last time (see +// strippedDetailKeys). +func liftedDetailKeys() map[string]bool { + return map[string]bool{ + detailKeyIconEmoji: true, + detailKeyIconImage: true, + detailKeyIconName: true, + detailKeyIconOption: true, + detailKeyCoverId: true, + detailKeyCoverType: true, + detailKeyCoverScale: true, + detailKeyCoverX: true, + detailKeyCoverY: true, + } +} + +// liftedKeyRepair names the envelope field a refused flat spelling belongs in, +// and shows the shape. deniedPropertyKey's other messages say what is wrong; +// this one says what to write instead, because unlike an internal key there +// IS something to write instead. +func liftedKeyRepair(key string) string { + switch key { + case detailKeyIconEmoji: + return `"icon": {"format": "emoji", "emoji": "…"}` + case detailKeyIconImage: + return `"icon": {"format": "file", "file": ""}` + case detailKeyIconName: + return `"icon": {"format": "icon", "name": "…"}` + case detailKeyIconOption: + return `the "color" member of "icon"` + case detailKeyCoverId, detailKeyCoverType: + return `"cover": {"format": "image"|"color"|"gradient", …}` + case detailKeyCoverScale, detailKeyCoverX, detailKeyCoverY: + return `the "scale"/"x"/"y" members of an image "cover"` + } + return "" +} + +// +// ---- values ---- +// + +// maxObjectRefLen and maxOpaqueNameLen mirror the schema's `objectRef` and +// `opaqueName` bounds. Export admits before it writes, so Marshal never emits +// a value its own Validate rejects (§11, I1) — which is the whole reason the +// 33 leaked filesystem paths in the corpus are dropped rather than carried. +const ( + maxObjectRefLen = 255 + maxOpaqueNameLen = 64 +) + +// isObjectRef reports whether a stored value can be written where the schema +// wants an object reference. The deny rule is `^[^/]+$`, not a URL-scheme +// lookahead: the compiler runs Go's RE2, which has no lookahead, and a slash +// is what every unwritable value in 36 966 real objects has in common — 33 +// absolute filesystem paths a Notion import left in `coverId`, and nothing +// else. A URL cannot be written either way, which is the layering rule (§2b): +// the format holds only what the store holds, and a URL is resolved to a file +// object id by a layer above before it reaches here. +func isObjectRef(s string) bool { + return s != "" && utf8.RuneCountInString(s) <= maxObjectRefLen && !strings.Contains(s, "/") +} + +// isOpaqueName reports whether a value can be written where the schema wants a +// name from a vocabulary this format does not enumerate — an icon name, a +// cover colour, a gradient. +func isOpaqueName(s string) bool { + return s != "" && utf8.RuneCountInString(s) <= maxOpaqueNameLen && !strings.Contains(s, "/") +} + +// iconColorNames is the palette, in the canonical order the stored +// `iconOption` numbers index: `iconOption: n` is `palette[n-1]`. The mapping +// is total and free — apimodel.IconOptionToColor maps 1..10 onto +// constant.OptionColors() positionally, and §2a already mandates the same +// palette for select options, so the format adopts one colour vocabulary +// rather than minting a second. +func iconColorNames() []string { + colors := constant.OptionColors() + out := make([]string, 0, len(colors)) + for _, c := range colors { + out = append(out, c.String()) + } + return out +} + +// iconColorValue renders a stored `iconOption` as the schema's `iconColor`: a +// palette name, or the raw number for a value the palette has no name for. +// ok is false when there is no colour at all — `iconOption: 0` is the proto +// zero, not the first colour, and treating it as one would invent a grey icon +// on 145 real objects. +// +// The integer escape is not decoration: two generators in this repo disagree +// about the range (`rand.Intn(16)+1` in the pb importer,`rand.Intn(10)+1` in +// the markdown one), so 12/13/15 exist in real data. It is the same device +// §3 already uses for a layout number outside the enum. +func iconColorValue(v *types.Value) (any, bool) { + if v == nil { + return nil, false + } + n := v.GetNumberValue() + if n != math.Trunc(n) || math.IsNaN(n) || math.IsInf(n, 0) { + return nil, false + } + i := int64(n) + if i < 1 { + return nil, false + } + names := iconColorNames() + if i <= int64(len(names)) { + return names[i-1], true + } + return i, true +} + +// iconOptionOf inverts iconColorValue: the stored number a written colour +// stands for. A name outside the palette cannot arrive — the schema refuses +// it — so the fallback is unreachable rather than lenient. +func iconOptionOf(color any) (float64, bool) { + switch c := color.(type) { + case string: + for i, name := range iconColorNames() { + if name == c { + return float64(i + 1), true + } + } + case float64: + if c >= 1 { + return c, true + } + case int64: + if c >= 1 { + return float64(c), true + } + } + return 0, false +} + +// coverSourceNames maps the stored `coverType` numbers that all mean "an +// image" onto the provenance the typed field carries. The relation's own +// bundled description is the union written as prose: "1-image, 2-color, +// 3-gradient, 4-prebuilt bg image, 5-unsplash image". +// +// One `image` branch with an output-only `source`, rather than three +// branches: a generator that just uploaded an image has no basis to choose +// between them, and choosing `unsplash` writes a permanent false provenance +// claim into cold storage. +const ( + coverTypeNone = 0 + coverTypeImage = 1 + coverTypeColor = 2 + coverTypeGradient = 3 + coverTypePrebuilt = 4 + coverTypeUnsplash = 5 + + coverSourceUnsplash = "unsplash" + coverSourcePrebuilt = "prebuilt" +) + +// +// ---- export ---- +// + +// iconField and coverField are the memoized typed envelope fields. They are +// built once and read twice — the id census (buildLabelPlan) needs the object +// ids they write, and buildDoc needs the objects themselves — so building on +// demand would report every warning twice under compaction. +func (e *exporter) iconField() *omap { + if !e.iconBuilt { + e.icon = e.buildIcon() + e.iconBuilt = true + } + return e.icon +} + +func (e *exporter) coverField() *omap { + if !e.coverBuilt { + e.cover = e.buildCover() + e.coverBuilt = true + } + return e.cover +} + +// buildIcon renders the four icon channels as one typed field (§2b), or nil +// when the object has no icon. +// +// The precedence is `iconName` → `iconEmoji` → `iconImage`, which is +// core/api/service/icon.go's rule and the only precedence implementation in +// heart — everywhere else (the dot, graphjson and publish converters) emits +// every channel and lets the consumer decide. `iconOption` is NOT a fourth +// step in that chain: it is orthogonal, and attaches as `color` to whichever +// channel won, standing alone only when none did (87 real objects attach a +// colour to something other than a named icon, and 29 carry a colour with no +// source at all). +// +// A source whose stored value is EMPTY is not a source. That is the one place +// this field overrides §3's presence-is-meaningful rule, and the carve-out is +// principled rather than convenient: all nine relations are `hidden: true`, +// so there is no property row for presence to be meaningful to. It is what +// deletes the format's largest class of fake ambiguity — 883 real objects +// carry both `iconEmoji` and `iconImage`, and in not one of them are both +// non-empty. +func (e *exporter) buildIcon() *omap { + if e.snapshot == nil || e.snapshot.Details == nil { + return nil + } + return iconOmap(iconOf(e.detail, e.warn, e.iconTargetDeleted)) +} + +// iconOf chooses the icon from the four stored channels (§2b), or returns nil +// when the object has no icon. It is the ONE implementation of the precedence +// described above: the object surface renders what it returns, and so does a +// bundle index (§2c), which is what keeps the space icon and the object icon +// from being two conventions for one concept. +// +// It reports through `warn` exactly where a stored value cannot be carried, +// and a caller that cannot afford to lose one — the index, which omits the +// document it read the icon from — treats any warning as a refusal. +// deleted reports that an icon image id names an object the space deleted. +// nil means "never ask" — a package-only export with no store wired, and the +// space-icon reader, which has no options to consult. +func iconOf(detail func(string) *types.Value, warn func(path, format string, args ...any), + deleted func(string) bool) *Icon { + color, hasColor := iconColorValue(detail(detailKeyIconOption)) + ic := &Icon{} + if hasColor { + ic.Color = color + } + + if name := detail(detailKeyIconName).GetStringValue(); name != "" { + if !isOpaqueName(name) { + warn("/icon", "icon name %q cannot be written in this format and is dropped", name) + } else { + ic.Format = "icon" + ic.Name = name + // the conflict carry-over: 200 real objects — every one a bundled + // type mid-migration from an emoji to a named icon — hold BOTH. + // `format` has already answered which icon wins, so the emoji is + // baggage rather than ambiguity, and a cold-storage backup format + // that silently deletes a non-empty stored value on export is + // disqualifying. It is annotated x-output-only: a document that + // supplies it is not choosing an icon. + if emoji := detail(detailKeyIconEmoji).GetStringValue(); emoji != "" { + ic.Emoji = emoji + warn("/icon", "this object holds both a named icon (%q) and an emoji (%q); "+ + "the name wins and the emoji is carried as output-only baggage", name, emoji) + } + return ic + } + } + if emoji := detail(detailKeyIconEmoji).GetStringValue(); emoji != "" { + ic.Format = "emoji" + ic.Emoji = emoji + return ic + } + if images := valueStringList(detail(detailKeyIconImage)); len(images) > 0 { + if len(images) > 1 { + warn("/icon", "the icon image list holds %d entries; only the first is an icon", len(images)) + } + if deleted != nil && deleted(images[0]) { + // the file object is a tombstone: the space kept the id but not + // the image, so carrying it would ship an icon that resolves to + // nothing. An icon is optional — unlike a link or a mention, + // which must have a target and get the sentinel instead (§9) — + // so it is dropped and the remaining channels answer. + warn("/icon", "icon image %q names an object this space deleted and is dropped", images[0]) + } else if !isObjectRef(images[0]) { + // there is no way to write it: the schema's objectRef refuses a + // URL and a filesystem path, so carrying it would make Marshal + // emit what its own Validate rejects (§11, I1) + warn("/icon", "icon image %q is not an object id and is dropped — "+ + "this format holds a reference to an image object, never a URL or a path", images[0]) + } else { + ic.Format = "file" + ic.File = images[0] + return ic + } + } + if hasColor { + // a colour with no source: the letter-avatar background. 29 real + // objects carry one, and the API reports every one of them as having + // no icon at all. + ic.Format = "color" + return ic + } + return nil +} + +// iconOmap renders a chosen icon as the format's typed `icon` field. It is +// the one renderer, shared by the object surface and the bundle index, so the +// two cannot drift into different spellings of the same icon. +// +// `format` is the discriminator and comes first; the colour attaches to +// whichever channel won; the named-icon variant carries its emoji last, as +// output-only baggage. +func iconOmap(ic *Icon) *omap { + if ic == nil { + return nil + } + format := ic.Format + if format == "" { + // a caller that filled the channel without naming the variant + switch { + case ic.Name != "": + format = "icon" + case ic.Emoji != "": + format = "emoji" + case ic.File != "": + format = "file" + case ic.Color != nil: + format = "color" + default: + return nil + } + } + m := &omap{} + m.set("format", format) + switch format { + case "icon": + m.set("name", ic.Name) + case "emoji": + m.set("emoji", ic.Emoji) + case "file": + m.set("file", ic.File) + } + if ic.Color != nil { + m.set("color", ic.Color) + } + if format == "icon" && ic.Emoji != "" { + m.set("emoji", ic.Emoji) + } + return m +} + +// buildCover renders the five cover channels as one typed field (§2b), or nil +// when the object has no cover. +// +// `coverType` is the discriminator and it is provably load-bearing: `"blue"` +// occurs in the corpus as `cover_type: 2` (a colour) AND as `cover_type: 3` +// (a gradient). `cover_id` alone is undecodable, so this pair cannot be +// simplified any other way — only typed. +func (e *exporter) buildCover() *omap { + if e.snapshot == nil || e.snapshot.Details == nil { + return nil + } + id := e.detail(detailKeyCoverId).GetStringValue() + raw := e.detail(detailKeyCoverType).GetNumberValue() + t := int(raw) + if raw != math.Trunc(raw) || t < coverTypeNone || t > coverTypeUnsplash { + e.warn("/cover", "cover type %v is not one of 0..5 (1-image, 2-color, 3-gradient, "+ + "4-prebuilt, 5-unsplash); the cover is dropped", raw) + return nil + } + if t == coverTypeNone { + if id != "" { + e.warn("/cover", "cover %q has no cover type, so nothing says how to read it; it is dropped", id) + } + return nil + } + if id == "" { + e.warn("/cover", "cover type %d has no cover id to go with it; the cover is dropped", t) + return nil + } + + m := &omap{} + switch t { + case coverTypeColor, coverTypeGradient: + if !isOpaqueName(id) { + e.warn("/cover", "cover %q cannot be written as a name in this format and is dropped", id) + return nil + } + if t == coverTypeColor { + m.set("format", "color") + m.set("color", id) + } else { + m.set("format", "gradient") + m.set("gradient", id) + } + return m + } + + if !isObjectRef(id) { + // 33 real objects reach here, every one an absolute path into a + // long-gone temp directory that core/block/import/notion wrote into + // coverId as if it were a file reference. The value is already dead; + // dropping it turns permanent silent corruption into a named event. + e.warn("/cover", "cover image %q is not an object id and is dropped — "+ + "this format holds a reference to an image object, never a URL or a path", id) + return nil + } + m.set("format", "image") + m.set("file", id) + switch t { + case coverTypeUnsplash: + m.set("source", coverSourceUnsplash) + case coverTypePrebuilt: + m.set("source", coverSourcePrebuilt) + } + // framing belongs to an image and to nothing else: in 36 966 real objects + // these three are non-zero only under cover types 1 and 5, though they + // are PRESENT and zero on colours, gradients and cleared covers alike + m.setNonEmpty("scale", e.detail(detailKeyCoverScale).GetNumberValue()) + m.setNonEmpty("x", e.detail(detailKeyCoverX).GetNumberValue()) + m.setNonEmpty("y", e.detail(detailKeyCoverY).GetNumberValue()) + return m +} + +// calloutIcon renders a callout block's two icon attributes as the same typed +// shape the envelope field uses, restricted to `emoji` and `file` (§5.2). +// +// Emoji beats image, the object rule's order minus the two channels a block +// has no room for. In 36 966 real objects there are 650 callouts — 256 with +// an emoji, 2 with an image, and NONE with both — so the precedence is +// unexercised, and the warning below says so if that ever changes. +func (e *exporter) calloutIcon(t *model.BlockContentText) *omap { + m := &omap{} + if t.IconEmoji != "" { + if t.IconImage != "" { + e.warn("/blocks", "callout %s holds both an emoji and an image icon; the emoji wins "+ + "and the image is dropped", t.IconImage) + } + m.set("format", "emoji") + m.set("emoji", t.IconEmoji) + return m + } + if t.IconImage == "" { + return nil + } + if !isObjectRef(t.IconImage) { + e.warn("/blocks", "callout icon %q is not an object id and is dropped — this format holds "+ + "a reference to an image object, never a URL or a path", t.IconImage) + return nil + } + m.set("format", "file") + m.set("file", t.IconImage) + return m +} + +// calloutIconFrom inverts calloutIcon. +func calloutIconFrom(ic *Icon, t *model.BlockContentText) { + if ic == nil { + return + } + switch ic.Format { + case "emoji": + t.IconEmoji = ic.Emoji + case "file": + t.IconImage = ic.File + } +} + +// liftedObjectIds is every OBJECT id the typed envelope fields write. The id +// census needs it explicitly: those ids used to reach the avoid-set through +// the ordinary property walk (`iconImage` is a `file` relation), and the lift +// takes them out of it — while `coverId` is a `longtext` relation and was +// never in it at all, so a compact block label could always have collided +// with a file-backed cover id (§9a). +func (e *exporter) liftedObjectIds() []string { + var out []string + if m := e.iconField(); m != nil { + if id, ok := omapString(m, "file"); ok { + out = append(out, id) + } + } + if m := e.coverField(); m != nil { + if id, ok := omapString(m, "file"); ok { + out = append(out, id) + } + } + return out +} + +// omapString reads a string member out of a built field. +func omapString(m *omap, key string) (string, bool) { + for i, k := range m.keys { + if k == key { + s, ok := m.vals[i].(string) + return s, ok + } + } + return "", false +} + +// +// ---- import ---- +// + +// Icon and Cover are the two typed fields (§2b), exported because they are +// the shape a caller writing a document or a bundle index has to build — and +// because the API layer needs the same union and should adopt this one rather +// than mint a second. +// +// `Color` is `any` because it is the union the schema states: one of the ten +// palette names, or a raw number for a stored value the palette has no name +// for. Everything else is typed. Exactly one variant's members are populated +// at a time; `Format` says which. +type Icon struct { + Format string `json:"format"` + Emoji string `json:"emoji"` + File string `json:"file"` + Name string `json:"name"` + Color any `json:"color"` +} + +type Cover struct { + Format string `json:"format"` + File string `json:"file"` + Source string `json:"source"` + Color string `json:"color"` + Gradient string `json:"gradient"` + Scale float64 `json:"scale"` + X float64 `json:"x"` + Y float64 `json:"y"` +} + +// applyIcon writes the stored keys the typed `icon` field stands for (§2b). +// Every emitted variant inverts to exactly the details that produced it, +// which is what makes Export ∘ Import a fixpoint over this field. +// +// It cannot fail: the schema has already refused every shape that is not one +// of the four variants, so there is no case left for the reader to judge. +func (imp *importer) applyIcon(details *types.Struct) { + ic := imp.doc.Icon + if ic == nil { + return + } + setNum := func(key string, n float64) { + details.Fields[key] = &types.Value{Kind: &types.Value_NumberValue{NumberValue: n}} + } + setStr := func(key, s string) { + details.Fields[key] = &types.Value{Kind: &types.Value_StringValue{StringValue: s}} + } + switch ic.Format { + case "emoji": + setStr(detailKeyIconEmoji, ic.Emoji) + case "file": + // `iconImage` is a `file` relation, so its stored shape is a list — + // the same shape the ordinary property path writes (wrapToList), and + // the shape all 12 011 populated cases in the corpus hold + details.Fields[detailKeyIconImage] = &types.Value{Kind: &types.Value_ListValue{ + ListValue: &types.ListValue{Values: []*types.Value{ + {Kind: &types.Value_StringValue{StringValue: ic.File}}, + }}, + }} + case "icon": + setStr(detailKeyIconName, ic.Name) + if ic.Emoji != "" { + setStr(detailKeyIconEmoji, ic.Emoji) + } + } + if n, ok := iconOptionOf(ic.Color); ok { + setNum(detailKeyIconOption, n) + } +} + +// applyCover writes the stored keys the typed `cover` field stands for (§2b). +func (imp *importer) applyCover(details *types.Struct) { + cv := imp.doc.Cover + if cv == nil { + return + } + setNum := func(key string, n float64) { + details.Fields[key] = &types.Value{Kind: &types.Value_NumberValue{NumberValue: n}} + } + setStr := func(key, s string) { + details.Fields[key] = &types.Value{Kind: &types.Value_StringValue{StringValue: s}} + } + switch cv.Format { + case "color": + setStr(detailKeyCoverId, cv.Color) + setNum(detailKeyCoverType, coverTypeColor) + return + case "gradient": + setStr(detailKeyCoverId, cv.Gradient) + setNum(detailKeyCoverType, coverTypeGradient) + return + } + setStr(detailKeyCoverId, cv.File) + switch cv.Source { + case coverSourceUnsplash: + setNum(detailKeyCoverType, coverTypeUnsplash) + case coverSourcePrebuilt: + setNum(detailKeyCoverType, coverTypePrebuilt) + default: + setNum(detailKeyCoverType, coverTypeImage) + } + if cv.Scale != 0 { + setNum(detailKeyCoverScale, cv.Scale) + } + if cv.X != 0 { + setNum(detailKeyCoverX, cv.X) + } + if cv.Y != 0 { + setNum(detailKeyCoverY, cv.Y) + } +} + +// LiftedPropertyKeys reports the stored property keys the typed envelope +// fields carry instead of `properties` (§2b). Exported for the same reason +// InternalPropertyKeys is: a round-trip checker comparing a snapshot with its +// re-import has to know which keys moved, or it reports a faithful export as +// data loss. +func LiftedPropertyKeys() map[string]bool { + return liftedDetailKeys() +} + +// DroppedEmptyIconCover reports a stored icon/cover value the typed fields +// treat as NO SOURCE at all — an empty string, an empty list, a zero number, +// a null (§2b, N(S)). Such a key does not survive a round trip, and that is +// the one place the typed fields override §3's presence-is-meaningful rule: +// all nine relations are `hidden: true`, so there is no property row for +// presence to be meaningful to. +// +// It exists so the round-trip comparator can suppress exactly that step and +// nothing else. In the 36 966-object corpus ~2 300 objects carry at least one +// present-but-empty icon or cover key, which is nearly twice the noise that +// buried a previous sweep (see snapshotdiff's recommendedListKeys comment) — +// and a comparator that suppressed the whole KEY instead would go blind to +// the 33 objects whose cover really is lost. +func DroppedEmptyIconCover(key string, v *types.Value) bool { + return liftedDetailKeys()[key] && !liftedValueIsSource(v) +} + +// liftedValueIsSource is the emptiness rule the export builders apply, in one +// place so the comparator and the builders cannot disagree about which values +// are sources. +func liftedValueIsSource(v *types.Value) bool { + switch k := v.GetKind().(type) { + case *types.Value_StringValue: + return k.StringValue != "" + case *types.Value_NumberValue: + return k.NumberValue != 0 + case *types.Value_ListValue: + return len(valueStringList(v)) > 0 + } + return false +} + +// +// ---- validation ---- +// + +// iconFormatIssues states the one thing the schema's own verdict cannot: what +// a typed field with no `format` should have said instead. `required: +// ["format"]` reports `missing property 'format'`, which tells an author a +// member is missing but not that it is a CHOICE, or what the choices are — +// and naming the alternatives at the moment the author is wrong is the whole +// reason the field is typed rather than flat. +// +// The alternatives are read out of the published schema rather than restated +// here, so a variant added at the extension seam (§2b) shows up in this +// message for free. +func iconFormatIssues(doc map[string]any, r *keySlotReport) { + check := func(path, noun, def string, node any) { + if issue, missing := missingFormatIssue(path, noun, def, node); missing { + r.rejectValueAt(issue.Path, issue.Message) + } + } + check("/icon", "an icon", "icon", doc["icon"]) + check("/cover", "a cover", "cover", doc["cover"]) + for i, raw := range blocksOf(doc) { + b, _ := raw.(map[string]any) + if b == nil { + continue + } + check(fmt.Sprintf("/blocks/%d/icon", i), "a callout icon", "plainIcon", b["icon"]) + } +} + +// missingFormatIssue is that verdict for one field. Split out because the +// bundle index carries the same typed icon and validates through a different +// entry point (UnmarshalIndex), and a second wording of one rule is how two +// surfaces of one format start disagreeing. +func missingFormatIssue(path, noun, def string, node any) (Issue, bool) { + obj, isObject := node.(map[string]any) + if !isObject { + return Issue{}, false // the schema types the field; this pass only shapes it + } + if _, has := obj["format"]; has { + return Issue{}, false + } + names := schemaFormatEnum(def) + if len(names) == 0 { + return Issue{}, false + } + return Issue{Path: path, Message: fmt.Sprintf( + "missing property 'format': %s is one of %s (\u00a72b)", noun, quotedList(names))}, true +} + +// quotedList renders the alternatives the way the schema library renders an +// enum, so one document's issues read alike whichever pass produced them. +func quotedList(names []string) string { + out := make([]string, 0, len(names)) + for _, n := range names { + out = append(out, "'"+n+"'") + } + return strings.Join(out, ", ") +} + +// iconTargetDeleted is the exporter's half of the icon rule: it asks the +// wired store whether the icon's image object is a tombstone. With no store +// wired it answers no, so a package-only export keeps every icon it is given. +func (e *exporter) iconTargetDeleted(id string) bool { + return DroppedDeletedIconRef(e.opts, id) +} diff --git a/pkg/lib/anyblockjson/iconcover_test.go b/pkg/lib/anyblockjson/iconcover_test.go new file mode 100644 index 0000000000..4734199660 --- /dev/null +++ b/pkg/lib/anyblockjson/iconcover_test.go @@ -0,0 +1,636 @@ +package anyblockjson + +// iconcover_test.go covers §2b: the typed `icon` and `cover` envelope fields. +// +// Every test here goes through a PUBLIC entry point — Marshal, Unmarshal, +// Validate — rather than calling buildIcon/applyIcon, because a test that +// calls the builder it is testing passes whatever the builder does. + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// iconSnapshot is a minimal page carrying the given details. +func iconSnapshot(details map[string]*types.Value) *model.SmartBlockSnapshotBase { + all := map[string]*types.Value{"id": str("obj1")} + for k, v := range details { + all[k] = v + } + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "obj1", Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + }, + Details: fields(all), + } +} + +// exportedIconCover marshals a details bag and hands back the two envelope +// fields as raw JSON, plus the property members that survived. +func exportedIconCover(t *testing.T, details map[string]*types.Value) (icon, cover string, props map[string]any, warnings []Issue) { + t.Helper() + opts := Options{OnWarning: func(i Issue) { warnings = append(warnings, i) }} + data, err := Marshal(model.SmartBlockType_Page, iconSnapshot(details), opts) + require.NoError(t, err) + // Marshal must never emit what its own Validate rejects (§11, I1) — the + // whole reason the export builders admit values before writing them + require.NoError(t, Validate(data), "%s", data) + + var doc struct { + Icon json.RawMessage `json:"icon"` + Cover json.RawMessage `json:"cover"` + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + return string(doc.Icon), string(doc.Cover), doc.Properties, warnings +} + +// TestExport_EveryEmittedIconShape walks the eight icon shapes and six cover +// shapes a 36 966-object corpus actually produces, each from the stored keys +// that produce it. The counts in the names are that corpus's, measured. +// +// How this can fail: change the precedence (name → emoji → file), drop the +// orthogonal colour, treat iconOption 0 as grey, or let an empty source count +// as a source, and exactly the affected rows fail. The shapes are asserted as +// whole JSON objects, so an extra or missing member fails too. +func TestExport_EveryEmittedIconShape(t *testing.T) { + for name, tc := range map[string]struct { + details map[string]*types.Value + icon string + cover string + }{ + "file — 11 956 documents": { + details: map[string]*types.Value{"iconImage": strList("bafyreicfdcmfn")}, + icon: `{"format":"file","file":"bafyreicfdcmfn"}`, + }, + "emoji — 1 376 documents": { + details: map[string]*types.Value{"iconEmoji": str("📕")}, + icon: `{"format":"emoji","emoji":"📕"}`, + }, + "named icon with a colour — 1 325 documents, always on a type": { + details: map[string]*types.Value{"iconName": str("hammer"), "iconOption": num(3)}, + icon: `{"format":"icon","name":"hammer","color":"orange"}`, + }, + "the conflict carry-over — 200 documents": { + details: map[string]*types.Value{ + "iconName": str("folder"), "iconOption": num(10), "iconEmoji": str("🌎")}, + icon: `{"format":"icon","name":"folder","color":"lime","emoji":"🌎"}`, + }, + "an avatar image and its colour — 55 documents": { + details: map[string]*types.Value{"iconImage": strList("bafybeic7zrh5fa"), "iconOption": num(5)}, + icon: `{"format":"file","file":"bafybeic7zrh5fa","color":"pink"}`, + }, + "a colour with no source — 29 documents": { + details: map[string]*types.Value{"iconOption": num(9)}, + icon: `{"format":"color","color":"teal"}`, + }, + "a named icon with no colour — 5 documents": { + details: map[string]*types.Value{"iconName": str("folder")}, + icon: `{"format":"icon","name":"folder"}`, + }, + "an emoji with a colour — 3 documents": { + details: map[string]*types.Value{"iconEmoji": str("🍷"), "iconOption": num(7)}, + icon: `{"format":"emoji","emoji":"🍷","color":"blue"}`, + }, + "the integer colour escape — 6 documents": { + details: map[string]*types.Value{"iconOption": num(13)}, + icon: `{"format":"color","color":13}`, + }, + + "a framed image cover — 73 documents": { + details: map[string]*types.Value{ + "coverId": str("bafyreigejpawpb"), "coverType": num(1), "coverY": num(-0.25)}, + cover: `{"format":"image","file":"bafyreigejpawpb","y":-0.25}`, + }, + "a bare image cover — 23 documents": { + details: map[string]*types.Value{"coverId": str("bafyreif3z354yh"), "coverType": num(1)}, + cover: `{"format":"image","file":"bafyreif3z354yh"}`, + }, + "a gradient — 14 documents": { + details: map[string]*types.Value{"coverId": str("pinkOrange"), "coverType": num(3)}, + cover: `{"format":"gradient","gradient":"pinkOrange"}`, + }, + "an unsplash image, provenance carried — 11 documents": { + details: map[string]*types.Value{ + "coverId": str("bafyreibzmdjk"), "coverType": num(5), "coverY": num(0.25)}, + cover: `{"format":"image","file":"bafyreibzmdjk","source":"unsplash","y":0.25}`, + }, + "a colour cover — 4 documents. `black` is NOT in the option palette": { + details: map[string]*types.Value{"coverId": str("black"), "coverType": num(2)}, + cover: `{"format":"color","color":"black"}`, + }, + "a fully framed image — 1 document": { + details: map[string]*types.Value{ + "coverId": str("bafyone"), "coverType": num(1), + "coverScale": num(0.5), "coverX": num(0.1), "coverY": num(0.2)}, + cover: `{"format":"image","file":"bafyone","scale":0.5,"x":0.1,"y":0.2}`, + }, + + // the discriminator is load-bearing: the SAME cover_id under two + // types is two different covers, and both spellings occur in the + // corpus + "`blue` as a colour": { + details: map[string]*types.Value{"coverId": str("blue"), "coverType": num(2)}, + cover: `{"format":"color","color":"blue"}`, + }, + "`blue` as a gradient": { + details: map[string]*types.Value{"coverId": str("blue"), "coverType": num(3)}, + cover: `{"format":"gradient","gradient":"blue"}`, + }, + } { + t.Run(name, func(t *testing.T) { + icon, cover, props, _ := exportedIconCover(t, tc.details) + assert.Equal(t, tc.icon, compactJSON(t, icon)) + assert.Equal(t, tc.cover, compactJSON(t, cover)) + for key := range tc.details { + assert.NotContains(t, props, key, "a lifted key is written nowhere else") + } + }) + } +} + +// compactJSON strips the whitespace out of a JSON fragment so a shape can be +// asserted as one readable line. It compacts the BYTES rather than re-encoding +// through a Go map, because member order is part of the canonical form (§4) +// and a map would silently sort it away. Empty in, empty out. +func compactJSON(t *testing.T, raw string) string { + t.Helper() + if raw == "" { + return "" + } + var buf bytes.Buffer + require.NoError(t, json.Compact(&buf, []byte(raw))) + return buf.String() +} + +// An EMPTY source is not a source. This is the format's largest class of fake +// ambiguity: 883 corpus objects carry both `iconEmoji` and `iconImage`, and in +// NOT ONE of them are both non-empty — the 737 "conflicts" a brief reported +// were empty siblings, present because §3 makes key presence meaningful. +// +// How this can fail: drop the emptiness guard in buildIcon/buildCover and the +// first row emits `{"format":"emoji","emoji":""}`, which Validate then refuses +// (minLength 1) — so exportedIconCover's I1 check fails too. +func TestExport_AnEmptySourceIsNotASource(t *testing.T) { + for name, details := range map[string]map[string]*types.Value{ + "an empty emoji beside an image": { + "iconEmoji": str(""), "iconImage": strList("bafyimage")}, + "an empty image list beside an emoji": { + "iconImage": strList(), "iconEmoji": str("📕")}, + } { + t.Run(name, func(t *testing.T) { + icon, _, props, _ := exportedIconCover(t, details) + assert.NotContains(t, icon, `""`, "the empty sibling is gone, not written empty") + assert.NotContains(t, icon, "[]") + assert.Empty(t, props, "and it does not survive in properties either") + }) + } + + t.Run("every source empty means no icon at all", func(t *testing.T) { + icon, cover, props, _ := exportedIconCover(t, map[string]*types.Value{ + "iconEmoji": str(""), "iconImage": strList(), "iconName": str(""), + "iconOption": num(0), "coverId": str(""), "coverType": num(0), + "coverScale": num(0), "coverX": num(0), "coverY": num(0), + }) + assert.Empty(t, icon) + assert.Empty(t, cover) + assert.Empty(t, props, "1 358 corpus objects are exactly this shape") + }) + + t.Run("iconOption 0 is the proto zero, not the first colour", func(t *testing.T) { + icon, _, _, _ := exportedIconCover(t, map[string]*types.Value{ + "iconEmoji": str("📕"), "iconOption": num(0)}) + assert.Equal(t, `{"format":"emoji","emoji":"📕"}`, compactJSON(t, icon), + "145 corpus objects carry iconOption 0; none of them is grey") + }) +} + +// A value the schema cannot hold is dropped with a warning rather than +// written, because Marshal emitting what its own Validate rejects is the +// failure nobody sees until the archive is needed (§11, I1). +// +// The cover case is real data: 33 corpus objects carry an absolute path into a +// long-gone temp directory, written by core/block/import/notion straight into +// coverId as if it were a file reference. 25% of every image cover in that +// corpus is permanently corrupt by exactly the mechanism the typed field +// exists to prevent. +// +// How this can fail: remove the isObjectRef guard and the Validate call inside +// exportedIconCover fails on `does not match pattern '^[^/]+$'`. +func TestExport_AValueTheFormatCannotWriteIsDropped(t *testing.T) { + const notionPath = "/var/folders/j0/T/anytype_notion_import/0df562e1.png" + + t.Run("a leaked filesystem path in coverId", func(t *testing.T) { + _, cover, props, warnings := exportedIconCover(t, map[string]*types.Value{ + "coverId": str(notionPath), "coverType": num(1)}) + assert.Empty(t, cover, "the cover is dropped: there is no way to write it") + assert.Empty(t, props) + require.Len(t, warnings, 1) + assert.Equal(t, "/cover", warnings[0].Path) + assert.Contains(t, warnings[0].Message, "never a URL or a path") + }) + + t.Run("a URL in iconImage falls through to the colour", func(t *testing.T) { + icon, _, _, warnings := exportedIconCover(t, map[string]*types.Value{ + "iconImage": strList("https://images.example/hero.jpg"), "iconOption": num(2)}) + assert.Equal(t, `{"format":"color","color":"yellow"}`, compactJSON(t, icon), + "an unusable source is not a source, so the chain continues") + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "not an object id") + }) + + t.Run("a cover type outside 0..5", func(t *testing.T) { + _, cover, _, warnings := exportedIconCover(t, map[string]*types.Value{ + "coverId": str("bafy"), "coverType": num(9)}) + assert.Empty(t, cover) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "not one of 0..5") + }) + + t.Run("a cover id with no type says nothing", func(t *testing.T) { + _, cover, _, warnings := exportedIconCover(t, map[string]*types.Value{ + "coverId": str("blue")}) + assert.Empty(t, cover, "`blue` under no type is a colour or a gradient and nothing says which") + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "no cover type") + }) +} + +// The conflict carry-over is the only emitted shape with two icon sources, and +// it is warned about rather than silent — 200 corpus objects, every one a +// bundled type mid-migration from an emoji to a named icon. +func TestExport_TheConflictCarryOverIsWarned(t *testing.T) { + icon, _, _, warnings := exportedIconCover(t, map[string]*types.Value{ + "iconName": str("extension-puzzle"), "iconEmoji": str("🥚"), "iconOption": num(6)}) + assert.Equal(t, `{"format":"icon","name":"extension-puzzle","color":"purple","emoji":"🥚"}`, + compactJSON(t, icon)) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "the name wins") +} + +// Every emitted shape inverts to exactly the details that produced it, which +// is what makes Export ∘ Import a fixpoint over these fields (§11 guarantees +// 2 and 3). Measured over the whole corpus: 36 966 objects, 0 byte-unstable. +// +// How this can fail: any disagreement between the export builder and the +// import inverter — a colour that maps one way and back another, a cover +// source that loses its type, an iconImage written as a scalar where the +// store holds a list — shows up either as a changed detail or as different +// bytes on the second export. +func TestRoundTrip_IconAndCoverAreAFixpoint(t *testing.T) { + for name, details := range map[string]map[string]*types.Value{ + "emoji": {"iconEmoji": str("📕")}, + "file": {"iconImage": strList("bafyimage")}, + "file with colour": {"iconImage": strList("bafyimage"), "iconOption": num(5)}, + "named icon": {"iconName": str("hammer"), "iconOption": num(3)}, + "carry-over": {"iconName": str("folder"), "iconOption": num(10), "iconEmoji": str("🌎")}, + "colour only": {"iconOption": num(9)}, + "integer colour": {"iconOption": num(13)}, + "image cover": {"coverId": str("bafycover"), "coverType": num(1), "coverY": num(-0.25)}, + "unsplash cover": {"coverId": str("bafycover"), "coverType": num(5)}, + "prebuilt cover": {"coverId": str("bafycover"), "coverType": num(4)}, + "colour cover": {"coverId": str("blue"), "coverType": num(2)}, + "gradient cover": {"coverId": str("blue"), "coverType": num(3)}, + "framed cover": {"coverId": str("bafycover"), "coverType": num(1), + "coverScale": num(0.5), "coverX": num(0.1), "coverY": num(0.2)}, + "both at once": {"iconEmoji": str("📕"), "coverId": str("pinkOrange"), "coverType": num(3)}, + } { + t.Run(name, func(t *testing.T) { + first, err := Marshal(model.SmartBlockType_Page, iconSnapshot(details), Options{}) + require.NoError(t, err) + + sbType, snap, err := Unmarshal(first, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + for key, want := range details { + got := snap.Details.Fields[key] + require.NotNil(t, got, "%q must come back", key) + if key == "iconImage" { + // a `file` relation stores a list, which is what the + // ordinary property path writes too (wrapToList) + assert.Equal(t, valueStringList(want), valueStringList(got), key) + continue + } + assert.Equal(t, want.String(), got.String(), key) + } + + second, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err) + assert.Equal(t, string(first), string(second), "Export ∘ Import must be byte-stable") + }) + } +} + +// Import refuses the nine flat spellings in `properties`, on the RESOLVED +// stored key, and the refusal names the repair. The refusal is derived from +// the export side's own lift list, never restated (deniedPropertyKey) — a +// restated list is how the two surfaces drifted apart the last time. +// +// How this can fail: drop the liftedDetailKeys arm of deniedPropertyKey and +// every row here accepts the document, storing an icon in a slot export never +// writes — so the value would be invisible to the next export. +func TestImport_TheFlatSpellingsAreRefused(t *testing.T) { + // The spellings that RESOLVE onto the lifted keys are their display + // names and their verbatim stored keys. The pre-change flat slugs + // (`icon_emoji`, `cover_id`) resolve nothing at all now — a denied + // key's fold class answers nothing — so they are ordinary custom keys + // that land on no icon slot, which the last arm pins. + for name, tc := range map[string]struct{ doc, repair string }{ + "the display name": { + doc: `{"version": 2, "properties": {"Emoji": "🔥"}}`, + repair: `"icon": {"format": "emoji", "emoji": "…"}`, + }, + "the stored key spelled verbatim": { + doc: `{"version": 2, "properties": {"iconImage": ["bafy"]}}`, + repair: `"icon": {"format": "file", "file": ""}`, + }, + "an icon name": { + doc: `{"version": 2, "properties": {"iconName": "folder"}}`, + repair: `"icon": {"format": "icon", "name": "…"}`, + }, + "an icon option": { + doc: `{"version": 2, "properties": {"iconOption": 3}}`, + repair: `the "color" member of "icon"`, + }, + "a cover id": { + doc: `{"version": 2, "properties": {"coverId": "blue"}}`, + repair: `"cover": {"format": "image"|"color"|"gradient", …}`, + }, + "cover framing": { + doc: `{"version": 2, "properties": {"coverY": -0.25}}`, + repair: `the "scale"/"x"/"y" members of an image "cover"`, + }, + // the laundering case: a legend can bind any spelling to any stored + // key, so admission runs on what the spelling RESOLVES to + "a spelling the legend binds to a lifted key": { + doc: `{"version": 2, "property_internal_keys": {"sneaky": "iconEmoji"}, + "properties": {"sneaky": "🔥"}}`, + repair: `"icon": {"format": "emoji", "emoji": "…"}`, + }, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(tc.doc)) + require.Error(t, err, "Unmarshal refuses this, so Validate must too (I2)") + assert.Contains(t, err.Error(), tc.repair, "the refusal names what to write instead") + + _, _, unmErr := Unmarshal([]byte(tc.doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr, "and the seam refuses it whatever vocabulary is wired") + }) + } + + t.Run("the retired flat slug is an ordinary custom key", func(t *testing.T) { + doc := `{"version": 2, "properties": {"icon_emoji": "🔥"}}` + require.NoError(t, Validate([]byte(doc))) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Nil(t, snap.Details.Fields["iconEmoji"], "it lands on no icon slot") + assert.NotNil(t, snap.Details.Fields["icon_emoji"], "it is its own key, verbatim") + }) +} + +// The refusal runs on the RESOLVED stored key, which is what keeps a +// space-minted relation whose OWN stored key is `icon_emoji` an ordinary +// property. 54 corpus objects are exactly this: a bundled `iconEmoji` that is +// empty, and a custom relation stored as `icon_emoji` holding a real emoji. +// Anything reading "the icon" out of `icon_emoji` in those documents today +// reads a coffee-tasting note. +// +// How this can fail: run the deny check on the SPELLING instead of the +// resolved key and this document is refused, taking a legitimate user +// relation with it. +func TestImport_ASpaceMintedIconEmojiRelationIsAnOrdinaryProperty(t *testing.T) { + doc := `{"version": 2, "id": "o1", + "property_internal_keys": {"icon_emoji": "icon_emoji"}, + "icon": {"format": "emoji", "emoji": "📕"}, + "properties": {"icon_emoji": "☕"}}` + + require.NoError(t, Validate([]byte(doc))) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + assert.Equal(t, "☕", snap.Details.Fields["icon_emoji"].GetStringValue(), + "the space's own relation keeps its value") + assert.Equal(t, "📕", snap.Details.Fields["iconEmoji"].GetStringValue(), + "and the envelope field wrote the bundled one") + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"icon": {`, "the two are visibly different things") + assert.Contains(t, string(data), `"icon_emoji": "☕"`) +} + +// The typed fields live in the ENVELOPE, and one reason is not aesthetic: +// `cover` is already a stored property key in real data — 30 corpus objects +// mint a relation whose stored key is literally `cover`, and 66 more spell it +// `pageCover`, both Notion imports, neither bundled. An envelope field is +// outside the key namespace the legend can rebind. +func TestImport_AStoredRelationNamedCoverIsUntouched(t *testing.T) { + doc := `{"version": 2, "id": "o1", + "property_internal_keys": {"cover": "cover"}, + "cover": {"format": "gradient", "gradient": "pinkOrange"}, + "properties": {"cover": "a photograph of the team"}}` + + require.NoError(t, Validate([]byte(doc))) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + assert.Equal(t, "a photograph of the team", snap.Details.Fields["cover"].GetStringValue()) + assert.Equal(t, "pinkOrange", snap.Details.Fields["coverId"].GetStringValue()) + assert.Equal(t, float64(3), snap.Details.Fields["coverType"].GetNumberValue()) +} + +// A callout carries the same typed icon, restricted to the two kinds a block +// can hold (§5.2). Shipping the envelope field without this would leave two +// icon conventions inside one document — the exact defect being removed. +func TestRoundTrip_CalloutIcon(t *testing.T) { + callout := func(emoji, image string) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"c1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "c1", Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Style: model.BlockContentText_Callout, Text: "Ship it.", + IconEmoji: emoji, IconImage: image}}}, + }, + Details: fields(map[string]*types.Value{"id": str("obj1")}), + } + } + for name, tc := range map[string]struct { + snap *model.SmartBlockSnapshotBase + want string + }{ + "an emoji": {callout("💡", ""), `{"format":"emoji","emoji":"💡"}`}, + "an image": {callout("", "bafyimage"), `{"format":"file","file":"bafyimage"}`}, + "neither": {callout("", ""), ``}, + } { + t.Run(name, func(t *testing.T) { + first, err := Marshal(model.SmartBlockType_Page, tc.snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(first), "%s", first) + + var doc struct { + Blocks []struct { + Icon json.RawMessage `json:"icon"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(first, &doc)) + require.Len(t, doc.Blocks, 1) + assert.Equal(t, tc.want, compactJSON(t, string(doc.Blocks[0].Icon))) + + sbType, snap, err := Unmarshal(first, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + second, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err) + assert.Equal(t, string(first), string(second)) + }) + } + + t.Run("the four-variant object icon is not a callout icon", func(t *testing.T) { + err := Validate([]byte(`{"version": 2, "blocks": [ + {"type": "callout", "icon": {"format": "icon", "name": "folder"}, "text": "x"}]}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "value must be one of 'emoji', 'file'") + }) +} + +// §12 promises one fault, one issue, and for a typed union that promise is the +// whole design: a model spends most of its time in the failure path, and the +// message is where the alternatives have to appear. +// +// Measured against the `oneOf` encoding both original designs specified: the +// first row below reported ELEVEN issues, three of them contradictory verdicts +// on `format`, twice telling the author to delete the CORRECT member. The +// `if`/`then` encoding reports one. `branchLeaves` cannot prune the difference +// — it prunes branches that failed on the instance's own TYPE, and every icon +// branch is `type: object`. +// +// How this can fail: re-encode the union as `oneOf` and every row here reports +// a handful of issues instead of one. +func TestValidate_AWrongIconGetsExactlyOneIssue(t *testing.T) { + for _, tc := range []struct{ doc, path, message string }{ + {`{"version":2,"icon":{"format":"emoji","emoji":"📕","name":"rocket"}}`, + "/icon/name", `property "name" is not allowed`}, + {`{"version":2,"icon":{"format":"image","url":"https://x/y.png"}}`, + "/icon/format", "value must be one of 'emoji', 'file', 'icon', 'color'"}, + {`{"version":2,"icon":{"format":"url","url":"https://x/y.png"}}`, + "/icon/format", "value must be one of 'emoji', 'file', 'icon', 'color'"}, + {`{"version":2,"icon":{"emoji":"🚀"}}`, + "/icon", "an icon is one of 'emoji', 'file', 'icon', 'color'"}, + {`{"version":2,"icon":{"format":"emoji"}}`, + "/icon", "missing property 'emoji'"}, + {`{"version":2,"icon":{"format":"icon","name":"rocket","color":"turquoise"}}`, + "/icon/color", "value must be one of 'grey', 'yellow'"}, + {`{"version":2,"icon":{"format":"file","file":"https://images.example/hero.jpg"}}`, + "/icon/file", `does not match pattern '^[^/]+$'`}, + {`{"version":2,"cover":{"format":"image","file":"/var/folders/j0/T/x.png"}}`, + "/cover/file", `does not match pattern '^[^/]+$'`}, + {`{"version":2,"cover":{"format":"unsplash","file":"bafy"}}`, + "/cover/format", "value must be one of 'image', 'color', 'gradient'"}, + {`{"version":2,"cover":{"gradient":"pinkOrange"}}`, + "/cover", "a cover is one of 'image', 'color', 'gradient'"}, + {`{"version":2,"blocks":[{"type":"callout","icon":{"emoji":"💡"},"text":"x"}]}`, + "/blocks/0/icon", "a callout icon is one of 'emoji', 'file'"}, + } { + t.Run(tc.doc, func(t *testing.T) { + err := Validate([]byte(tc.doc)) + require.Error(t, err) + var ve *ValidationError + require.ErrorAs(t, err, &ve) + require.Len(t, ve.Issues, 1, "one fault, one issue (§12): %v", ve.Issues) + assert.Equal(t, tc.path, ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, tc.message) + }) + } +} + +// The union a `format` verdict names is read out of the published schema, not +// restated in Go — which is what makes the extension seam free: a layer that +// appends a `url` variant to the schema gets it named in the reader's own +// diagnostics without touching this package (§2b). +// +// How this can fail: hardcode the variant names in iconFormatIssues and this +// test still passes on the happy path but the appended variant never appears. +func TestValidate_TheFormatUnionIsReadFromTheSchema(t *testing.T) { + assert.Equal(t, []string{"emoji", "file", "icon", "color"}, schemaFormatEnum("icon")) + assert.Equal(t, []string{"image", "color", "gradient"}, schemaFormatEnum("cover")) + assert.Equal(t, []string{"emoji", "file"}, schemaFormatEnum("plainIcon"), + "the narrowed definition answers with the narrowed set, not the one it refs") +} + +// The nine lifted keys are the whole family and nothing else, and every one is +// `hidden: true`. The hidden-ness is the entire justification for overriding +// §3's presence-is-meaningful rule: a hidden relation has no property row for +// presence to be meaningful TO. Add a visible relation to the lift list and +// this fails, which is the point. +func TestLiftedKeysAreHiddenRelations(t *testing.T) { + want := []string{ + "coverId", "coverScale", "coverType", "coverX", "coverY", + "iconEmoji", "iconImage", "iconName", "iconOption", + } + got := make([]string, 0, len(LiftedPropertyKeys())) + for k := range LiftedPropertyKeys() { + got = append(got, k) + } + assert.ElementsMatch(t, want, got) + + for _, key := range want { + rel, err := bundle.GetRelation(domain.RelationKey(key)) + require.NoError(t, err, key) + assert.True(t, rel.Hidden, "%q must be hidden, or dropping it when empty is real loss", key) + } +} + +// An icon's `file` holds TWO address spaces, and the format has to say so +// because a reader cannot tell from the slot alone. +// +// Normally it is the id of a file object in the bundle — 11,251 of 12,378 in +// a 77-space export. But a participant avatar and a space invite icon are +// stored by the app as the raw content cid of the image itself +// (core/acl/aclservice.go writes SpaceIconCid into iconImage, whose format is +// `file`), and those 992 can never resolve as object ids: a content cid is +// raw/dag-pb and begins `bafybei`, an object id is dag-cbor and begins +// `bafyrei`. A reader dereferencing one against the bundle finds nothing — +// not because the object is missing, but because it was never an object. +// +// Both shapes are legal and both must stay legal; what the format owes is the +// distinction, which now lives in $defs/objectRef and on the file variant. +// +// How this can fail: constrain the slot to one address space and 992 real +// participant icons become invalid; drop the description and a reader has no +// way to learn the rule. +func TestIcon_FileHoldsEitherAnObjectIdOrAContentCid(t *testing.T) { + for _, tc := range []struct{ name, id string }{ + {"an object id", "bafyreiarrls75xlsmbc4hwhjuht34fgkzz5xpvpkexmzqov4oxssgckohy"}, + {"a raw content cid, as a participant avatar carries", "bafybeig56mk6qmlv624q3ykecbok5v7wmzeqc5zlci2h6d42w5pg3hx6bu"}, + } { + t.Run(tc.name, func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "icon": {"format": "file", "file": "` + tc.id + `"}}` + require.NoError(t, Validate([]byte(doc))) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, strList(tc.id), snap.Details.Fields[detailKeyIconImage], + "the stored value is the same slot either way") + }) + } + + t.Run("the schema says how to tell them apart", func(t *testing.T) { + var s struct { + Defs map[string]struct { + Description string `json:"description"` + } `json:"$defs"` + } + require.NoError(t, json.Unmarshal(schemaJSON, &s)) + d := s.Defs["objectRef"].Description + assert.Contains(t, d, "bafybei", "the content-cid codec must be named") + assert.Contains(t, d, "bafyrei", "and the object-id codec beside it") + }) +} diff --git a/pkg/lib/anyblockjson/iconunion_test.go b/pkg/lib/anyblockjson/iconunion_test.go new file mode 100644 index 0000000000..c1f14f1a4d --- /dev/null +++ b/pkg/lib/anyblockjson/iconunion_test.go @@ -0,0 +1,77 @@ +package anyblockjson + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A slot that restricts the icon variants must name ITS OWN set once, not the +// object's set and then its own. `plainIcon` used to be +// `allOf: [{$ref: icon}, {format: {enum: [emoji, file]}}]`, so a value outside +// BOTH enums failed both and the reader got two contradictory verdicts, the +// WIDER one first: +// +// /blocks/0/icon/format: value must be one of 'emoji', 'file', 'icon', 'color' +// /blocks/0/icon/format: value must be one of 'emoji', 'file' +// +// A greedy repairer reads line 1, writes `format: "icon"`, and gets a fresh +// error — two round trips where one would do. That is the disease this whole +// section exists to cure, in miniature, on the surface where the wrong guess is +// most likely: `image` is both a block type name and the cover variant name. +// +// The fix splits the variant machinery (`iconVariants`) from the variant NAMES, +// so each slot states its own enum once. These tests can only fail if a slot +// starts inheriting a second enum: each asserts the COUNT and the wording, so +// re-introducing the wider verdict fails on the count even if the narrow one is +// still present. +func TestValidate_ARestrictedIconSlotNamesOneUnion(t *testing.T) { + firstIssues := func(t *testing.T, doc string) []Issue { + t.Helper() + err := Validate([]byte(doc)) + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + return ve.Issues + } + + for name, doc := range map[string]string{ + "a cover variant on a callout": `{"version": 2, "type": "page", "blocks": [ + {"type": "callout", "icon": {"format": "image", "file": "bafy1"}, "text": "x"}]}`, + "an invented format on a callout": `{"version": 2, "type": "page", "blocks": [ + {"type": "callout", "icon": {"format": "url", "url": "http://x"}, "text": "x"}]}`, + "an object-only variant on a callout": `{"version": 2, "type": "page", "blocks": [ + {"type": "callout", "icon": {"format": "icon", "name": "rocket"}, "text": "x"}]}`, + } { + t.Run(name, func(t *testing.T) { + issues := firstIssues(t, doc) + require.Len(t, issues, 1, "one fault, one issue (§12): %v", issues) + assert.Contains(t, issues[0].Message, "'emoji', 'file'", + "and it names the slot's OWN union") + assert.NotContains(t, issues[0].Message, "'icon'", + "never the object's wider one — that is what sent a repairer the wrong way") + }) + } + + // the control: the OBJECT slot still names all four, so the fix cannot pass + // by narrowing every slot to the callout's set + t.Run("the object slot still names all four", func(t *testing.T) { + issues := firstIssues(t, `{"version": 2, "type": "page", "icon": {"format": "url", "url": "http://x"}}`) + require.Len(t, issues, 1) + assert.Contains(t, issues[0].Message, "'emoji', 'file', 'icon', 'color'") + }) + + // and both slots still accept what they should + t.Run("valid icons still validate", func(t *testing.T) { + for _, doc := range []string{ + `{"version": 2, "type": "page", "icon": {"format": "emoji", "emoji": "📕"}}`, + `{"version": 2, "type": "page", "icon": {"format": "icon", "name": "rocket"}}`, + `{"version": 2, "type": "page", "blocks": [{"type": "callout", "icon": {"format": "emoji", "emoji": "📕"}, "text": "x"}]}`, + } { + require.NoError(t, Validate([]byte(doc)), fmt.Sprintf("doc: %s", doc)) + } + }) +} diff --git a/pkg/lib/anyblockjson/idcensus_test.go b/pkg/lib/anyblockjson/idcensus_test.go new file mode 100644 index 0000000000..f35d3cb487 --- /dev/null +++ b/pkg/lib/anyblockjson/idcensus_test.go @@ -0,0 +1,289 @@ +package anyblockjson + +// idcensus_test.go pins buildLabelPlan's LOCAL census (§9a): the population +// is what the export SERVES, not what the snapshot holds. objectcensus_test.go +// pins the other half — the object ids the same plan reserves. + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// censusOf runs the label plan's population probe over a snapshot. +func censusOf(sn *model.SmartBlockSnapshotBase, opts Options) map[string]bool { + e := &exporter{opts: opts, snapshot: sn, blocks: map[string]*model.Block{}, visited: map[string]bool{}} + e.indexBlocks() + return e.emittedLocalIds() +} + +// servedLocalIds reads back every doc-local id a rendered document spells: +// block ids, table row/column ids, dataview view ids. It parses the OUTPUT, +// so it is an independent statement of "what the document says" — the census +// probe walks the snapshot, this walks the bytes. +func servedLocalIds(t *testing.T, data []byte) map[string]bool { + t.Helper() + var doc struct { + Blocks []struct { + Id string `json:"id"` + Columns []struct { + Id string `json:"id"` + } `json:"columns"` + Rows []struct { + Id string `json:"id"` + } `json:"rows"` + Views []struct { + Id string `json:"id"` + } `json:"views"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + out := map[string]bool{} + add := func(id string) { + if id != "" { + out[id] = true + } + } + for _, b := range doc.Blocks { + add(b.Id) + for _, c := range b.Columns { + add(c.Id) + } + for _, r := range b.Rows { + add(r.Id) + } + for _, v := range b.Views { + add(v.Id) + } + } + return out +} + +// TestExport_CensusPopulationIsWhatExportEmits is the agreement check the +// probe rests on: the census population must be exactly the ids the served +// document spells. A recording site missed (table rows, dataview views) or a +// drop rule the probe disagrees with shows up here as a set difference. +// +// The snapshots below all carry clean ids, so the stored id and the id +// written for it are the same string — which is what lets the output be read +// back as the population. +func TestExport_CensusPopulationIsWhatExportEmits(t *testing.T) { + for _, tc := range []struct { + name string + snap *model.SmartBlockSnapshotBase + }{ + {"rich snapshot", richSnapshot()}, + {"content-less and structural blocks", &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "root", ChildrenIds: []string{"title", "ghost", "para", "orphan"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + textBlock("title", model.BlockContentText_Title, "Title"), + {Id: "ghost"}, // content-less, childless: dropped (§7) + textBlock("para", model.BlockContentText_Paragraph, "kept"), + {Id: "orphan", ChildrenIds: []string{"unreachable"}}, // content-less with children + textBlock("unreachable", model.BlockContentText_Paragraph, "under the content-less block"), + }, + Details: fields(map[string]*types.Value{"id": str("root"), "name": str("Title")}), + }}, + {"children of a leaf block are not emitted", &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "root", ChildrenIds: []string{"bm"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "bm", ChildrenIds: []string{"belowLeaf"}, + Content: &model.BlockContentOfBookmark{Bookmark: &model.BlockContentBookmark{Url: "https://anytype.io"}}}, + textBlock("belowLeaf", model.BlockContentText_Paragraph, "never served"), + }, + Details: fields(map[string]*types.Value{"id": str("root")}), + }}, + } { + t.Run(tc.name, func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, tc.snap, testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data)) + + // The census is what the round trip REBUILDS, which is what the + // document spells PLUS the cell ids a table implies. A cell + // carries no id in the flat form, but import re-derives + // `rowId-colId` from row and column ids that are spelled, so the + // same ids come back — and they have to be reserved, or a column + // compacts to a label its own cells share as a suffix in the live + // object (see emittedLocalIds). Every OTHER unspelled block — + // container, structural, content-less, unreachable — is genuinely + // gone and must stay out, which is what the two shapes below pin. + want := servedLocalIds(t, data) + for _, id := range derivedCellIdsOf(tc.snap, testOptions()) { + want[id] = true + } + assert.Equal(t, want, censusOf(tc.snap, testOptions())) + }) + } +} + +// TestExport_CompactIdsSurviveARoundTrip is guarantee 3 (§11) on the API's +// default read shape: a block the document does not spell must not reserve a +// suffix slot, or the read before a round trip and the read after it disagree +// about whether a served block compacts. +// +// The two ids below share the 5-char tail `183ba`, which is the collision a +// real production template carries between a `Layout_Div` and a paragraph. +func TestExport_CompactIdsSurviveARoundTrip(t *testing.T) { + const ( + servedId = "aaaaaaaaaaaaaaaaaaa183ba" // 24 lowercase hex: relabels + ghostId = "bbbbbbbbbbbbbbbbbbb183ba" // same tail, never served + ) + for _, tc := range []struct { + name string + ghost *model.Block + extra []*model.Block + }{ + // the production shape: a Layout_Div wrapper and a paragraph whose + // minted ids end in the same five characters + {"transparent container", divBlock(ghostId, "inner"), + []*model.Block{textBlock("inner", model.BlockContentText_Paragraph, "inside the container")}}, + {"structural block", textBlock(ghostId, model.BlockContentText_Title, "Title"), nil}, + {"content-less leaf", &model.Block{Id: ghostId}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: append([]*model.Block{ + {Id: "root", ChildrenIds: []string{ghostId, servedId}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + tc.ghost, + textBlock(servedId, model.BlockContentText_Paragraph, "served"), + }, tc.extra...), + Details: fields(map[string]*types.Value{"id": str("root"), "name": str("Title")}), + } + opts := testOptions() + opts.CompactIds = true + + first, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + require.NoError(t, Validate(first)) + // the positive half: the ghost does not hold the slot, so the + // served block relabels. Without this the test would pass on a + // document that simply never compacts anything. + assert.Contains(t, string(first), `"id": "183ba"`) + + _, reimported, err := Unmarshal(first, opts) + require.NoError(t, err) + second, err := Marshal(model.SmartBlockType_Page, reimported, opts) + require.NoError(t, err) + + assert.Equal(t, string(first), string(second), + "Export(S) and Export(Import(Export(S))) must agree (§11 guarantee 3)") + }) + } +} + +// derivedCellIdsOf lists the `rowId-colId` ids a snapshot's tables imply. It +// walks the snapshot itself rather than calling the exporter's own helper, so +// it is an INDEPENDENT statement of what the census owes — the same reason +// servedLocalIds parses the output bytes instead of asking the exporter. +// +// Shape (§6.1): a table's children are two layout wrappers, TableColumns and +// TableRows; the column ids and row ids are their children, and every +// (row, column) pair implies a cell. +func derivedCellIdsOf(snap *model.SmartBlockSnapshotBase, _ Options) []string { + byId := map[string]*model.Block{} + for _, b := range snap.GetBlocks() { + if b != nil && b.Id != "" { + byId[b.Id] = b + } + } + var out []string + for _, b := range snap.GetBlocks() { + if b == nil { + continue + } + if _, isTable := b.Content.(*model.BlockContentOfTable); !isTable { + continue + } + var cols, rows []string + for _, wrapperId := range b.ChildrenIds { + w := byId[wrapperId] + l, ok := w.GetContent().(*model.BlockContentOfLayout) + if !ok { + continue + } + switch l.Layout.GetStyle() { + case model.BlockContentLayout_TableColumns: + cols = append(cols, w.ChildrenIds...) + case model.BlockContentLayout_TableRows: + rows = append(rows, w.ChildrenIds...) + } + } + for _, r := range rows { + for _, c := range cols { + out = append(out, r+"-"+c) + } + } + } + return out +} + +// countingOptions counts how many times export asks it to name an option, +// which is how a test can SEE the census probe: the probe is a second full +// block emit, so a wired resolver is asked twice. The rich fixture's dataview +// carries select filter values, so this hook is genuinely reached. +type countingOptions struct { + n *int + inner OptionResolver +} + +func (c countingOptions) OptionName(key domain.RelationKey, id string) (string, bool) { + *c.n++ + return c.inner.OptionName(key, id) +} +func (c countingOptions) OptionId(key domain.RelationKey, name string) (string, bool) { + return c.inner.OptionId(key, name) +} + +// The census probe (emittedLocalIds) is a SECOND full block emit, and the most +// expensive thing a compact export does. OmitIds writes no id at all, so a +// label plan has nothing to label — running the probe for that combination +// costs a whole extra emit for output that carries no ids. +// +// The bytes are byte-identical either way, which is exactly why the waste went +// unnoticed. So this counts RESOLVER CALLS through the public Marshal instead: +// the probe asks the resolver a second time, and that is observable. A test +// that re-implemented the gate would pass no matter what the export does — +// this one fails when the gate is removed. +func TestExport_NoCensusProbeWhenNoIdIsWritten(t *testing.T) { + snap := richSnapshot() + + callsFor := func(opts Options) int { + n := 0 + opts.ResolveFormat = testFormatResolver + opts.ResolveOptions = countingOptions{n: &n, inner: testResolver} + _, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + return n + } + + plain := callsFor(Options{OmitIds: true}) + compact := callsFor(Options{OmitIds: true, CompactIds: true}) + require.NotZero(t, plain, "the fixture must reach the resolver, or this proves nothing") + assert.Equal(t, plain, compact, + "OmitIds writes no id, so compaction must not run the census probe") + + // the control: WITHOUT OmitIds, compaction does run the probe, so the + // assertion above cannot pass by never probing at all + withIds := callsFor(Options{}) + withIdsCompact := callsFor(Options{CompactIds: true}) + assert.Greater(t, withIdsCompact, withIds, + "a compact read still pays for its census (the probe is a second emit)") + + // and the bytes are unchanged by the gate + a, err := Marshal(model.SmartBlockType_Page, snap, Options{OmitIds: true}) + require.NoError(t, err) + b, err := Marshal(model.SmartBlockType_Page, snap, Options{OmitIds: true, CompactIds: true}) + require.NoError(t, err) + assert.Equal(t, string(a), string(b), + "OmitIds decides the ids; compaction adds nothing to decide") +} diff --git a/pkg/lib/anyblockjson/import.go b/pkg/lib/anyblockjson/import.go new file mode 100644 index 0000000000..f87025f9ef --- /dev/null +++ b/pkg/lib/anyblockjson/import.go @@ -0,0 +1,1418 @@ +package anyblockjson + +// import.go reconstructs a snapshot from a validated AnyBlock JSON document +// (§2–§7, §9). + +import ( + "encoding/json" + "fmt" + "sort" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func jsonUnmarshal(data []byte, v any) error { + return json.Unmarshal(data, v) +} + +type jsonDoc struct { + Schema string `json:"$schema"` + // json.Number for the same reason as every other schema-integer field + // below: 2.0 and 2e0 are integers to JSON Schema, so checkVersion accepts + // them, and a plain int would then fail to decode a document Validate + // declared valid. + Version json.Number `json:"version"` + Kind string `json:"kind"` + Id string `json:"id"` + Type string `json:"type"` + TemplateFor string `json:"template_for"` + InternalKey string `json:"internal_key"` + // PropertySettings is a kind:property document's definition group (§2d): + // one propertyDefinition, whose three travelling members stand for the + // stored relation-definition keys that `properties` refuses. + PropertySettings *jsonPropertySettings `json:"property_settings"` + // Icon and Cover are the typed envelope fields (§2b). Each is one object + // whose `format` member selects the variant, and each stands for a family + // of hidden stored keys that `properties` refuses. + Icon *Icon `json:"icon"` + Cover *Cover `json:"cover"` + Properties map[string]any `json:"properties"` + // TypeSettings is a kind:object_type document's definition group (§2a): + // the five lifted settings plus property_definitions. + TypeSettings *jsonTypeSettings `json:"type_settings"` + // PropertyKeys is the §3 spelling→stored-key legend: what this document says + // its own key spellings mean, consulted before any vocabulary so a reader + // without the space still lands on the right relation. Its values are + // AUTHORITATIVE — taken as the stored key, not liveness-checked (§3). + PropertyKeys map[string]string `json:"property_internal_keys"` + // TypeKeys is the same legend for the TYPE namespace — separate map, + // because a space may name a relation and a type one word (§3). + TypeKeys map[string]string `json:"type_internal_keys"` + // OptionIds is the §9a option legend, nested {property spelling: {option + // name: option id}}. Unlike the two above its values are HINTS, honoured + // only where the id still names a live option of that relation (§3). + OptionIds map[string]map[string]string `json:"option_ids"` + Blocks []*jsonBlock `json:"blocks"` + Items []string `json:"items"` + Store map[string]any `json:"store"` + Root *jsonRootEscape `json:"root"` +} + +// jsonPropertySettings is the decoded `property_settings` group (§2d). The +// two RawMessage members are raw because each has THREE states the schema +// admits — absent, null, and a value — and a decoded Go pointer collapses +// the first two: member presence mirrors stored-key presence exactly, and a +// stored null is a value (§3, 80 production relations hold one). +type jsonPropertySettings struct { + Format string `json:"format"` + IncludeTime json.RawMessage `json:"include_time"` + TargetTypes json.RawMessage `json:"object_types"` +} + +type jsonRootEscape struct { + Fields map[string]any `json:"fields"` + BackgroundColor string `json:"background_color"` +} + +// jsonBlock is the union of every §5 block shape; the schema guarantees only +// type-appropriate fields are present. +type jsonBlock struct { + // json.Number, not int, for every schema-integer field: JSON Schema counts + // 2048.0 and 1e3 as integers, so Validate accepts them, and a typed int + // field would then fail to decode a document Validate declared valid — + // reported as a bare Go decode error with no JSON pointer, outside the + // path-addressed error contract (§13). The schema bounds each field to the + // stored type's range, so the conversion helpers cannot truncate. + Indent json.Number `json:"indent"` + Id string `json:"id"` + Type string `json:"type"` + + Checked bool `json:"checked"` + Color string `json:"color"` + Text string `json:"text"` + Language string `json:"language"` + Icon *Icon `json:"icon"` + + ObjectId string `json:"object_id"` + Name string `json:"name"` + MimeType string `json:"mime_type"` + Size json.Number `json:"size"` + Style string `json:"style"` + AddedAt string `json:"added_at"` + Hash string `json:"hash"` + Url string `json:"url"` + CardStyle string `json:"card_style"` + IconSize string `json:"icon_size"` + Description string `json:"description"` + Properties json.RawMessage `json:"properties"` // link: []string; dataview: []jsonDvProperty + Processor string `json:"processor"` + Property string `json:"property"` + + Layout string `json:"layout"` + Limit json.Number `json:"limit"` + ViewId string `json:"view_id"` + AutoAdded bool `json:"auto_added"` + + Columns []jsonTableColumn `json:"columns"` + Rows []jsonTableRow `json:"rows"` + + IsCollection bool `json:"is_collection"` + Source []string `json:"source"` + Views []jsonView `json:"views"` + + Align string `json:"align"` + VerticalAlign string `json:"vertical_align"` + BackgroundColor string `json:"background_color"` + Fields map[string]any `json:"fields"` +} + +// Unmarshal validates data and reconstructs a snapshot (§13). Errors wrap +// *ValidationError with JSON-path-addressed issues. +func Unmarshal(data []byte, opts Options) (model.SmartBlockType, *model.SmartBlockSnapshotBase, error) { + if _, err := validateToDoc(data, opts.NormalizeIndent, opts.OnWarning); err != nil { + return 0, nil, err + } + var doc jsonDoc + if err := json.Unmarshal(data, &doc); err != nil { + return 0, nil, fmt.Errorf("decode document: %w", err) + } + imp := &importer{opts: opts, doc: &doc} + return imp.build() +} + +type importer struct { + opts Options + doc *jsonDoc + // usedIds is every id this document will contain: the ones it authored + // (envelope, blocks, table rows and columns, derived cell ids) plus the + // ones minted since. One set for all of them, because they end up in one + // block graph — a generated id that lands on an authored one produces two + // blocks with the same id, which validation has already finished checking + // by the time it happens (§9). + usedIds map[string]struct{} + // refusal is the first key slot whose resolution named nothing — see + // propertyKeyAt. Deferred because the slots that can produce it are deep + // inside block construction; build turns it into the ValidationError + // before it returns a snapshot. + refusal *Issue + // foldedUnrebuilt records that this document carries a FOLDED participant + // reference (§9) this reader cannot rebuild, because it names no space. + // Set during the walk and reported once in build: the fault is the + // reader's wiring, not any one slot's, and a document can hold thousands + // of them. + foldedUnrebuilt bool + // scopeType is the resolved stored key of the document's declared type — + // for a template, the TARGET type, whose instances the template's + // properties describe. It is the disambiguating scope for a shared + // property name (§3): a legendless document naming a property two live + // properties answer to resolves against this type's own property list + // first, and errors loudly when the type is not enough. Set by build + // right after the type slots resolve, before any property slot is read. + scopeType string + // warnedPropertyTerms / warnedTypeTerms deduplicate the two + // verbatim-resolution warnings (the phantom-key and glued-annotation + // diagnoses): one term can be named by a dozen slots, and the diagnosis + // is a fact about the term, not about any one slot. + warnedPropertyTerms map[string]bool + warnedTypeTerms map[string]bool + // propLegend/typeLegend/optLegend are the document's legends expanded to + // also answer for the NFC form of any non-NFC-spelled entry (§3, + // nfcExpandLegend), built once on first use. legendsBuilt marks them, + // because the fast path hands the doc's own maps back and a nil legend + // stays nil. + propLegendNFC map[string]string + typeLegendNFC map[string]string + optLegendNFC map[string]map[string]string + legendsBuilt bool +} + +// propertyLegend / typeLegend / optionLegend are the §3 chain-step-1 tables: +// the document's own legends, with non-NFC spellings also answering under +// their canonical form. Values pass byte-verbatim — a legend value is a +// stored key, and a stored key's bytes are its address. +func (imp *importer) propertyLegend() map[string]string { + imp.buildLegends() + return imp.propLegendNFC +} + +func (imp *importer) typeLegend() map[string]string { + imp.buildLegends() + return imp.typeLegendNFC +} + +func (imp *importer) optionLegend() map[string]map[string]string { + imp.buildLegends() + return imp.optLegendNFC +} + +func (imp *importer) buildLegends() { + if imp.legendsBuilt { + return + } + imp.legendsBuilt = true + imp.propLegendNFC = nfcExpandLegend(imp.doc.PropertyKeys) + imp.typeLegendNFC = nfcExpandLegend(imp.doc.TypeKeys) + imp.optLegendNFC = nfcExpandLegend(imp.doc.OptionIds) +} + +// claimAuthoredIds records every id the document names before anything is +// generated. It has to run before the first genId call, which is why build +// calls it first rather than leaving it to the lazy path. +func (imp *importer) claimAuthoredIds() { + imp.usedIds = map[string]struct{}{} + var walk func(jbs []*jsonBlock) + walk = func(jbs []*jsonBlock) { + for _, jb := range jbs { + if jb == nil { + continue + } + imp.claimId(jb.Id) + for _, col := range jb.Columns { + imp.claimId(col.Id) + } + for _, row := range jb.Rows { + imp.claimId(row.Id) + // a cell's id is derived (§6.1) but it is still a block id, + // and the table owns the whole grid whether or not the cell is + // written: the editor materializes the missing cell at exactly + // that id the first time it is filled, so a generated id may + // not be sitting on it. This is the same claim validation + // makes (§4). + for _, col := range jb.Columns { + if row.Id != "" && col.Id != "" { + imp.claimId(row.Id + "-" + col.Id) + } + } + for _, cell := range row.Cells { + if cell.Block != nil { + walk([]*jsonBlock{cell.Block}) + } + walk(cell.Blocks) + } + } + } + } + imp.claimId(imp.doc.Id) + walk(imp.doc.Blocks) +} + +func (imp *importer) claimId(id string) string { + if id == "" { + return id + } + if imp.usedIds == nil { + imp.claimAuthoredIds() + } + imp.usedIds[id] = struct{}{} + return id +} + +func (imp *importer) idTaken(id string) bool { + if imp.usedIds == nil { + imp.claimAuthoredIds() + } + _, taken := imp.usedIds[id] + return taken +} + +// genId mints an id no other id in the document uses. The generator belongs to +// the caller — the convert wiring derives ids from a file path and a counter, +// both halves author-controlled — so its answer is disambiguated rather than +// trusted. +func (imp *importer) genId() string { + mint := defaultGenerateId + if imp.opts.GenerateId != nil { + mint = imp.opts.GenerateId + } + return imp.claimId(uniqueLabel(mint(), imp.idTaken)) +} + +// propertyKey inverts a key slot: the document's own legend first (§3), then +// the vocabulary in force. The legend wins because it is the only statement +// made by the document itself — a vocabulary belongs to the reader, and two +// readers disagreeing about a spelling is exactly how a property ends up pointing +// at a different relation than it was exported from. +// +// Names are not unique, so a space-backed vocabulary (ScopedKeyVocabulary, +// discovered by assertion) adds two steps a legendless term needs: +// +// - **A shared name resolves within the declared type.** Two live +// properties bearing one name is an ambiguity the plain vocabulary +// refuses; the document's type is the disambiguating scope, and a name +// unambiguous among the type's own properties resolves there. A name +// the type cannot place raises a LOUD error asking for the legend — +// never a guess, and never a phantom key minted while two live +// properties bear that exact name. +// - **A verbatim resolution is diagnosed.** A term that is no live +// entity's stored key is stored verbatim all the same — that is chain +// step 5, and the price of any name-addressed scheme — but the importer +// says so once per term: a stale or guessed name minting a phantom key, +// or an annotation glued onto a copied name. +// +// The bare form resolves for a caller with no pointer to offer, and reports +// an ambiguity against the `properties` member — the slot the overwhelming +// majority of property spellings are read from. propertyKeyIn is the form +// that names its own slot. +func (imp *importer) propertyKey(slug string) string { + return imp.propertyKeyIn(slug, "/properties") +} + +// propertyKeyIn is propertyKey with the JSON pointer of the slot that spelled +// the term, used for one thing: the ambiguity refusal below. +// +// It used to report every ambiguity at `/property_internal_keys`, which reads +// as a pointer and is not one — that member is ABSENT in precisely the +// documents that reach the refusal, because a legend entry is what would have +// settled the spelling and the whole complaint is that none was written. A +// reader following the pointer arrives at nothing and has no way back to the +// slot that named the term. The message still asks for the legend entry, which +// is the repair; the pointer names the fault's location, which is the slot. +// +// The pointer a caller passes is only ever as precise as the slot it read +// from: the detail seam knows the exact member, the block slots do not (they +// are built without a pointer, and the empty-key refusal beside them reports +// the coarse `/blocks` for the same reason — a coarse true pointer beats a +// precise-looking wrong one). +func (imp *importer) propertyKeyIn(slug, path string) string { + if key, ok := imp.propertyLegend()[slug]; ok && key != "" { + return key + } + // §3: a spelling resolves under its canonical NFC form (nfcTerm) — after + // the two steps that legitimately bind exact non-NFC bytes: the legend + // above (export writes an identity entry for a stored key it spells + // verbatim, and the expanded legend already answered for the NFC form) + // and the verbatim-first stored-key rule here (chain step 2 — a stored + // key's bytes are its address, whatever their normal form). + if n := nfcTerm(slug); n != slug { + if scoped, ok := imp.opts.keys().(ScopedKeyVocabulary); ok && + scoped.PropertyTermFacts(slug).LiveStoredKey { + return slug + } + slug = n + if key, ok := imp.propertyLegend()[slug]; ok && key != "" { + return key + } + } + scoped, ok := imp.opts.keys().(ScopedKeyVocabulary) + if !ok { + key := imp.opts.propertyKey(slug) + if key == slug { + imp.warnVerbatimPropertyTerm(slug, nil) + } + return key + } + facts := scoped.PropertyTermFacts(slug) + if facts.LiveStoredKey { + return slug // chain step 2: an exact stored key wins, verbatim + } + cands := distinctKeys(scoped.PropertyKeyCandidates(slug)) + switch len(cands) { + case 1: + return cands[0] + case 0: + key := imp.opts.propertyKey(slug) // the fold layer, then verbatim + if key == slug { + imp.warnVerbatimPropertyTerm(slug, &facts) + } + return key + } + if imp.scopeType != "" { + var inScope []string + for _, key := range distinctKeys(scoped.TypePropertyKeys(imp.scopeType)) { + for _, c := range cands { + if c == key { + inScope = append(inScope, c) + break + } + } + } + if len(inScope) == 1 { + return inScope[0] + } + } + imp.refuse(path, fmt.Sprintf( + "the spelling %q names %d live properties in this space and the declared "+ + "type does not single one out; add a %s entry binding the spelling to "+ + "the intended stored key", slug, len(cands), memberPropertyInternalKeys)) + return slug +} + +// distinctKeys is what makes a vocabulary's two list answers behave as the +// SETS ScopedKeyVocabulary says they are. +// +// Both lists are read as COUNTS — "how many live entities answer to this +// spelling", "does the declared type single one of them out" — and a count is +// the one thing a bookkeeping slip in the producer can falsify while leaving +// every key in the list correct. One entity listed twice then reads as two, +// the importer refuses a document its own exporter had just written, and the +// reader has nothing to compare the list against to notice. The refusal is +// also the unrecoverable outcome: a resolution can be overridden with a legend +// entry, a refusal stops the import. +// +// storeresolver keeps both lists sets at the source (addClaimant, +// TypePropertyKeys) and that is where the fix belongs; this is the reader's +// half of the same guarantee, because ScopedKeyVocabulary is a public +// interface and Options.Keys accepts an implementation from anyone. Cost is +// one map per ambiguous term. Order is preserved, so the sorted list a +// conforming vocabulary returns stays sorted. +func distinctKeys(keys []string) []string { + if len(keys) < 2 { + return keys + } + seen := make(map[string]bool, len(keys)) + out := make([]string, 0, len(keys)) + for _, key := range keys { + if seen[key] { + continue + } + seen[key] = true + out = append(out, key) + } + return out +} + +// warnVerbatimPropertyTerm reports, once per term, what a verbatim +// resolution means when the term is nobody's stored key. facts == nil means +// the reader has no liveness knowledge (the bundled-only vocabulary), so +// only the glued-annotation check — answerable from the shipped table — +// runs; the phantom diagnosis needs a space to ask. +func (imp *importer) warnVerbatimPropertyTerm(term string, facts *KeyTermFacts) { + if imp.warnedPropertyTerms[term] { + return + } + if imp.warnedPropertyTerms == nil { + imp.warnedPropertyTerms = map[string]bool{} + } + imp.warnedPropertyTerms[term] = true + if name, ok := BundledPropertyNameExtendedBy(term); ok { + imp.warn("", "the property spelling %q extends the bundled property name %q "+ + "with trailing text — an annotation glued onto a copied name? It is "+ + "stored verbatim, as its own key", term, name) + return + } + if facts == nil { + return + } + if facts.ExtendsName != "" { + imp.warn("", "the property spelling %q extends the live property name %q "+ + "with trailing text — an annotation glued onto a copied name? It is "+ + "stored verbatim, as its own key", term, facts.ExtendsName) + return + } + imp.warn("", "the property spelling %q is not the name or stored key of any "+ + "live property in this space and is stored verbatim — a stale or guessed "+ + "name mints a phantom key", term) +} + +// propertyKeyAt is propertyKey at a slot that can REFUSE — every key slot +// outside `/properties`, which runs its own admission at the detail seam. +// +// The rule is one sentence, and it is the same one the document side now +// carries at every slot: **a key slot has to name something**. The document +// half is the schema (`minLength: 1`); this is the resolution half, and it is +// reachable only through Options.Keys, which §3 accepts from anyone: a +// vocabulary answering ("", true) for a non-empty spelling. `/properties`, +// `/type`, `/template_for` and a property definition's `object_types` refused that +// from the start; the nine slots that did not stored the empty key and then +// LOST the slot on the way back out — a column and a sort vanish, a property +// block and a link's shown-property list come back nameless, a filter +// re-exports as a node that filters on nothing. +// +// The refusal is deferred rather than returned, because these slots sit deep +// inside block construction (dataview views, filter trees) where an error +// return would have to be threaded through a dozen signatures for a fault +// none of them can repair. build reports the first one before it hands back a +// snapshot, so no caller ever sees the damaged object. +// The `slot` names which kind of slot was reading, and the message names the +// SPELLING rather than leaning on the pointer: the fault is in the reader's +// vocabulary, not in the document, so "which spelling does your table answer +// nothing for" is the question a caller can act on. The pointer stays coarse +// (`/blocks`) because these slots are built without one and inventing a +// precise-looking pointer that is wrong is worse than a coarse true one. +func (imp *importer) propertyKeyAt(slug, slot string) string { + key := imp.propertyKeyIn(slug, "/blocks") + if slug != "" && key == "" { + imp.refuse("/blocks", fmt.Sprintf( + "the vocabulary resolves the %s spelling %q to the empty key; "+ + "a key slot has to name something", slot, slug)) + } else if slug != "" && !isWritablePropertyKey(key) { + // the same seam /properties has always run (§3): a vocabulary can + // resolve a legal spelling onto a key no spelling can carry — over + // the bound, or holding control bytes — which export can only DROP + // on the way back out. Validate cannot see this (it takes no + // vocabulary), so the codec is the door that refuses. + imp.refuse("/blocks", fmt.Sprintf("the %s spelling %q resolves onto a key "+ + "this format cannot write back: %s", + slot, slug, unwritableKeyReason("resolved property key", key))) + } + return key +} + +// propertyKeysAt is the list form (a link block's shown properties). +func (imp *importer) propertyKeysAt(slugs []string, slot string) []string { + if len(slugs) == 0 { + return slugs + } + out := make([]string, len(slugs)) + for i, slug := range slugs { + out[i] = imp.propertyKeyAt(slug, slot) + } + return out +} + +// refuse records the first key-slot refusal. First and not all of them: the +// issue list §12 promises is one fault per slot, and a vocabulary answering +// "" answers "" everywhere, so collecting them would print the same fault +// once per slot in the document. +func (imp *importer) refuse(path, message string) { + if imp.refusal == nil { + imp.refusal = &Issue{Path: path, Message: message} + } +} + +// typeKey inverts a TYPE key slot: the document's own legend first (§3), +// then the vocabulary in force — propertyKey on the type namespace's legend. +// +// It used to carry a reservation, the mirror of writableTypeSlug's: the +// vocabulary could not move the `template` spelling in either direction, +// because the kind derivation and the /template_for gate read the same field +// through a DIFFERENT chain (docTypeKey — the document's own, deliberately +// blind to the reader's vocabulary, because Validate has no vocabulary and +// §12 requires the two to agree). A vocabulary answering +// TypeKey("template") == "69bbfc…" therefore produced a Template smartblock +// whose ObjectTypeKeys do not contain `template` — which every downstream +// template check misses, since they all test +// lo.Contains(ObjectTypeKeys, TypeKeyTemplate). +// +// `kind` answers both questions now, off a field no chain touches, so the two +// halves cannot disagree and there is nothing left to reserve. The path is +// still the SLOT being read, because the empty-key refusal below fires in +// three of them: the envelope `type`, `template_for`, and every +// `type_settings.property_definitions[i].object_types[j]` (§2a). +func (imp *importer) typeKey(slug, path string) string { + if key, ok := imp.typeLegend()[slug]; ok && key != "" { + return key + } + // §3's canonical form, in the same order as propertyKeyIn: exact legend, + // exact stored key, then the NFC form + if n := nfcTerm(slug); n != slug { + if scoped, ok := imp.opts.keys().(ScopedKeyVocabulary); ok && + scoped.TypeTermFacts(slug).LiveStoredKey { + return slug + } + slug = n + if key, ok := imp.typeLegend()[slug]; ok && key != "" { + return key + } + } + scoped, ok := imp.opts.keys().(ScopedKeyVocabulary) + if !ok { + key := imp.opts.typeKey(slug) + if key == slug { + imp.warnVerbatimTypeTerm(slug, nil) + } + return key + } + facts := scoped.TypeTermFacts(slug) + if facts.LiveStoredKey { + return slug + } + cands := distinctKeys(scoped.TypeKeyCandidates(slug)) + switch len(cands) { + case 1: + return cands[0] + case 0: + key := imp.opts.typeKey(slug) + if key == slug { + imp.warnVerbatimTypeTerm(slug, &facts) + } + return key + } + // a shared TYPE name has no wider scope to resolve inside — the type is + // the scope — so the ambiguity is refused outright, the same loud error + // a shared property name gets when its type cannot place it + imp.refuse(path, fmt.Sprintf( + "the spelling %q names %d live types in this space; add a %s entry "+ + "binding the spelling to the intended stored key", + slug, len(cands), memberTypeInternalKeys)) + return slug +} + +// warnVerbatimTypeTerm is warnVerbatimPropertyTerm on the type namespace. +func (imp *importer) warnVerbatimTypeTerm(term string, facts *KeyTermFacts) { + if imp.warnedTypeTerms[term] { + return + } + if imp.warnedTypeTerms == nil { + imp.warnedTypeTerms = map[string]bool{} + } + imp.warnedTypeTerms[term] = true + if name, ok := BundledTypeNameExtendedBy(term); ok { + imp.warn("", "the type spelling %q extends the bundled type name %q with "+ + "trailing text — an annotation glued onto a copied name? It is stored "+ + "verbatim, as its own key", term, name) + return + } + if facts == nil { + return + } + if facts.ExtendsName != "" { + imp.warn("", "the type spelling %q extends the live type name %q with "+ + "trailing text — an annotation glued onto a copied name? It is stored "+ + "verbatim, as its own key", term, facts.ExtendsName) + return + } + imp.warn("", "the type spelling %q is not the name or stored key of any live "+ + "type in this space and is stored verbatim — a stale or guessed name "+ + "mints a phantom key", term) +} + +// warn reports a warning-grade issue through the caller's sink (§13) — the +// import half of exporter.warn. Silent when no sink is wired. +func (imp *importer) warn(path, format string, args ...any) { + if imp.opts.OnWarning == nil { + return + } + imp.opts.OnWarning(Issue{Path: path, Message: fmt.Sprintf(format, args...)}) +} + +// declaredFormat maps a document's format name to a stored format. "text" +// is deliberately ambiguous (§3): it names both longtext and the legacy +// shorttext, so for that one name the property's *existing* format decides, +// which is what keeps a bundled short-text property (name, iconEmoji, …) +// from being rewritten to longtext on every round-trip. An absent or +// unrecognized name resolves the same way — it too lands on longtext. Every +// other name is taken literally: the document is authoritative about which +// format a property has, and only the text/text collapse needs repairing. +func (imp *importer) declaredFormat(key, name string) model.RelationFormat { + return declaredFormatWith(imp.opts, key, name) +} + +// declaredFormatWith is that rule with nothing but Options behind it, because +// the §2a array arrives through TWO doors and the rule is the array's, not the +// document's: BuildRecommendedLists — the API's PATCH-type channel — read the +// name literally, so `{"property": "name", "format": "text"}` created the bundled +// `name` property as longtext through one door and kept it shorttext through +// the other. The whole point of the collapse is that `text` resolves per key +// (§3); a door that skips the resolution re-introduces exactly the loss the +// collapse was designed not to have, and the two doors then disagree about +// what one array means. +func declaredFormatWith(opts Options, key, name string) model.RelationFormat { + // An ABSENT format is not a declaration of text. `format` is optional + // in both slots that carry it, so a document that omits it has said + // nothing about the property — and the answer to nothing is the chain + // (§3), not longtext. Treating absence as `text` silently OVERRODE the + // bundled table: `{"property": "due_date"}` in a dataview's property list + // pinned a bundled DATE property to longtext, so its filters stopped + // being dates, while omitting the list entirely resolved correctly. + // Listing a property without its format was worse than not listing it + // at all — the opposite of what any author would assume, and reported + // by nothing. + // + // Canonical export always writes a format (formatName answers "text" + // even for longtext), so absence only ever arrives from a hand-written + // document — exactly the population that means "I did not say". A + // declared "text" still stands, and still folds per key below. + if name == "" { + if resolved, ok := resolveFormatWith(opts, key); ok { + return resolved + } + return model.RelationFormat_longtext + } + f := formatNames.value(name) + if f != model.RelationFormat_longtext { + return f + } + if resolved, ok := resolveFormatWith(opts, key); ok && resolved == model.RelationFormat_shorttext { + return resolved + } + return f +} + +func (imp *importer) resolveFormat(key string) (model.RelationFormat, bool) { + return resolveFormatWith(imp.opts, key) +} + +func (imp *importer) build() (model.SmartBlockType, *model.SmartBlockSnapshotBase, error) { + doc := imp.doc + imp.claimAuthoredIds() + // `kind` is the whole of it (§2). There is no derivation from the type + // term any more: the spelling `template` resolved through the document's + // own chain used to mean Template, which meant the same field answered + // two unrelated questions — which type this object has, and what kind of + // object it is — and only stayed consistent by forbidding every reader + // from spelling that one type key differently. An absent `kind` on a + // document whose type is literally `template` was the legacy spelling, + // and Validate refused it by name until the freeze; the version gate + // answers for every pre-freeze document now, so at version 2 that + // document arrives here and is a Page, exactly as its kind says + // (§10, §15 #9). + sbType := model.SmartBlockType_Page + if doc.Kind != "" { + sbType = kindNames.value(doc.Kind) + } + + // the envelope id goes through the reference reader like any object + // reference (§9): a stray informative suffix is trimmed, and a bare + // identity — the participant document's own folded id — rebuilds this + // space's participant id. Claimed so a generated block id cannot land on + // the rebuilt form. + objectId := imp.claimId(imp.objectRef(doc.Id)) + if objectId == "" { + objectId = imp.genId() + } + + var objectTypes []string + if doc.Type != "" { + // the seam refuses a resolution onto the empty key (§3): a + // vocabulary can answer "" for a non-empty spelling, which became + // the ObjectTypes entry "ot-" and re-exported as no type at all — + // silently. That is the only refusable resolution here: a non-empty + // stored key of any shape round-trips verbatim, unlike a property + // key, which has to survive as a JSON member name. + typeKey := imp.typeKey(doc.Type, "/type") + if typeKey == "" { + return 0, nil, &ValidationError{Issues: []Issue{{ + Path: "/type", + Message: unwritableKeyReason("resolved type key", typeKey), + }}} + } + objectTypes = append(objectTypes, domain.TypeKey(typeKey).URL()) + // the declared type is the disambiguating scope for a shared + // property name (propertyKey); set before any property slot reads + imp.scopeType = typeKey + if sbType == model.SmartBlockType_Template && doc.TemplateFor != "" { + target := imp.typeKey(doc.TemplateFor, "/template_for") + if target == "" { + return 0, nil, &ValidationError{Issues: []Issue{{ + Path: "/template_for", + Message: unwritableKeyReason("resolved type key", target), + }}} + } + objectTypes = append(objectTypes, domain.TypeKey(target).URL()) + // a template's properties describe the TARGET type's instances, + // so the target is the scope that can disambiguate them + imp.scopeType = target + } + } + + details := &types.Struct{Fields: map[string]*types.Value{}} + details.Fields[detailKeyId] = &types.Value{Kind: &types.Value_StringValue{StringValue: objectId}} + // Sorted, and a REFUSAL when two spellings canonicalize onto one stored + // key — the mirror of the export-side collapse guard (§3's + // duplicate-binding refusal). Ranging the + // map made "which of two spellings wins" a per-run coin flip: the same + // request stored a different object run to run. The API layer refuses + // first, with a better-worded message (canonicalizeDocumentKeys), but the + // type-create channel skips it by design and so does every direct package + // caller (cmd/anyblockroundtrip, cmd/anyblockrecover, + // cmd/internal/anyblockbatch) — so the codec is the backstop. + boundBy := make(map[string]string, len(doc.Properties)) + for _, slug := range sortedPropertySlugs(doc.Properties) { + if slug == detailKeyId || slug == detailKeyType { + continue // lifted into the envelope; a stray copy must not leak + } + // the document spells display names (§3); the store binds stored keys + key := imp.propertyKeyIn(slug, "/properties/"+escapeJSONPointer(slug)) + // admission runs on the FINAL resolved key, here at the seam where + // details are written (§3). Validate already refused everything its + // bundled chain could resolve, but a caller-supplied vocabulary can + // bind a spelling to a stored key the bundled table never knew — including + // the internal keys the deny rule exists for — and Validate takes no + // vocabulary, deliberately (§13). + if reason, denied := deniedPropertyKey(key); denied { + return 0, nil, &ValidationError{Issues: []Issue{{ + Path: "/properties/" + escapeJSONPointer(slug), + Message: reason, + }}} + } + // the §2a type-settings lift is KIND-SCOPED (typesettings.go): on a + // TYPE document the five stored keys live in the group and their flat + // spellings are refused with the repair named, while on every other + // kind they stay ordinary properties (apiObjectKey is real data on + // 9,725 relation documents) + if isTypeSmartBlock(sbType) { + if typeSettingsLiftedDetailKeys()[key] { + return 0, nil, &ValidationError{Issues: []Issue{{ + Path: "/properties/" + escapeJSONPointer(slug), + Message: fmt.Sprintf("%q is written on a type document as %s in type_settings, "+ + "not as a property", key, typeSettingsLiftedKeyRepair(key)), + }}} + } + // the install-provenance keys are dropped, not refused, on a type + // document: a document carrying one is stale rather than wrong — + // the transientProperties policy, scoped by kind (§2a) + if _, stale := typeProvenanceKeys[key]; stale { + continue + } + } + // a participant's createdDate is the same policy on the other + // machine-derived kind (§3): the stored value is a load timestamp, + // so a document carrying one is stale rather than wrong — dropped, + // and re-stamped by the destination the way every derived detail is + if DroppedParticipantProvenanceKey(sbType, key) { + continue + } + // transient state and the attribution keys are dropped, not refused: a + // document carrying one is stale rather than wrong. Export writes no + // transient key at all, and writes the attribution keys as derived + // captions — `#` recovered from the tree on every rebuild, + // which no write path could honour (§3) — so this fires on every + // document this package produces for an object with a creator, and + // on a stale or hand-written one for the rest. + if isDroppedOnImport(key) { + continue + } + // the seam admits only keys export could write (§3): a wider + // vocabulary can resolve a spelling onto a key with no writable form + // — the empty string included — which used to land details[""] + // silently, Validate clean and Unmarshal clean, and the re-export + // then dropped the property with only a warning. + if !isWritablePropertyKey(key) { + return 0, nil, &ValidationError{Issues: []Issue{{ + Path: "/properties/" + escapeJSONPointer(slug), + Message: unwritableKeyReason("resolved property key", key), + }}} + } + if first, dup := boundBy[key]; dup { + msg := fmt.Sprintf("%q and %q both address property %q — keep one", first, slug, key) + if first != slug && nfcTerm(first) == nfcTerm(slug) { + // the two spellings render identically, so %q would print + // the same glyphs twice; %+q names the code points apart + msg = fmt.Sprintf("%+q and %+q are one name in two Unicode normal forms, "+ + "and both address property %q — keep one; NFC is the canonical spelling (§3)", + first, slug, key) + } + return 0, nil, &ValidationError{Issues: []Issue{{ + Path: "/properties/" + escapeJSONPointer(slug), + Message: msg, + }}} + } + boundBy[key] = slug + if v := imp.propertyValue(key, slug, doc.Properties[slug]); v != nil { + details.Fields[key] = v + } + } + // the typed envelope fields are written after the property loop and can + // never collide with it: the nine keys they stand for are refused in + // `properties` on the RESOLVED stored key (deniedPropertyKey), which is + // what keeps a space-minted relation whose own stored key is `icon_emoji` + // an ordinary property (§2b) + imp.applyIcon(details) + imp.applyCover(details) + if err := imp.applyPropertySettings(details, sbType); err != nil { + return 0, nil, err + } + if err := imp.applyTypeSettings(details, sbType); err != nil { + return 0, nil, err + } + + root := &model.Block{ + Id: objectId, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + } + if doc.Root != nil { + if len(doc.Root.Fields) > 0 { + // a stale bundle still imports, but its analytics keys do not + // reach the snapshot — same rule the analytics details follow + // (accepted by Validate, dropped by Unmarshal). + fields := make(map[string]any, len(doc.Root.Fields)) + for k, v := range doc.Root.Fields { + if !analyticsRootFields[k] { + fields[k] = v + } + } + if len(fields) > 0 { + root.Fields = jsonMapToProtoStruct(fields) + } + } + root.BackgroundColor = doc.Root.BackgroundColor + } + + all := []*model.Block{root} + jbs, indents := imp.topLevelBlocks(details) + blocks, err := imp.flatSubtree(jbs, indents, root, -1) + if err != nil { + return 0, nil, fmt.Errorf("build blocks: %w", err) + } + all = append(all, blocks...) + + // a key slot that resolved onto nothing refuses the document rather than + // handing back an object with the slot missing (propertyKeyAt) + if imp.refusal != nil { + return 0, nil, &ValidationError{Issues: []Issue{*imp.refusal}} + } + // the folded participant references this reader could not rebuild (§9). + // One line for the document, not one per slot: the fault is a reader + // wired without a space, and every such reference in the object shares it. + if imp.foldedUnrebuilt { + imp.warn("", "this document was written with participants folded and "+ + "Options.SpaceId names no space: their references import as bare "+ + "identities, which address no object. Set SpaceId to the space this "+ + "document is being read into.") + } + + snapshot := &model.SmartBlockSnapshotBase{ + Blocks: all, + Details: details, + ObjectTypes: objectTypes, + Collections: imp.buildCollections(), + Key: doc.InternalKey, + } + return sbType, snapshot, nil +} + +// absorbIntoProperty merges a top-level title/description block's text into +// the matching property when unset (§7). +func (imp *importer) absorbIntoProperty(details *types.Struct, key, md string) { + if md == "" { + return + } + if existing := details.Fields[key]; existing.GetStringValue() != "" { + return + } + plain, _, err := parseInline(md) + if err != nil || plain == "" { + return + } + details.Fields[key] = &types.Value{Kind: &types.Value_StringValue{StringValue: plain}} +} + +func (imp *importer) buildCollections() *types.Struct { + doc := imp.doc + if len(doc.Items) == 0 && len(doc.Store) == 0 { + return nil + } + coll := &types.Struct{Fields: map[string]*types.Value{}} + for k, v := range doc.Store { + coll.Fields[k] = jsonToProtoValue(v) + } + if len(doc.Items) > 0 { + vals := make([]*types.Value, 0, len(doc.Items)) + for _, id := range doc.Items { + vals = append(vals, &types.Value{Kind: &types.Value_StringValue{StringValue: imp.objectRef(id)}}) + } + coll.Fields[storeKeyItems] = &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: vals}}} + } + return coll +} + +// propertyValue decodes a property per its resolved format (§3). Scalars of +// list-shaped formats normalize to single-element lists (§11). An explicit +// null stays a null value — presence of the key is preserved (§3). +// +// Both spellings travel: `key` is the stored key the value lands on, `slug` +// the term the document spelled it with. The slug is not decoration — the +// option legend's outer key is the SPELLING (optionrefs.go), because the +// reader that resolves it is reading the document, not the store. +func (imp *importer) propertyValue(key, slug string, v any) *types.Value { + if v == nil { + return &types.Value{Kind: &types.Value_NullValue{}} + } + // a name-over-number key is named in the format, stored as a number + // (§3). A number is still accepted so legacy documents keep importing + // unchanged; a string that is not a vocabulary name never reaches here — + // validation refused the document. + if vocab, named := namedEnumProperty(key); named { + if s, isStr := v.(string); isStr && vocab.has(s) { + return &types.Value{Kind: &types.Value_NumberValue{ + NumberValue: vocab.value(s), + }} + } + } + format, ok := imp.resolveFormat(key) + if !ok { + return jsonToProtoValue(v) + } + switch format { + case model.RelationFormat_date: + if s, isStr := v.(string); isStr { + if sec, parsed := parseDate(s); parsed { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: float64(sec)}} + } + } + case model.RelationFormat_status, model.RelationFormat_tag: + return wrapToList(mapJSONStrings(v, func(name string) string { return imp.resolveOption(key, slug, name) })) + case model.RelationFormat_object, model.RelationFormat_file: + return wrapToList(mapJSONStrings(v, imp.objectRef)) + } + return jsonToProtoValue(v) +} + +func wrapToList(v *types.Value) *types.Value { + if _, isList := v.GetKind().(*types.Value_ListValue); isList { + return v + } + return &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: []*types.Value{v}}}} +} + +// +// ---- blocks ---- +// + +// blockIndents extracts the effective indent of every entry, clamping per +// §4's lenient rule when NormalizeIndent is set (base is the indent of the +// run's implicit parent: −1 for the document, 0 for a cell's descendants). +// Clamps are silent here — validation already reported each one as a +// warning-grade issue on the same input, with the same rule. +func (imp *importer) blockIndents(jbs []*jsonBlock, base int) []int { + indents := make([]int, len(jbs)) + for i, jb := range jbs { + if jb == nil || jb.Indent == "" { + continue + } + if v, ok := jsonIntValue(jb.Indent); ok { + indents[i] = int(v) + } + } + if imp.opts.NormalizeIndent { + clampIndents(indents, base, nil) + } + return indents +} + +// dataviewBlockId is the editor's fixed id for an object's *own* dataview +// (mirrors state.DataviewBlockID, which this package must not import, §12). +// Object types, sets and collections all reconstruct their primary dataview +// at this id and merge into an existing block rather than adding a second +// one, so a document that omits the id has to resolve to it (§7). +const dataviewBlockId = "dataview" + +// pinPrimaryDataview gives the document's own dataview the editor's fixed id +// (§7). The primary dataview is the first indent-0 dataview block carrying +// neither an explicit id nor an objectId — an objectId means the block is an +// inline view of some *other* set or collection (§6.2) and keeps a generated +// id, as does any dataview nested below indent 0. A block that already claims +// the id anywhere in the document wins, so an explicit "id": "dataview" stays +// authoritative and no duplicate is minted (§13). +func (imp *importer) pinPrimaryDataview(raw []*jsonBlock, indents []int) { + // anything already using the id wins, and "anything" means the whole + // document: a table row named "dataview" is a block too, and it is not in + // this array. Minting the id anyway produced a duplicate *after* + // validation had passed, and the re-export then dropped the table body + // whole when the id resolved to the wrong block. + if imp.idTaken(dataviewBlockId) { + return + } + for i, jb := range raw { + if jb == nil || indents[i] != 0 { + continue + } + if jb.Type != "dataview" || jb.Id != "" || jb.ObjectId != "" { + continue + } + jb.Id = imp.claimId(dataviewBlockId) + return + } +} + +// topLevelBlocks resolves the document's blocks array for the tree rebuild: +// structural blocks at indent 0 are absorbed into properties or dropped, +// together with their whole subtree (§7 — matching the nested encoding, which +// never descended into them). +func (imp *importer) topLevelBlocks(details *types.Struct) ([]*jsonBlock, []int) { + raw := imp.doc.Blocks + indents := imp.blockIndents(raw, -1) + // §7a first, and the order is load-bearing: a wrapped dataview is only + // visible to the §7 pin at the indent-0 position the pin requires once + // the container above it is gone (160 corpus objects gain the `dataview` + // id under OmitIds this way, 0 lose it), and a wrapped title is only + // absorbed into `properties.name` once the lift has put it at indent 0. + raw, indents = liftTransparentContainers(raw, indents) + imp.pinPrimaryDataview(raw, indents) + jbs := make([]*jsonBlock, 0, len(raw)) + kept := make([]int, 0, len(raw)) + for i := 0; i < len(raw); i++ { + jb := raw[i] + if jb == nil { + continue + } + // structuralBlockTypes (blockvocab.go) is the one statement of which + // types these are — the API surfaces that publish an authorable + // vocabulary read the same map + if indents[i] == 0 && structuralBlockTypes[jb.Type] { + switch jb.Type { + case "title": + imp.absorbIntoProperty(details, "name", jb.Text) + case "description": + imp.absorbIntoProperty(details, "description", jb.Text) + } + for i+1 < len(raw) && indents[i+1] > 0 { + i++ + } + continue + } + jbs = append(jbs, jb) + kept = append(kept, indents[i]) + } + return jbs, kept +} + +// liftTransparentContainers is the import half of §7a: a `group` entry +// contributes no block, and every following entry indented deeper than it +// re-bases one level shallower — recursively, so a chain of n containers +// removes n levels. Any attribute on the container is ignored, and so is its +// id: a container is not a block, so nothing can address it. +// +// It is a pre-pass over the flat run rather than a case inside the rebuild, +// which is what lets it run at all three flatSubtree entry points — the +// document body, a table cell's array form, and the fragment WRITE path. A +// path that misses it does not merely keep a `group` in the JSON: it mints a +// real Layout_Div that no read will ever show and that normalization never +// removes while it has children — a phantom indent level living in the +// object forever. +// +// Monotonicity survives by construction, so the lift can never manufacture +// an F6 violation: a container at indent g had g ≤ p+1, and its first child, +// at g+1, lands at g. +func liftTransparentContainers(jbs []*jsonBlock, indents []int) ([]*jsonBlock, []int) { + // open holds the indents of the containers this entry is inside, so the + // shift is just how many of them there are + var open []int + outJbs := make([]*jsonBlock, 0, len(jbs)) + outIndents := make([]int, 0, len(indents)) + for i, jb := range jbs { + if jb == nil { + continue + } + k := indents[i] + for len(open) > 0 && open[len(open)-1] >= k { + open = open[:len(open)-1] + } + if transparentBlockTypes[jb.Type] { + open = append(open, k) + continue + } + outJbs = append(outJbs, jb) + outIndents = append(outIndents, k-len(open)) + } + return outJbs, outIndents +} + +type stackEntry struct { + b *model.Block + indent int +} + +// flatSubtree rebuilds the tree from a flat pre-order run (§4 F6): walk with +// a stack seeded (root, rootIndent); a block at indent k attaches to the +// nearest stack entry shallower than k. Validation guarantees indents are +// monotone (or already clamped), so that entry is exactly at k−1. +func (imp *importer) flatSubtree(jbs []*jsonBlock, indents []int, root *model.Block, rootIndent int) ([]*model.Block, error) { + var all []*model.Block + stack := []stackEntry{{root, rootIndent}} + for i, jb := range jbs { + if jb == nil { + continue + } + blocks, err := imp.blockFromJSON(jb, "") + if err != nil { + return nil, err + } + k := indents[i] + for len(stack) > 1 && stack[len(stack)-1].indent >= k { + stack = stack[:len(stack)-1] + } + parent := stack[len(stack)-1].b + parent.ChildrenIds = append(parent.ChildrenIds, blocks[0].Id) + all = append(all, blocks...) + stack = append(stack, stackEntry{blocks[0], k}) + } + return all, nil +} + +// textStyleAliases extends the canonical inventory with the §5 input aliases. +var textStyleAliases = map[string]model.BlockContentTextStyle{ + "heading_4": model.BlockContentText_Header3, + "header_4": model.BlockContentText_Header3, +} + +func (imp *importer) parseText(md string) (string, *model.BlockContentTextMarks, error) { + if md == "" { + return "", nil, nil + } + text, marks, err := parseInline(md) + if err != nil { + return "", nil, err + } + if len(marks) == 0 { + return text, nil, nil + } + return text, &model.BlockContentTextMarks{Marks: marks}, nil +} + +// textFromJSON builds a text-family block content (§5), applying the +// heading4/header4 aliases and the per-style prop rules. +func (imp *importer) textFromJSON(jb *jsonBlock) (*model.BlockContentText, error) { + style, isAlias := textStyleAliases[jb.Type] + if !isAlias { + style = textStyleNames.value(jb.Type) + } + text, marks, err := imp.parseText(jb.Text) + if err != nil { + return nil, err + } + t := &model.BlockContentText{Style: style, Text: text, Marks: marks, Color: jb.Color} + if style == model.BlockContentText_Checkbox { + t.Checked = jb.Checked + } + if style == model.BlockContentText_Callout { + calloutIconFrom(jb.Icon, t) + } + return t, nil +} + +// fileFromJSON builds a file-family block content (§5); state is recomputed, +// never serialized. +func (imp *importer) fileFromJSON(jb *jsonBlock) *model.BlockContentFile { + f := &model.BlockContentFile{ + Type: fileTypeNames.value(jb.Type), + TargetObjectId: imp.objectRef(jb.ObjectId), + Hash: jb.Hash, + Name: jb.Name, + Mime: jb.MimeType, + Size_: jsonInt64(jb.Size), + Style: fileStyleNames.value(jb.Style), + } + if jb.AddedAt != "" { + if sec, ok := parseDate(jb.AddedAt); ok { + f.AddedAt = sec + } + } + if f.TargetObjectId != "" || f.Hash != "" { + f.State = model.BlockContentFile_Done + } + return f +} + +// bookmarkFromJSON builds a bookmark content; state is recomputed (§5). +func (imp *importer) bookmarkFromJSON(jb *jsonBlock) *model.BlockContentBookmark { + bm := &model.BlockContentBookmark{ + Url: jb.Url, + TargetObjectId: imp.objectRef(jb.ObjectId), + } + if bm.TargetObjectId != "" { + bm.State = model.BlockContentBookmark_Done + } + return bm +} + +// linkFromJSON builds a link content, decoding the shown-property key list. +func (imp *importer) linkFromJSON(jb *jsonBlock) (*model.BlockContentLink, error) { + var propKeys []string + if len(jb.Properties) > 0 { + if err := jsonUnmarshal(jb.Properties, &propKeys); err != nil { + return nil, fmt.Errorf("link properties: %w", err) + } + } + return &model.BlockContentLink{ + TargetBlockId: imp.objectRef(jb.ObjectId), + CardStyle: cardStyleNames.value(jb.CardStyle), + IconSize: iconSizeNames.value(jb.IconSize), + Description: linkDescriptionNames.value(jb.Description), + Relations: imp.propertyKeysAt(propKeys, "link block `properties`"), + }, nil +} + +// blockFromJSON converts one block; the returned slice has the block first, +// followed by any internal blocks it owns (the table subtree). Document +// children are attached by the flatSubtree stack rebuild, not here. forcedId +// overrides the block id (used for derived table cell ids). +func (imp *importer) blockFromJSON(jb *jsonBlock, forcedId string) ([]*model.Block, error) { + id := forcedId + if id == "" { + id = jb.Id + } + if id == "" { + id = imp.genId() + } + b := &model.Block{Id: id} + var extra []*model.Block + liftedLang := "" + + switch { + case jb.Type == "code": + b.Content = &model.BlockContentOfText{Text: &model.BlockContentText{ + Style: model.BlockContentText_Code, + Text: jb.Text, // literal (§8.4) + }} + liftedLang = jb.Language + case textStyleNames.has(jb.Type) || textStyleAliases[jb.Type] != 0: + t, err := imp.textFromJSON(jb) + if err != nil { + return nil, fmt.Errorf("block %s: %w", id, err) + } + b.Content = &model.BlockContentOfText{Text: t} + case fileTypeNames.has(jb.Type): + b.Content = &model.BlockContentOfFile{File: imp.fileFromJSON(jb)} + case jb.Type == "bookmark": + b.Content = &model.BlockContentOfBookmark{Bookmark: imp.bookmarkFromJSON(jb)} + case jb.Type == "link": + link, err := imp.linkFromJSON(jb) + if err != nil { + return nil, fmt.Errorf("block %s: %w", id, err) + } + b.Content = &model.BlockContentOfLink{Link: link} + case jb.Type == "divider": + b.Content = &model.BlockContentOfDiv{Div: &model.BlockContentDiv{ + Style: divStyleNames.value(jb.Style), + }} + case jb.Type == "row": + b.Content = &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Row}} + case jb.Type == "column": + b.Content = &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Column}} + case transparentBlockTypes[jb.Type]: + // DEFENSIVE: unreachable through every public entry point today — + // Unmarshal and UnmarshalBlock both validate first, and the validation + // side refuses a container in a cell and in a single-block fragment on + // its own (probed: mutating this line leaves the whole package green). + // It stays as the backstop for a future caller that skips validation, + // and it must not be deleted on the strength of a coverage report. + // + // §7a: a transparent container contributes no block of its own, and + // every flat run lifts it away before this. What reaches here is a + // caller that addresses exactly ONE block — UnmarshalBlock, or a + // table cell, which is a position rather than a run — and a caller + // that asked for one block must be told it named nothing, not handed + // a wrapper no read will ever show it again. + return nil, fmt.Errorf("block %s: %q is a transparent container and contributes no block of its own", id, jb.Type) + case jb.Type == "table": + table, tExtra, err := imp.tableFromJSON(jb, id) + if err != nil { + return nil, err + } + b = table + extra = tExtra + case jb.Type == "embed" || jb.Type == "equation": + processor := processorNames.value(jb.Processor) + text := jb.Text + if text == "" && !sourceProcessors[processor] { + text = jb.Url // input alias for service processors (§5.2) + } + b.Content = &model.BlockContentOfLatex{Latex: &model.BlockContentLatex{ + Text: text, + Processor: processor, + }} + case jb.Type == "table_of_contents": + b.Content = &model.BlockContentOfTableOfContents{TableOfContents: &model.BlockContentTableOfContents{}} + case jb.Type == "property": + b.Content = &model.BlockContentOfRelation{Relation: &model.BlockContentRelation{ + Key: imp.propertyKeyAt(jb.Property, "property block `property`")}} + case jb.Type == "dataview": + dv, err := imp.dataviewFromJSON(jb) + if err != nil { + return nil, fmt.Errorf("block %s: %w", id, err) + } + b.Content = &model.BlockContentOfDataview{Dataview: dv} + case jb.Type == "widget": + b.Content = &model.BlockContentOfWidget{Widget: &model.BlockContentWidget{ + Layout: widgetLayoutNames.value(jb.Layout), + Limit: jsonInt32(jb.Limit), + ViewId: jb.ViewId, + AutoAdded: jb.AutoAdded, + }} + case jb.Type == "chat": + b.Content = &model.BlockContentOfChat{Chat: &model.BlockContentChat{}} + case jb.Type == "featured_properties": + b.Content = &model.BlockContentOfFeaturedRelations{FeaturedRelations: &model.BlockContentFeaturedRelations{}} + case jb.Type == "icon": + b.Content = &model.BlockContentOfIcon{Icon: &model.BlockContentIcon{Name: jb.Name}} + default: + return nil, fmt.Errorf("block %s: unknown type %q", id, jb.Type) + } + + imp.applyBlockCommon(b, jb, liftedLang) + return append([]*model.Block{b}, extra...), nil +} + +// applyBlockCommon writes the shared block tail: align, verticalAlign, +// backgroundColor, fields, and the lifted code language (§4, §5.1). +func (imp *importer) applyBlockCommon(b *model.Block, jb *jsonBlock, liftedLang string) { + b.Align = alignNames.value(jb.Align) + b.VerticalAlign = verticalAlignNames.value(jb.VerticalAlign) + b.BackgroundColor = jb.BackgroundColor + if len(jb.Fields) > 0 { + b.Fields = jsonMapToProtoStruct(jb.Fields) + } + if liftedLang != "" { + if b.Fields == nil { + b.Fields = &types.Struct{Fields: map[string]*types.Value{}} + } + b.Fields.Fields[codeLangField] = &types.Value{Kind: &types.Value_StringValue{StringValue: liftedLang}} + } +} + +// sortedPropertySlugs returns the document's property spellings in a fixed +// order, so which of two colliding spellings the refusal names — and which +// value a non-colliding document binds — never depends on map iteration. +func sortedPropertySlugs(props map[string]any) []string { + out := make([]string, 0, len(props)) + for slug := range props { + out = append(out, slug) + } + sort.Strings(out) + return out +} diff --git a/pkg/lib/anyblockjson/index.go b/pkg/lib/anyblockjson/index.go new file mode 100644 index 0000000000..5b54b15306 --- /dev/null +++ b/pkg/lib/anyblockjson/index.go @@ -0,0 +1,692 @@ +package anyblockjson + +// index.go implements §2c: the bundle-level index.json. Every other document +// in this format describes one object; index.json describes the set — the +// space's name, what opens on entry, and what the sidebar shows. +// +// It exists because none of that is expressible per-object. The wiring splits +// it across two outputs, because the installer takes them from two places: a +// `profile` file at the archive root (pb.Profile, read by util/builtinobjects) +// carries spaceDashboardId, and the sidebar travels as a Widget snapshot the +// wiring BUILDS from index.widgets (WidgetsSnapshot) — a bundle itself +// carries no widget document, the way it carries no space document. See §2c. +// A bundle without an index imports as an undifferentiated object list. + +import ( + "bytes" + _ "embed" + "fmt" + "sort" + "strings" + "sync" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +//go:embed schema/index.schema.json +var indexSchemaJSON []byte + +// IndexFileName is the name a bundle's index must have, at the bundle root. +const IndexFileName = "index.json" + +// PlatformPrefix opens the platform's own address space. A value beginning +// with it — `_otpage`, `_brdue_date`, `_missing_object`, `_participant_…` — +// names something the platform provides, never something a document or a +// bundle mints (§1). The format borrows the same namespace for the built-in +// screens and listings an index.json may name, which is what makes the two +// sets provably disjoint: **no bundle-local object id may begin with `_`**. +// +// That disjointness is the whole point, and it is not decorative. The pb +// importer resolves a widget's target through the bundle's own id map FIRST +// (common.UpdateLinksToObjects) and only then asks +// widget.IsPredefinedWidgetTargetId. While the reserved listings were bare +// words — `set`, `favorite` — a bundle shipping an object with id `set` +// silently captured the widget that meant *the Sets listing*, and +// Index.EntryPoint, which skips reserved targets, silently disagreed with +// EffectiveEntryPoint about what the bundle even opens. A prefix rule needs +// only "no minted id STARTS with `_`", which is permanently true, where a +// per-word reservation needs a new ban every time a listing is added. +const PlatformPrefix = "_" + +// IsPlatformId reports whether id lies in the reserved `_` namespace, and so +// addresses the platform rather than anything a bundle ships. +func IsPlatformId(id string) bool { + return strings.HasPrefix(id, PlatformPrefix) +} + +// Reserved homepage values. Anything else is an object id. +// +// These are the format's spellings, not the wire's: core/domain/homepage.go +// carries them as the bare `widgets` / `graph`, and builtinobjects switches on +// those names BEFORE looking an id up — the opposite precedence from widget +// targets, and the reason a bundle object with id `graph` can never be a +// homepage. Both directions close once the reserved spellings live in the +// platform namespace. The `_` is translated away at the wire boundary, by +// WireHomepage. +const ( + HomepageWidgets = "_widgets" + HomepageGraph = "_graph" +) + +// wireHomepages is the format spelling of each reserved homepage and the +// spelling core/domain/homepage.go uses for the same screen. WireHomepage +// walks it one way and FormatHomepage the other: the STORE holds the wire +// spelling, so a value lifted out of a space object has to be translated +// before it becomes an index field, exactly as one written to a profile has +// to be translated on the way out. +var wireHomepages = map[string]string{ + HomepageWidgets: "widgets", + HomepageGraph: "graph", +} + +// reservedWidgetTargets are the widget targets that name a built-in listing +// rather than an object in the bundle (core/block/editor/widget). +var reservedWidgetTargets = map[string]struct{}{ + "_favorite": {}, "_recent": {}, "_set": {}, "_collection": {}, + "_all_objects": {}, "_recent_open": {}, "_chat": {}, "_bin": {}, +} + +// importableWidgetTargets are the reserved targets the *importer* recognises: +// exactly widget.IsPredefinedWidgetTargetId, which is what +// common.handleLinkBlock consults before deciding a link target it cannot +// resolve is broken. The wire spellings are the live space's own — bare and +// sometimes camelCase (`favorite`, `allObjects`) — and the wiring translates +// them at the boundary, both ways (WireWidgetTarget / FormatWidgetTarget). +// +// The inventory is the full set of listings live spaces actually hold: +// measured over a 77-space account, 33 of 218 widget links name one, and the +// population is chat 11 · bin 10 · allObjects 8 · recent 1 · set 1 (plus two +// strays no client defines, which stay inexpressible). For one revision the +// importer knew only four of these and this map said so — a bundle naming +// `_all_objects` was refused up front, because the importer would have +// rewritten the link to addr.MissingObject and WidgetObject.Init would have +// stripped it, losing the widget with no error. The importer knows all eight +// now, so all eight travel. +// +// The map is the translation table as well as the membership test, so the two +// cannot drift: adding a listing without giving it a wire spelling is not +// expressible. +var importableWidgetTargets = map[string]string{ + "_favorite": "favorite", "_recent": "recent", "_set": "set", "_collection": "collection", + "_all_objects": "allObjects", "_recent_open": "recentOpen", "_chat": "chat", "_bin": "bin", +} + +// wireWidgetTargetsByWire inverts importableWidgetTargets, so a stored link +// target can be lifted into the format's spelling by lookup rather than by a +// second table that would drift. +var wireWidgetTargetsByWire = func() map[string]string { + out := make(map[string]string, len(importableWidgetTargets)) + for format, wire := range importableWidgetTargets { + out[wire] = format + } + return out +}() + +// IsReservedWidgetTarget reports whether target names a built-in listing, in +// which case it does not name an object in the bundle. +func IsReservedWidgetTarget(target string) bool { + _, ok := reservedWidgetTargets[target] + return ok +} + +// IsImportableWidgetTarget reports whether a reserved target survives import. +// Every listing in today's inventory does — the importer knows all eight — +// but the two questions stay separate, because they can come apart again the +// day a listing is added here before the importer learns it: a reserved +// target the importer does not know is the one case where a widget is +// dropped silently, so callers must reject it rather than emit it. +func IsImportableWidgetTarget(target string) bool { + _, ok := importableWidgetTargets[target] + return ok +} + +// WireWidgetTarget returns the id to write into a link block's target for a +// widget target: the bare listing name for a reserved one, the id itself for +// anything else. +// +// The translation is not cosmetic and is the reason the rename is safe. +// common.handleLinkBlock leaves a target alone only when +// widget.IsPredefinedWidgetTargetId knows it, and that function knows the four +// bare words and nothing else — write `_set` and the link is rewritten to +// addr.MissingObject and then stripped along with its wrapper, losing the +// widget with no error. A reserved-but-not-importable target has no wire +// spelling at all, and is returned unchanged so it fails loudly rather than +// impersonating an id; CheckIndexTargets refuses those before conversion. +func WireWidgetTarget(target string) string { + if wire, ok := importableWidgetTargets[target]; ok { + return wire + } + return target +} + +// FormatWidgetTarget is WireWidgetTarget's inverse: the format's `_`-prefixed +// spelling for a wire listing name, the id itself for anything else. It is +// what a lift out of a stored widget object applies to every target-shaped +// value — the link targets and the auto-widget list alike — so the stored +// `bin` becomes the `_bin` no bundle object may claim (§1). +func FormatWidgetTarget(wire string) string { + if format, ok := wireWidgetTargetsByWire[wire]; ok { + return format + } + return wire +} + +// WireHomepage returns the value pb.Profile.SpaceDashboardId should carry for +// a homepage: the bare screen name for a reserved one, the object id +// otherwise. builtinobjects.setWorkspaceSettings matches those bare names +// before it tries to resolve an id, so an untranslated `_graph` would be +// looked up as an object, fail, and fall back to the widgets screen. +func FormatHomepage(wire string) string { + for format, w := range wireHomepages { + if w == wire { + return format + } + } + return wire +} + +func WireHomepage(homepage string) string { + if wire, ok := wireHomepages[homepage]; ok { + return wire + } + return homepage +} + +// IsReservedBundleId reports whether id is one no object in a bundle may +// claim. +// +// Two populations, and the second is the one that is easy to miss. The `_` +// namespace is the format's own guarantee (§1). But the importer's spellings +// are BARE — WireWidgetTarget translates `_set` to `set` on the way out — and +// common.handleLinkBlock resolves a link target through the bundle's id map +// before it asks widget.IsPredefinedWidgetTargetId. So an object with id `set` +// captures the translated widget exactly the way it captured the untranslated +// one: renaming the format's spelling moves the collision downstream rather +// than removing it, unless the wire spellings are unmintable too. +// +// The homepages are the same story with the precedence reversed: +// setWorkspaceSettings matches `graph` before resolving an id, so a bundle +// object with that id is simply unreachable as a homepage. +func IsReservedBundleId(id string) bool { + if IsPlatformId(id) { + return true + } + for _, wire := range importableWidgetTargets { + if id == wire { + return true + } + } + for _, wire := range wireHomepages { + if id == wire { + return true + } + } + return false +} + +// ReservedWidgetTargets lists every reserved listing, sorted, so a diagnostic +// can name the inventory instead of asking the author to guess it. +func ReservedWidgetTargets() []string { + out := make([]string, 0, len(reservedWidgetTargets)) + for t := range reservedWidgetTargets { + out = append(out, t) + } + sort.Strings(out) + return out +} + +// IsReservedHomepage reports whether homepage names a built-in screen rather +// than an object in the bundle. +func IsReservedHomepage(homepage string) bool { + _, ok := wireHomepages[homepage] + return ok +} + +// Widget is one sidebar widget (§2c) — flat, though the wire carries it as +// two blocks. A stored sidebar is a widget WRAPPER block with an indented +// LINK child naming the target, and measured over 77 real spaces the pairing +// is perfectly regular: 218 wrapper blocks, 218 link children, nothing else. +// The pair carries no information beyond its members, so the format does not +// ask an author to build block scaffolding to get a sidebar. +// +// The members are the two blocks' §5 members, verbatim: `layout`, `limit`, +// `view_id` and `auto_added` are the wrapper's, and `card_style`, +// `icon_size`, `description` and `properties` are the link's own display +// members, with the link's `object_id` renamed `target` because here it may +// also name a reserved listing. +type Widget struct { + Target string `json:"target"` + Layout string `json:"layout"` + Limit int32 `json:"limit"` + ViewId string `json:"view_id"` + AutoAdded bool `json:"auto_added"` + // the link child's display members (§5): how the widget's row renders. + CardStyle string `json:"card_style"` + IconSize string `json:"icon_size"` + Description string `json:"description"` + // Properties are the property keys shown on the widget's card, held as + // STORED keys in this struct and WRITTEN in the canonical spelling the + // manifest's type keys use — the bundled display name, or the stored + // key verbatim: the index has no per-document legend, so the spelling + // must be a pure function of the key, and the dictionary's own spelling + // pair is that function (§2f). + Properties []string `json:"properties"` +} + +// Manifest says where to find what a reader must resolve by key rather than +// by walking (§2c): the format defines no folder layout, and an object names +// its type by spelling alone, so without one a reader resolves a type by +// scanning every document for a matching key. Types are keyed by STORED type +// key — the spelling that survives a rename — and Properties points at the +// dictionary (§2f), which answers for stored property keys the same way. +// Paths are relative to the index file. +// +// It does NOT locate options, and that is deliberate. A manifest +// exists to answer a lookup a reader would otherwise have to scan for, and +// no reader has that lookup for an option: the dictionary states a +// property's whole vocabulary inline — each option's name, colour, position +// and, since the vocabulary learned `internal_key`, its stored key — so +// everything an option MEANS is already in hand before any document is +// opened. The map was 2,641 entries pointing at documents nothing needed to +// read. +// +// The `option_ids` legend keeps carrying option OBJECT ids, which is a +// different job and unaffected: those are resolved against the importing +// space's live store to survive a rename (§9a), never against the bundle, so +// they never needed a path beside them. +// +// It DOES locate file blobs. Files is the map every importer holding +// a file_object document needs — object id → the blob's archive-relative +// path — and it is the format's replacement for the legacy exporter's +// `source`-clobber, which stuffed the blob path into a real, user-facing, +// editable relation on the document itself. A document member is not a slot +// for archive bookkeeping; the manifest's whole charter is "where to find +// what a reader must resolve by id rather than by walking", and blobs are +// exactly that. Keys are object ids verbatim, so — unlike Types — they take +// no re-spelling on either side. Adjacency of blob and document in `files/` +// is one exporter's layout convention riding on top; the map is the only +// binding a reader may rely on (§2c). +type Manifest struct { + Types map[string]string `json:"types"` + Properties string `json:"properties"` + Files map[string]string `json:"files"` +} + +// empty reports whether the manifest locates nothing — the shape setNonEmpty +// cannot judge for a struct. +func (m *Manifest) empty() bool { + return m == nil || (len(m.Types) == 0 && m.Properties == "" && len(m.Files) == 0) +} + +// Index is a bundle's index.json (§2c). +type Index struct { + Schema string `json:"$schema"` + Version int `json:"version"` + Name string `json:"name"` + Description string `json:"description"` + // Icon is the space's icon in the typed shape every icon in this format + // has (§2b), restricted to the two kinds a bundle can hold: an emoji, or + // the object id of an image IN THE BUNDLE. Two flat keys used to stand + // here, with no rule for which one wins and with `icon_image` spelled as + // a scalar while the object surface spelled it as a list — one concept, + // two conventions, in one format. + // + // The installer resolves the space icon by image *name* + // (builtinobjects.getNewAvatarId queries name + image layout), so the + // wiring looks the name up from this id; that asymmetry is the wire + // format's, not the author's. An image needs the image object and its + // file in the archive, which is why a generated bundle uses an emoji. + Icon *Icon `json:"icon"` + // Entrypoint is the object opened once, right after the space is created + // — the first thing a user ever sees. Distinct from Homepage, which is + // what opens on every later entry, and deliberately not the widget order: + // the wire format carries the entry point as widgets[0] + // (builtinobjects.inject), but making authors express it by sorting a list + // means reordering the sidebar silently changes what opens. + Entrypoint string `json:"entrypoint"` + Homepage string `json:"homepage"` + Widgets []Widget `json:"widgets"` + // AutoWidgetTargets are the targets the client has already auto-added a + // widget for — its ledger for not re-adding one the user then deleted. + // Space state, not widget state: an entry usually names a widget that is + // NOT in the sidebar any more, which is the whole point of the ledger. + // Spelled like widget targets (a reserved listing or an object id), + // because that is what the entries are. 21 of 77 real spaces carry one. + AutoWidgetTargets []string `json:"auto_widget_targets"` + // AutoWidgetDisabled records that the user turned automatic widgets off + // for this space entirely. 2 of 77 real spaces. + AutoWidgetDisabled bool `json:"auto_widget_disabled"` + // Manifest locates types, file blobs and the property dictionary + // without a folder convention (§2c). Optional: a bundle without one is + // walked the way every bundle was before it existed. + Manifest *Manifest `json:"manifest"` +} + +// EntryPoint returns the entry point the bundle *declares*: the entrypoint +// field, or for a bundle written before it existed, the first widget naming an +// object. +// +// TEMPORARY: this is intent, not behaviour. pb.Profile has no field for an +// entry point — builtinobjects.inject opens widgets[0].targetObjectId — so +// until the profile handling grows one, what actually opens is +// EffectiveEntryPoint. The two differ exactly when a bundle declares an +// entrypoint that is not its first widget, which is worth reporting. +func (i *Index) EntryPoint() string { + if i.Entrypoint != "" { + return i.Entrypoint + } + for _, w := range i.Widgets { + if !IsReservedWidgetTarget(w.Target) { + return w.Target + } + } + return "" +} + +// EffectiveEntryPoint returns what the installer opens *today*: the first +// widget naming an object, which is all pb.Profile can express. Compare with +// EntryPoint to detect a declared entry point that will not be honoured yet. +func (i *Index) EffectiveEntryPoint() string { + for _, w := range i.Widgets { + if !IsReservedWidgetTarget(w.Target) { + return w.Target + } + } + return "" +} + +// SpaceHomepage returns what opens on entering the space: the declared +// homepage, else the entry point. Only an explicit reserved value gives up a +// real page — omitting homepage does not. +func (i *Index) SpaceHomepage() string { + if i.Homepage != "" { + return i.Homepage + } + return i.EntryPoint() +} + +var compileIndexSchema = sync.OnceValues(func() (*jsonschema.Schema, error) { + doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(indexSchemaJSON)) + if err != nil { + return nil, fmt.Errorf("decode embedded index schema: %w", err) + } + c := jsonschema.NewCompiler() + // the object schema is added alongside, because the index's `icon` is a + // $ref into it (§2b): one definition of the icon shape for both surfaces, + // rather than a copy in each that drifts. Both files are published at + // these URLs, so an external validator resolves the same reference. + objectDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaJSON)) + if err != nil { + return nil, fmt.Errorf("decode embedded schema: %w", err) + } + if err := c.AddResource(SchemaURL, objectDoc); err != nil { + return nil, fmt.Errorf("add schema resource: %w", err) + } + if err := c.AddResource(IndexSchemaURL, doc); err != nil { + return nil, fmt.Errorf("add index schema resource: %w", err) + } + sch, err := c.Compile(IndexSchemaURL) + if err != nil { + return nil, fmt.Errorf("compile index schema: %w", err) + } + return sch, nil +}) + +// UnmarshalIndex validates data against the index schema and decodes it +// (§2c). Errors wrap *ValidationError with path-addressed issues, like +// Unmarshal. +// +// Whether the ids it names exist is a cross-document question the wiring +// answers, not this package: an index is valid on its own terms while +// pointing at an object no document defines. +func UnmarshalIndex(data []byte) (*Index, error) { + raw, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + if err != nil { + return nil, &ValidationError{Issues: []Issue{{Message: fmt.Sprintf("invalid JSON: %v", err)}}} + } + doc, ok := raw.(map[string]any) + if !ok { + return nil, &ValidationError{Issues: []Issue{{Message: "index must be a JSON object"}}} + } + // An index shares the format version and its rules with object documents + // (§10): gate on it here, before the schema can turn a newer version into + // a generic "value must be 1" that says nothing about why. + if err := checkVersion(doc); err != nil { + return nil, err + } + if issues := misroutedIssues(data, KindIndex); len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + // The `_` namespace is checked here, ahead of the schema, for the same + // reason the version is: the schema states the rule machine-readably (a + // pattern and an enum, so a generator reading the schema obeys it), but + // its failure is an anonymous "does not match ^[^_]" that tells an author + // nothing about which namespace they walked into or what to do about it. + if issues := platformNameIssues(doc); len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + // the typed icon's discriminator, worded the same way it is on an object + // document: `required` can say `format` is missing but not that it is a + // CHOICE, and naming the alternatives is the whole point of typing it + if issue, missing := missingFormatIssue("/icon", "a space icon", "plainIcon", doc["icon"]); missing { + return nil, &ValidationError{Issues: []Issue{issue}} + } + // MIGRATION SEAM: an older version is migrated forward here, between the + // version gate and schema validation. The schema pins the version to a + // const, so it doubles as the assertion that migration ran (§10). + sch, err := compileIndexSchema() + if err != nil { + return nil, fmt.Errorf("embedded index schema: %w", err) + } + if err := sch.Validate(raw); err != nil { + return nil, &ValidationError{Issues: schemaIssues(err, keySlotReport{})} + } + + var idx Index + if err := jsonUnmarshal(data, &idx); err != nil { + return nil, fmt.Errorf("decode index: %w", err) + } + // the manifest's type keys arrive in the format's spelling and are held + // as STORED keys, the way the dictionary holds its property keys: the + // wire says "Chat", the codec says `chatDerived`, and a caller + // looking a type up by stored key finds it (§2c). + if idx.Manifest != nil { + idx.Manifest.Types = reKeyed(idx.Manifest.Types, StoredTypeKey) + } + // a widget's shown properties follow the same rule: spelled on the wire, + // held as STORED keys, resolved by the chain every key slot uses. An + // ambiguous spelling stays verbatim — the index is display state, and + // which property a fold means is the tooling's cross-document question. + for i := range idx.Widgets { + idx.Widgets[i].Properties = mapStrings(idx.Widgets[i].Properties, func(s string) string { + stored, _ := dictionaryStoredKey(s) + return stored + }) + } + return &idx, nil +} + +// platformNameIssues reports index members that walk into the reserved `_` +// namespace: an entry point or homepage that is not a screen the platform +// provides, and a widget target spelled like a reserved listing that is not +// one of the six. +// +// Without it a typo in the namespace lands in the worst possible place. A +// widget target the reader treats as an object id resolves to nothing, and an +// unresolvable link target becomes addr.MissingObject, after which +// WidgetObject.Init strips the link and its wrapper — the widget is gone with +// no error. Saying "unknown reserved listing, here is the inventory" points at +// the repair; "no object with that id in the bundle" points away from it. +func platformNameIssues(doc map[string]any) []Issue { + var issues []Issue + str := func(key string) string { + v, _ := doc[key].(string) + return v + } + if e := str("entrypoint"); IsPlatformId(e) { + issues = append(issues, Issue{ + Path: "/entrypoint", + Message: fmt.Sprintf("%q begins with %q, which is the platform's own address space: "+ + "an entry point must be an object id from this bundle, and no built-in screen can be one", + e, PlatformPrefix), + }) + } + if h := str("homepage"); IsPlatformId(h) && !IsReservedHomepage(h) { + issues = append(issues, Issue{ + Path: "/homepage", + Message: fmt.Sprintf("%q begins with %q, which is the platform's own address space: "+ + "the only reserved homepages are %q and %q, and an object id from this bundle may not begin with %q", + h, PlatformPrefix, HomepageWidgets, HomepageGraph, PlatformPrefix), + }) + } + widgets, _ := doc["widgets"].([]any) + for i, raw := range widgets { + w, _ := raw.(map[string]any) + target, _ := w["target"].(string) + if !IsPlatformId(target) || IsReservedWidgetTarget(target) { + continue + } + issues = append(issues, Issue{ + Path: fmt.Sprintf("/widgets/%d/target", i), + Message: fmt.Sprintf("%q is not a reserved listing; the whole inventory is %s. "+ + "An object id from this bundle may not begin with %q, so this target names nothing "+ + "— and a widget target that resolves to nothing is dropped on install without an error", + target, strings.Join(ReservedWidgetTargets(), ", "), PlatformPrefix), + }) + } + // the auto-widget ledger's entries are target-shaped and get the same + // diagnostic: a `_`-typo there would otherwise read as an object id that + // names nothing + autos, _ := doc["auto_widget_targets"].([]any) + for i, raw := range autos { + target, _ := raw.(string) + if !IsPlatformId(target) || IsReservedWidgetTarget(target) { + continue + } + issues = append(issues, Issue{ + Path: fmt.Sprintf("/auto_widget_targets/%d", i), + Message: fmt.Sprintf("%q is not a reserved listing; the whole inventory is %s. "+ + "An object id from this bundle may not begin with %q, so this entry names nothing", + target, strings.Join(ReservedWidgetTargets(), ", "), PlatformPrefix), + }) + } + return issues +} + +// indexIconOmap renders a bundle index's icon in canonical form (§2c). +// +// It is `iconOmap`, the object surface's own renderer, and deliberately not a +// second one: the index used to render its own two variants, and a space +// whose icon is a bare COLOUR would have had it silently dropped on the way +// into the index. Measured over 77 real spaces: 55 resolve to an image, and +// their colour rides along on the file variant, which the narrow shape did +// allow — but 20 have the colour and nothing else, and those are the letter +// avatars the client actually draws. The index now admits the full `icon` +// shape, which is also what its schema $refs. +func indexIconOmap(ic *Icon) *omap { + return iconOmap(ic) +} + +// IconImageId is the object id of an index's image icon, or "" when the index +// has no icon or an emoji one. The bundle wiring resolves that id to an image +// NAME, which is what the installer actually takes. +func (i *Index) IconImageId() string { + if i == nil || i.Icon == nil || i.Icon.Format != "file" { + return "" + } + return i.Icon.File +} + +// reKeyed re-spells a manifest map's keys, keeping its values. +func reKeyed(in map[string]string, spell func(string) string) map[string]string { + if len(in) == 0 { + return in + } + out := make(map[string]string, len(in)) + for k, v := range in { + out[spell(k)] = v + } + return out +} + +// MarshalIndex renders an index in the canonical byte form (§4). +func MarshalIndex(idx *Index) ([]byte, error) { + if idx == nil { + return nil, fmt.Errorf("nil index") + } + doc := &omap{} + doc.set("$schema", IndexSchemaURL) + doc.set("version", FormatVersion) + doc.setNonEmpty("name", idx.Name) + doc.setNonEmpty("description", idx.Description) + doc.setNonEmpty("icon", indexIconOmap(idx.Icon)) + doc.setNonEmpty("entrypoint", idx.Entrypoint) + doc.setNonEmpty("homepage", idx.Homepage) + + var widgets []any + for _, w := range idx.Widgets { + if w.Target == "" { + continue + } + wm := &omap{} + wm.set("target", w.Target) + // the §4 omit-empty canon, member by member: `link` is the wrapper's + // default layout, and the three link display defaults are the same + // ones the link BLOCK omits (§5) — `text`, `none`, `none` + if w.Layout != "" && w.Layout != "link" { + wm.set("layout", w.Layout) + } + wm.setNonEmpty("limit", w.Limit) + wm.setNonEmpty("view_id", w.ViewId) + wm.setNonEmpty("auto_added", w.AutoAdded) + if w.CardStyle != "" && w.CardStyle != "text" { + wm.set("card_style", w.CardStyle) + } + if w.IconSize != "" && w.IconSize != "none" { + wm.set("icon_size", w.IconSize) + } + if w.Description != "" && w.Description != "none" { + wm.set("description", w.Description) + } + // spelled the way the dictionary spells a stored key (§2f): the + // index has no legend, so the spelling is a pure function of the key + wm.setNonEmpty("properties", stringsToAny(mapStrings(w.Properties, dictionaryKeySpelling))) + widgets = append(widgets, wm) + } + doc.setNonEmpty("widgets", widgets) + doc.setNonEmpty("auto_widget_targets", stringsToAny(idx.AutoWidgetTargets)) + doc.setNonEmpty("auto_widget_disabled", idx.AutoWidgetDisabled) + if !idx.Manifest.empty() { + m := &omap{} + // the manifest keys types the way the dictionary keys properties and + // the way a type document spells a target type: one spelling per + // concept (§2c, §2f). It carried `chatDerived`, `objectType`, + // `relationOption` and `spaceView` — 308 camelCase keys across 77 + // bundles — while the documents beside it said `chat_derived`. + m.setNonEmpty("types", sortedStringOmap(reKeyed(idx.Manifest.Types, TypeKeySpelling))) + m.setNonEmpty("properties", idx.Manifest.Properties) + // file blob bindings are keyed by object id VERBATIM (§2c): an id is + // its own spelling, so unlike `types` there is nothing to re-key — + // only the canonical sort + m.setNonEmpty("files", sortedStringOmap(idx.Manifest.Files)) + doc.setNonEmpty("manifest", m) + } + return marshalCanonical(doc) +} + +// sortedStringOmap renders a string map with sorted keys — the canonical +// order for the manifest's lookup tables (§4), or nil when there is +// nothing to render. An empty VALUE is omitted like any other empty member +// (the §4 canon): it locates nothing, and writing it produced bytes the +// index's own Unmarshal refuses (the schema's minLength on every manifest +// path) — I1 broken from the Go API. +func sortedStringOmap(m map[string]string) *omap { + out := &omap{} + for _, k := range sortedStringKeys(m) { + out.setNonEmpty(k, m[k]) + } + if len(out.keys) == 0 { + return nil + } + return out +} diff --git a/pkg/lib/anyblockjson/index_test.go b/pkg/lib/anyblockjson/index_test.go new file mode 100644 index 0000000000..9f3cab4f59 --- /dev/null +++ b/pkg/lib/anyblockjson/index_test.go @@ -0,0 +1,402 @@ +package anyblockjson + +// index.json (§2c) is the only document that describes the bundle rather than +// one object: the space's name, what opens on entry, what the sidebar shows. + +import ( + "errors" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIndex_Roundtrip(t *testing.T) { + doc := `{ + "$schema": "https://schemas.anytype.io/anyblock/1.0/index.schema.json", + "version": 2, + "name": "Company Wiki", + "description": "Everything we know, with an owner.", + "icon": { "format": "emoji", "emoji": "📚" }, + "homepage": "page-wiki-home", + "widgets": [ + { "target": "page-wiki-home" }, + { "target": "type-wiki-page", "layout": "view", "limit": 6, "view_id": "view-board" }, + { "target": "_favorite", "layout": "compact_list" }, + { "target": "_all_objects", "auto_added": true, + "card_style": "card", "icon_size": "medium", "description": "content", + "properties": ["name", "created_date"] } + ], + "auto_widget_targets": ["_bin", "type-wiki-page"], + "auto_widget_disabled": true + }` + idx, err := UnmarshalIndex([]byte(doc)) + require.NoError(t, err) + + assert.Equal(t, "Company Wiki", idx.Name) + assert.Equal(t, "page-wiki-home", idx.Homepage) + require.Len(t, idx.Widgets, 4) + assert.Equal(t, "", idx.Widgets[0].Layout, "omitted layout stays empty; link is the default") + assert.Equal(t, "view", idx.Widgets[1].Layout) + assert.Equal(t, int32(6), idx.Widgets[1].Limit) + assert.Equal(t, "view-board", idx.Widgets[1].ViewId) + assert.True(t, idx.Widgets[3].AutoAdded) + assert.Equal(t, "card", idx.Widgets[3].CardStyle) + assert.Equal(t, "medium", idx.Widgets[3].IconSize) + assert.Equal(t, "content", idx.Widgets[3].Description) + assert.Equal(t, []string{"name", "createdDate"}, idx.Widgets[3].Properties, + "shown properties are held as STORED keys, like the manifest's type keys (§2c)") + assert.Equal(t, []string{"_bin", "type-wiki-page"}, idx.AutoWidgetTargets) + assert.True(t, idx.AutoWidgetDisabled) + + // the install opens the first widget's target + assert.Equal(t, "page-wiki-home", idx.EntryPoint()) + + out, err := MarshalIndex(idx) + require.NoError(t, err) + again, err := UnmarshalIndex(out) + require.NoError(t, err) + out2, err := MarshalIndex(again) + require.NoError(t, err) + assert.Equal(t, string(out), string(out2), "export must be byte-stable (§11)") +} + +func TestIndex_EntryPoint(t *testing.T) { + t.Run("the declared entrypoint wins over widget order", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, + "entrypoint": "page-home", + "widgets": [{"target": "type-task", "layout": "view"}]}`)) + require.NoError(t, err) + assert.Equal(t, "page-home", idx.EntryPoint(), + "reordering the sidebar must not change what opens") + }) + + t.Run("nothing declared means no entry point", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, "name": "X"}`)) + require.NoError(t, err) + assert.Empty(t, idx.EntryPoint()) + }) + + // bundles written before entrypoint existed carried it as widgets[0] + t.Run("falls back to the first widget naming an object", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, + "widgets": [{"target": "_recent"}, {"target": "page-home"}]}`)) + require.NoError(t, err) + assert.Equal(t, "page-home", idx.EntryPoint(), "reserved listings are skipped") + }) + + // The listing this skips is spelled `_recent`, and an object id may not + // begin with `_`, so "skip the reserved ones" is now a statement about + // disjoint namespaces rather than a race between two flat word lists. It + // used to be the latter: a bundle shipping an object with id `recent` made + // EntryPoint skip its own object, so the entry point the tooling reported + // was not the one the installer opened. + t.Run("the skipped targets cannot be bundle ids", func(t *testing.T) { + for _, target := range ReservedWidgetTargets() { + assert.True(t, IsPlatformId(target), + "a reserved listing must live in the platform namespace, or an object can shadow it: %q", target) + } + for _, home := range []string{HomepageWidgets, HomepageGraph} { + assert.True(t, IsPlatformId(home), home) + } + }) + + t.Run("a reserved listing cannot be an entrypoint", func(t *testing.T) { + for _, bad := range []string{"_widgets", "_graph", "_favorite", "_recent", "_all_objects"} { + _, err := UnmarshalIndex([]byte(`{"version": 2, "entrypoint": "` + bad + `"}`)) + require.Error(t, err, bad) + } + }) + + // the reason the entrypoint ban is a prefix and not a word list + t.Run("no entrypoint may enter the platform namespace", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version": 2, "entrypoint": "_otpage"}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/entrypoint") + assert.Contains(t, err.Error(), "platform") + }) +} + +// homepage is what opens on every later entry; omitting it means "the same +// page you landed on", never the widgets screen +func TestIndex_SpaceHomepage(t *testing.T) { + t.Run("defaults to the entrypoint", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, "entrypoint": "page-home"}`)) + require.NoError(t, err) + assert.Equal(t, "page-home", idx.SpaceHomepage()) + }) + t.Run("an explicit value wins", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, + "entrypoint": "page-welcome", "homepage": "page-dashboard"}`)) + require.NoError(t, err) + assert.Equal(t, "page-dashboard", idx.SpaceHomepage()) + }) + t.Run("a reserved homepage is still allowed, deliberately", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, + "entrypoint": "page-home", "homepage": "_graph"}`)) + require.NoError(t, err) + assert.Equal(t, "_graph", idx.SpaceHomepage()) + assert.True(t, IsReservedHomepage(idx.SpaceHomepage())) + }) + + // The bare word is what core/domain/homepage.go and builtinobjects use, and + // setWorkspaceSettings matches it BEFORE trying to resolve an id — so while + // the format spelled it `graph` too, an object with that id could never be + // a homepage. Here it is an ordinary id and nothing reserved is involved. + t.Run("the wire spelling of a reserved screen is an ordinary id", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, "homepage": "graph"}`)) + require.NoError(t, err) + assert.False(t, IsReservedHomepage("graph")) + assert.Equal(t, "graph", idx.SpaceHomepage()) + }) + + // what pb.Profile.SpaceDashboardId has to carry + t.Run("a reserved screen is translated for the wire", func(t *testing.T) { + assert.Equal(t, "graph", WireHomepage(HomepageGraph)) + assert.Equal(t, "widgets", WireHomepage(HomepageWidgets)) + assert.Equal(t, "page-home", WireHomepage("page-home"), "an object id passes through") + }) +} + +func TestIndex_Validation(t *testing.T) { + for _, tc := range []struct{ name, doc, want string }{ + {"version required", `{"name": "X"}`, "version"}, + {"unknown layout", `{"version": 2, "widgets": [{"target": "a", "layout": "grid"}]}`, "layout"}, + {"widget needs a target", `{"version": 2, "widgets": [{"layout": "link"}]}`, "target"}, + {"unknown property", `{"version": 2, "startingPage": "a"}`, "startingPage"}, + {"limit must be an integer", `{"version": 2, "widgets": [{"target": "a", "limit": 1.5}]}`, "limit"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := UnmarshalIndex([]byte(tc.doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } + + t.Run("reserved homepage names are accepted", func(t *testing.T) { + for _, h := range []string{"_widgets", "_graph"} { + _, err := UnmarshalIndex([]byte(`{"version": 2, "homepage": "` + h + `"}`)) + assert.NoError(t, err, h) + assert.True(t, IsReservedHomepage(h)) + } + }) + + t.Run("an object id is not reserved", func(t *testing.T) { + assert.False(t, IsReservedHomepage("page-wiki-home")) + assert.False(t, IsReservedWidgetTarget("page-wiki-home")) + assert.True(t, IsReservedWidgetTarget("_favorite")) + }) + + // The listings the importer knows are bare words; the format spells them + // with the platform prefix so nothing a bundle ships can collide. The + // unprefixed spelling is therefore an ordinary bundle id and reserves + // nothing — which is the whole content of the rename. + t.Run("the bare listing name reserves nothing", func(t *testing.T) { + for _, bare := range []string{"favorite", "recent", "set", "collection"} { + assert.False(t, IsReservedWidgetTarget(bare), bare) + _, err := UnmarshalIndex([]byte(`{"version": 2, "entrypoint": "` + bare + `"}`)) + assert.NoError(t, err, "an ordinary id: %s", bare) + } + }) + + // A `_` target that is not one of the six resolves to nothing, and a + // widget target that resolves to nothing is dropped on install with no + // error at all — so it has to be refused here, by name, with the + // inventory in the message. + // The schema states the rule too, so asserting only that this is refused + // would stay green with the gate deleted. What the gate is FOR is the + // message: an anonymous `does not match pattern '^[^_]'` names neither the + // namespace nor the repair, and the failure it precedes is invisible. + t.Run("an unknown reserved listing is refused, and named", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version": 2, "widgets": [ + {"target": "page-home"}, {"target": "_favourite"}]}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/widgets/1/target") + assert.Contains(t, err.Error(), "_favourite") + assert.Contains(t, err.Error(), "is not a reserved listing") + assert.Contains(t, err.Error(), "_favorite", "the message must carry the inventory") + assert.Contains(t, err.Error(), "dropped on install without an error", + "and must say what happens if it is not caught here") + }) + + t.Run("an unknown reserved homepage is refused", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version": 2, "homepage": "_last_opened"}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/homepage") + assert.Contains(t, err.Error(), "the only reserved homepages are") + }) + + // an index shares the format version and its rules with object documents + // (§10): a newer one is rejected with both versions named, not with a + // generic schema constraint failure + t.Run("a newer version is rejected, naming both versions", func(t *testing.T) { + // given a bundle index from a future format version, carrying a key + // this reader has never heard of + data := []byte(`{"version": 3, "name": "Wiki", "futureKey": true}`) + + // when + _, err := UnmarshalIndex(data) + + // then + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + assert.True(t, ve.NewerFormat, "must be flagged as a newer format, not a constraint failure") + assert.Contains(t, err.Error(), "newer version") + assert.Contains(t, err.Error(), "3") + assert.Contains(t, err.Error(), strconv.Itoa(FormatVersion)) + // the version gate ran before the schema, so the unknown key never + // produced an issue of its own + assert.NotContains(t, err.Error(), "futureKey") + }) + + t.Run("a missing version is rejected", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"name": "Wiki"}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "version is required") + }) + + t.Run("a non-object index is rejected cleanly", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`[1, 2, 3]`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "index must be a JSON object") + }) +} + +// Every reserved listing survives import now. This test used to pin the +// OPPOSITE for _all_objects and _recent_open — the importer knew only four +// listings, so a bundle naming the others lost the widget silently and the +// tooling refused them up front. widget.IsPredefinedWidgetTargetId knows the +// whole inventory since GO-7383, so the pin flips: reserved and importable +// must now agree member for member, and a listing added to one set without +// the other is exactly what this catches. +func TestIndex_ImportableWidgetTargets(t *testing.T) { + for _, target := range ReservedWidgetTargets() { + assert.True(t, IsImportableWidgetTarget(target), + "a reserved listing the importer does not know loses the widget silently on install: "+target) + } + assert.False(t, IsImportableWidgetTarget("page-wiki-home")) + assert.False(t, IsImportableWidgetTarget("_widgets"), "a homepage screen is not a widget target") +} + +// The importable listings are the ones the pb importer knows by their bare, +// unprefixed wire names (widget.IsPredefinedWidgetTargetId), so the format's +// `_` spelling only holds together if every one of them has a wire spelling +// and the wiring applies it BOTH ways. Writing `_set` into a link block +// instead of `set` is strictly worse than the shadowing bug this replaces: +// handleLinkBlock rewrites the unrecognised target to addr.MissingObject and +// WidgetObject.Init then strips the link and its wrapper, so the widget +// vanishes with no error. And lifting a stored `bin` without translating it +// puts a bare wire word where the index promises an object id or a `_` name. +func TestIndex_WireWidgetTargets(t *testing.T) { + want := map[string]string{ + "_favorite": "favorite", "_recent": "recent", "_set": "set", "_collection": "collection", + "_all_objects": "allObjects", "_recent_open": "recentOpen", "_chat": "chat", "_bin": "bin", + } + for target, wire := range want { + assert.Equal(t, wire, WireWidgetTarget(target), target) + assert.NotEqual(t, target, WireWidgetTarget(target), + "the platform prefix must be translated away, or the importer drops the widget") + assert.Equal(t, target, FormatWidgetTarget(wire), + "the lift must invert the wire spelling exactly, or a round trip respells the target: "+wire) + } + for _, target := range ReservedWidgetTargets() { + assert.Contains(t, want, target, "every listing needs a wire spelling") + } + assert.Equal(t, "page-wiki-home", WireWidgetTarget("page-wiki-home"), "an object id passes through") + assert.Equal(t, "bafyrei123", FormatWidgetTarget("bafyrei123"), "an object id passes through the lift too") +} + +// The index schema names the object schema by its published URL to $ref the +// shared icon definition (§2b). That URL is derived from FormatVersion +// everywhere else, so a version bump would leave this one spelling behind — +// the compiler catches it (every index test fails at once), but only this +// says which line to fix. +func TestIndexSchema_RefsThePublishedObjectSchema(t *testing.T) { + assert.Contains(t, string(indexSchemaJSON), SchemaURL+"#/$defs/icon") +} + +// A bundle index carries the SAME typed icon an object does (§2b, §2c) — the +// whole shape, not a restriction of it. The image variant names an object id; +// the wiring resolves it to the image's name, because the installer looks the +// space icon up by name (getNewAvatarId). +// +// How this can fail: give the index its own icon shape, or let both an emoji +// and an image be present at once, and one of these breaks. The old two-key +// index accepted both with no rule for which wins — the last assertion here +// used to say "the installer prefers the image", which was a rule written +// nowhere in the schema. +func TestIndex_Icon(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, "name": "Wiki", + "icon": {"format": "file", "file": "acme-logo"}}`)) + require.NoError(t, err) + assert.Equal(t, "acme-logo", idx.IconImageId()) + + out, err := MarshalIndex(idx) + require.NoError(t, err) + assert.Contains(t, string(out), `"icon": {`) + assert.Contains(t, string(out), `"file": "acme-logo"`) + + t.Run("an emoji index icon has no image id", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, "icon": {"format": "emoji", "emoji": "📚"}}`)) + require.NoError(t, err) + assert.Equal(t, "📚", idx.Icon.Emoji) + assert.Empty(t, idx.IconImageId()) + }) + + t.Run("an emoji and an image are no longer both writable", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version": 2, + "icon": {"format": "emoji", "emoji": "📚", "file": "logo"}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), `property "file" is not allowed`) + }) + + t.Run("the discriminator names the alternatives when it is missing", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version": 2, "icon": {"emoji": "📚"}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/icon: missing property 'format'") + assert.Contains(t, err.Error(), "'emoji', 'file'") + }) + + // The index took the narrow emoji-or-image shape until the space's own + // document stopped being exported (§2c) and the index became the only + // place a space icon could live. A LETTER AVATAR — a colour and nothing + // else — is the icon of 20 of 77 real spaces, and the narrow shape had + // no way to spell one, so omitting the document would have deleted it. + t.Run("a letter avatar is a colour and nothing else", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, "icon": {"format": "color", "color": "red"}}`)) + require.NoError(t, err) + assert.Equal(t, "red", idx.Icon.Color) + assert.Empty(t, idx.IconImageId(), "a colour names no image") + + out, err := MarshalIndex(idx) + require.NoError(t, err) + assert.Contains(t, string(out), `"format": "color"`) + assert.Contains(t, string(out), `"color": "red"`) + }) + + t.Run("a colour rides along with the icon it tints", func(t *testing.T) { + idx, err := UnmarshalIndex([]byte(`{"version": 2, + "icon": {"format": "file", "file": "acme-logo", "color": "red"}}`)) + require.NoError(t, err) + out, err := MarshalIndex(idx) + require.NoError(t, err) + assert.Contains(t, string(out), `"file": "acme-logo"`) + assert.Contains(t, string(out), `"color": "red"`) + }) + + // One renderer, so the index and the object surface cannot drift into two + // spellings of one icon (§2b). + t.Run("the index renders through the object surface's own renderer", func(t *testing.T) { + for _, ic := range []*Icon{ + {Format: "emoji", Emoji: "📚"}, + {Format: "file", File: "logo"}, + {Format: "file", File: "logo", Color: "red"}, + {Format: "color", Color: "red"}, + {Format: "icon", Name: "folder", Color: "red"}, + } { + assert.Equal(t, iconOmap(ic), indexIconOmap(ic), "icon %+v", ic) + } + }) +} diff --git a/pkg/lib/anyblockjson/inline.go b/pkg/lib/anyblockjson/inline.go new file mode 100644 index 0000000000..f8fb2b446d --- /dev/null +++ b/pkg/lib/anyblockjson/inline.go @@ -0,0 +1,1846 @@ +package anyblockjson + +// inline.go implements the §8 inline-markup codec: canonical rendering of +// text marks into the Markdown subset, and the inverse parser. The parser is +// the exact inverse of the renderer: it resolves emphasis with a +// deterministic delimiter stack (not CommonMark's delimiter-run algorithm), +// which is what makes Export ∘ Import byte-stable over arbitrarily +// overlapping mark ranges. All offsets are UTF-16 code units (§8.3). + +import ( + "fmt" + "net/url" + "sort" + "strings" + "unicode" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/text" +) + +// An Object mark's target renders as Anytype's deep link. The form is exact: +// scheme "anytype", host "object", and a single "object_id" query parameter — +// nothing else. Matching it by string prefix instead would take everything +// after "object_id=" as the id, so the platform's own two-parameter link +// (core/block/export/writer.go) would yield the id "&spaceId=", +// and an id could smuggle query parameters into a link other tools resolve. +const ( + objectLinkScheme = "anytype" + objectLinkHost = "object" + objectLinkParam = "objectId" +) + +// objectLinkDest renders an object id as the canonical deep link, percent- +// encoding the id so it cannot introduce a second parameter. Ordinary ids are +// CIDs, so in practice nothing is escaped and the bytes are unchanged. +func objectLinkDest(id string) string { + return objectLinkScheme + "://" + objectLinkHost + "?" + + url.Values{objectLinkParam: {id}}.Encode() +} + +// parseObjectLink reports the object id a destination names, and whether the +// destination is exactly that canonical link. Anything else — extra query +// parameters, another host, another scheme, a path — is not an object +// reference: it stays a plain Link, preserved verbatim, which is lossless. +// Accepting a superset and dropping the extras would not be (§8.1). +func parseObjectLink(dest string) (string, bool) { + if !strings.HasPrefix(dest, objectLinkScheme+"://"+objectLinkHost+"?") { + return "", false + } + u, err := url.Parse(dest) + if err != nil || u.Scheme != objectLinkScheme || u.Host != objectLinkHost || + u.Path != "" || u.Fragment != "" || u.User != nil { + return "", false + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil || len(q) != 1 || len(q[objectLinkParam]) != 1 { + return "", false + } + id := q[objectLinkParam][0] + if id == "" { + return "", false + } + return id, true +} + +// isObjectLink reports whether a Link mark's target is the canonical object +// deep link, and so renders as an Object mark (§8.3). +func isObjectLink(dest string) bool { + _, ok := parseObjectLink(dest) + return ok +} + +// markNesting is the fixed outermost→innermost nesting order (§8.3 step 5). +var markNesting = []model.BlockContentTextMarkType{ + model.BlockContentTextMark_Mention, + model.BlockContentTextMark_Object, + model.BlockContentTextMark_Link, + model.BlockContentTextMark_TextColor, + model.BlockContentTextMark_BackgroundColor, + model.BlockContentTextMark_Underscored, + model.BlockContentTextMark_Strikethrough, + model.BlockContentTextMark_Bold, + model.BlockContentTextMark_Italic, + model.BlockContentTextMark_Keyboard, +} + +var markPriority = func() map[model.BlockContentTextMarkType]int { + m := make(map[model.BlockContentTextMarkType]int, len(markNesting)) + for i, t := range markNesting { + m[t] = i + } + return m +}() + +func markNeedsParam(t model.BlockContentTextMarkType) bool { + switch t { + case model.BlockContentTextMark_Link, + model.BlockContentTextMark_TextColor, + model.BlockContentTextMark_BackgroundColor, + model.BlockContentTextMark_Mention, + model.BlockContentTextMark_Object, + model.BlockContentTextMark_Emoji: + return true + } + return false +} + +// span is a mark over UTF-16 code-unit offsets. +type span struct { + typ model.BlockContentTextMarkType + param string + from, to int +} + +func sortSpans(s []span) { + sort.SliceStable(s, func(i, j int) bool { + if s[i].from != s[j].from { + return s[i].from < s[j].from + } + if s[i].to != s[j].to { + return s[i].to > s[j].to + } + return s[i].param < s[j].param + }) +} + +// renderInline serializes text and its marks into §8 inline Markdown. +func renderInline(txt string, marks []*model.BlockContentTextMark) string { + u16 := text.StrToUTF16(txt) + spans := sanitizeSpans(u16, marks) + u16, spans = materializeEmoji(u16, spans) + spans = shrinkWhitespaceBoundaries(u16, spans) + spans = resolveSameTypeOverlaps(spans) + spans = splitEmphasisAtBoundaryWhitespace(u16, spans) + return emitSegments(u16, spans) +} + +// sanitizeSpans drops nil, zero-length, out-of-bounds and surrogate-splitting +// ranges, unknown mark types and empty params on param-carrying marks (§8.3 +// step 1). +func sanitizeSpans(u16 []uint16, marks []*model.BlockContentTextMark) []span { + spans := make([]span, 0, len(marks)) + for _, m := range marks { + if m == nil || m.Range == nil { + continue + } + from, to := int(m.Range.From), int(m.Range.To) + if from < 0 || to > len(u16) || from >= to { + continue + } + if splitsSurrogatePair(u16, from) || splitsSurrogatePair(u16, to) { + continue + } + if _, known := markPriority[m.Type]; !known && m.Type != model.BlockContentTextMark_Emoji { + continue + } + typ := m.Type + param := m.Param + // a Link whose target is an object deep-link renders identically to + // an Object mark, so it normalizes to one — otherwise the parse-back + // type flip would break same-type overlap resolution (§8.3) + if typ == model.BlockContentTextMark_Link { + if id, ok := parseObjectLink(param); ok { + typ = model.BlockContentTextMark_Object + param = id + } + } + if markNeedsParam(typ) { + if param == "" { + continue + } + } else { + // a param on a param-less mark type is noise; normalizing it to + // empty lets equal-range marks merge (§8.3) + param = "" + } + // params beyond the §8 resource bounds are invalid: the parser will + // not recognize them, so rendering them would not round-trip + switch typ { + case model.BlockContentTextMark_Link: + if text.UTF16RuneCountString(param) > maxLinkDestLen { + continue + } + case model.BlockContentTextMark_Object: + if text.UTF16RuneCountString(objectLinkDest(param)) > maxLinkDestLen { + continue + } + case model.BlockContentTextMark_Emoji: + if text.UTF16RuneCountString(param) > maxEmojiParamLen { + continue + } + } + spans = append(spans, span{typ: typ, param: param, from: from, to: to}) + } + return spans +} + +func splitsSurrogatePair(u16 []uint16, i int) bool { + if i <= 0 || i >= len(u16) { + return false + } + return isHighSurrogate(u16[i-1]) && isLowSurrogate(u16[i]) +} + +func isHighSurrogate(u uint16) bool { return u >= 0xD800 && u <= 0xDBFF } +func isLowSurrogate(u uint16) bool { return u >= 0xDC00 && u <= 0xDFFF } + +// materializeEmoji splices each Emoji mark's emoji over its covered text +// (§8.1), adjusting the remaining marks' offsets. Overlapping emoji marks are +// truncated earlier-start-wins first (§8.3 step 3 semantics). +func materializeEmoji(u16 []uint16, spans []span) ([]uint16, []span) { + var emoji, rest []span + for _, s := range spans { + if s.typ == model.BlockContentTextMark_Emoji { + emoji = append(emoji, s) + } else { + rest = append(rest, s) + } + } + if len(emoji) == 0 { + return u16, rest + } + sortSpans(emoji) + var acc []span + for _, e := range emoji { + for i := range acc { + if e.from < acc[i].to { + e.from = acc[i].to + } + } + if e.from < e.to && !splitsSurrogatePair(u16, e.from) { + acc = append(acc, e) + } + } + // splice all replacements in one pass; acc is ascending and disjoint + reps := make([][]uint16, len(acc)) + total := 0 + for i, e := range acc { + reps[i] = text.StrToUTF16(e.param) + total += len(reps[i]) - (e.to - e.from) + } + nu := make([]uint16, 0, len(u16)+total) + prev := 0 + for i, e := range acc { + nu = append(nu, u16[prev:e.from]...) + nu = append(nu, reps[i]...) + prev = e.to + } + nu = append(nu, u16[prev:]...) + u16 = nu + for j := range rest { + rest[j].from = adjustAcrossSplices(rest[j].from, acc, reps, true) + rest[j].to = adjustAcrossSplices(rest[j].to, acc, reps, false) + } + out := rest[:0] + for _, s := range rest { + if s.from < s.to { + out = append(out, s) + } + } + return u16, out +} + +// adjustAcrossSplices maps offset p across the ordered disjoint splices. +// Points strictly inside a replaced range retract to exclude the +// replacement: starts land after it, ends before it. +func adjustAcrossSplices(p int, acc []span, reps [][]uint16, isStart bool) int { + delta := 0 + for k, e := range acc { + if p <= e.from { + break + } + if p >= e.to { + delta += len(reps[k]) - (e.to - e.from) + continue + } + if isStart { + return e.from + delta + len(reps[k]) + } + return e.from + delta + } + return p + delta +} + +func isMarkdownDelimited(t model.BlockContentTextMarkType) bool { + switch t { + case model.BlockContentTextMark_Bold, + model.BlockContentTextMark_Italic, + model.BlockContentTextMark_Strikethrough, + model.BlockContentTextMark_Keyboard: + return true + } + return false +} + +// shrinkWhitespaceBoundaries shrinks Markdown-delimited marks past +// leading/trailing whitespace (§8.3 step 2). +func shrinkWhitespaceBoundaries(u16 []uint16, spans []span) []span { + out := spans[:0] + for _, s := range spans { + if isMarkdownDelimited(s.typ) { + for s.from < s.to && isWSUnit(u16, s.from) { + s.from++ + } + for s.to > s.from && isWSUnit(u16, s.to-1) { + s.to-- + } + if s.from >= s.to { + continue + } + } + out = append(out, s) + } + return out +} + +func isWSUnit(u16 []uint16, i int) bool { + u := u16[i] + if isHighSurrogate(u) || isLowSurrogate(u) { + return false + } + return unicode.IsSpace(rune(u)) +} + +// resolveSameTypeOverlaps applies §8.3 step 3: same-type marks with equal +// params merge when overlapping or adjacent; with different params the +// earlier-starting mark wins and the later is truncated to start where the +// earlier ends. At equal starts the longer range wins (sort order). A merge +// that extends an accepted range can create a fresh overlap with a +// later-accepted range, so each group re-runs until stable — resolution must +// be idempotent for §11 byte-stability. +func resolveSameTypeOverlaps(spans []span) []span { + byType := make(map[model.BlockContentTextMarkType][]span) + for _, s := range spans { + byType[s.typ] = append(byType[s.typ], s) + } + var out []span + for _, t := range markNesting { + group := byType[t] + if len(group) == 0 { + continue + } + for { + sortSpans(group) + var acc []span + extended := false + for _, m := range group { + dropped := false + for i := range acc { + a := &acc[i] + if m.from > a.to { + continue + } + if m.param == a.param { + if m.to > a.to { + a.to = m.to + extended = true + } + dropped = true + break + } + if m.from < a.to { + m.from = a.to + if m.from >= m.to { + dropped = true + break + } + } + } + if !dropped { + acc = append(acc, m) + } + } + group = acc + // every extending pass consumed at least one span, so this + // terminates + if !extended { + break + } + } + out = append(out, group...) + } + sortSpans(out) + return out +} + +func isEmphasisFamily(t model.BlockContentTextMarkType) bool { + switch t { + case model.BlockContentTextMark_Bold, + model.BlockContentTextMark_Italic, + model.BlockContentTextMark_Strikethrough: + return true + } + return false +} + +// splitEmphasisAtBoundaryWhitespace removes from emphasis-family marks every +// whitespace run that a stack-outer mark's endpoint touches. Such an endpoint +// forces the emphasis delimiter to close/reopen inside the run, and an +// emphasis delimiter emitted against whitespace cannot re-parse (flanking). +// Whitespace carries no visible styling for these types, so splitting the +// span around the run is a rendering no-op. Endpoints of inner marks are +// harmless: the shared-prefix emission keeps the outer emphasis open across +// them. Runs to a fixpoint because each split introduces new endpoints. +func splitEmphasisAtBoundaryWhitespace(u16 []uint16, spans []span) []span { + type sortKey struct { + prio int + param string + } + keyOf := func(s span) sortKey { return sortKey{markPriority[s.typ], s.param} } + outerBefore := func(a, b sortKey) bool { + if a.prio != b.prio { + return a.prio < b.prio + } + return a.param < b.param + } + for { + changed := false + out := make([]span, 0, len(spans)) + for _, s := range spans { + if !isEmphasisFamily(s.typ) { + out = append(out, s) + continue + } + sk := keyOf(s) + cutsRun := func(i, j int) bool { + for _, t := range spans { + if !outerBefore(keyOf(t), sk) { + continue + } + if (t.from >= i && t.from <= j) || (t.to >= i && t.to <= j) { + return true + } + } + return false + } + cur := s.from + for i := s.from; i < s.to; { + if !isWSUnit(u16, i) { + i++ + continue + } + j := i + for j < s.to && isWSUnit(u16, j) { + j++ + } + if cutsRun(i, j) { + if cur < i { + out = append(out, span{typ: s.typ, param: s.param, from: cur, to: i}) + } + cur = j + changed = true + } + i = j + } + // spans are already whitespace-shrunk, so cur < s.to always holds + out = append(out, span{typ: s.typ, param: s.param, from: cur, to: s.to}) + } + spans = out + if !changed { + sortSpans(spans) + return spans + } + } +} + +// stackItem identifies one active mark on a segment. +type stackItem struct { + typ model.BlockContentTextMarkType + param string +} + +// treeNode / treeKid form the well-nested render tree built from segment +// stacks (§8.3 steps 4–5). +type treeNode struct { + item stackItem + root bool + kids []*treeKid +} + +type treeKid struct { + node *treeNode // nil for a text segment + from, to int + atBOL bool + atEOL bool +} + +func emitSegments(u16 []uint16, spans []span) string { + root := &treeNode{root: true} + if len(u16) > 0 { + bounds := collectBounds(len(u16), spans) + path := []*treeNode{root} + for i := 0; i+1 < len(bounds); i++ { + segFrom, segTo := bounds[i], bounds[i+1] + target := activeItems(spans, segFrom, segTo) + l := 0 + for l < len(target) && l+1 < len(path) && path[l+1].item == target[l] { + l++ + } + path = path[:l+1] + for _, it := range target[l:] { + n := &treeNode{item: it} + top := path[len(path)-1] + top.kids = append(top.kids, &treeKid{node: n}) + path = append(path, n) + } + top := path[len(path)-1] + top.kids = append(top.kids, &treeKid{ + from: segFrom, + to: segTo, + atBOL: segFrom == 0 && len(target) == 0, + atEOL: segTo == len(u16) && len(target) == 0, + }) + } + } + var b strings.Builder + renderNode(&b, u16, root, false) + return b.String() +} + +func collectBounds(length int, spans []span) []int { + set := map[int]struct{}{0: {}, length: {}} + for _, s := range spans { + set[s.from] = struct{}{} + set[s.to] = struct{}{} + } + bounds := make([]int, 0, len(set)) + for b := range set { + bounds = append(bounds, b) + } + sort.Ints(bounds) + return bounds +} + +func activeItems(spans []span, from, to int) []stackItem { + var items []stackItem + for _, s := range spans { + if s.from <= from && s.to >= to { + items = append(items, stackItem{typ: s.typ, param: s.param}) + } + } + sort.SliceStable(items, func(i, j int) bool { + pi, pj := markPriority[items[i].typ], markPriority[items[j].typ] + if pi != pj { + return pi < pj + } + return items[i].param < items[j].param + }) + return items +} + +func renderNode(b *strings.Builder, u16 []uint16, n *treeNode, inLabel bool) { + renderKids := func(label bool) { + for _, k := range n.kids { + if k.node != nil { + renderNode(b, u16, k.node, label) + } else { + b.WriteString(escapeProse(text.UTF16ToStr(u16[k.from:k.to]), k.atBOL, k.atEOL, label)) + } + } + } + if n.root { + renderKids(inLabel) + return + } + switch n.item.typ { + case model.BlockContentTextMark_Mention: + b.WriteString(``) + renderKids(inLabel) + b.WriteString(``) + case model.BlockContentTextMark_Object: + b.WriteByte('[') + renderKids(true) + b.WriteString("](" + escapeDest(objectLinkDest(n.item.param)) + ")") + case model.BlockContentTextMark_Link: + b.WriteByte('[') + renderKids(true) + b.WriteString("](" + escapeDest(n.item.param) + ")") + case model.BlockContentTextMark_TextColor: + // Coincident color+background ranges combine into one tag (§8.1): + // in the tree that is a TextColor node whose sole child is a + // BackgroundColor node. + if len(n.kids) == 1 && n.kids[0].node != nil && + n.kids[0].node.item.typ == model.BlockContentTextMark_BackgroundColor { + bg := n.kids[0].node + b.WriteString(``) + renderNode(b, u16, &treeNode{root: true, kids: bg.kids}, inLabel) + b.WriteString(``) + return + } + b.WriteString(``) + renderKids(inLabel) + b.WriteString(``) + case model.BlockContentTextMark_BackgroundColor: + b.WriteString(``) + renderKids(inLabel) + b.WriteString(``) + case model.BlockContentTextMark_Underscored: + b.WriteString(``) + renderKids(inLabel) + b.WriteString(``) + case model.BlockContentTextMark_Strikethrough: + b.WriteString(`~~`) + renderKids(inLabel) + b.WriteString(`~~`) + case model.BlockContentTextMark_Bold: + b.WriteString(`**`) + renderKids(inLabel) + b.WriteString(`**`) + case model.BlockContentTextMark_Italic: + b.WriteString(`*`) + renderKids(inLabel) + b.WriteString(`*`) + case model.BlockContentTextMark_Keyboard: + // Keyboard is innermost: its content is always a single text segment. + var content strings.Builder + for _, k := range n.kids { + if k.node == nil { + content.WriteString(text.UTF16ToStr(u16[k.from:k.to])) + } + } + writeCodeSpan(b, content.String()) + } +} + +// writeCodeSpan emits a CommonMark code span: the delimiter is the shortest +// backtick run absent from the content, space-padded when the content starts +// or ends with a backtick or would trigger the strip rule (§8.2). +func writeCodeSpan(b *strings.Builder, content string) { + runs := map[int]bool{} + run := 0 + for _, r := range content { + if r == '`' { + run++ + } else if run > 0 { + runs[run] = true + run = 0 + } + } + if run > 0 { + runs[run] = true + } + n := 1 + for runs[n] { + n++ + } + delim := strings.Repeat("`", n) + pad := strings.HasPrefix(content, "`") || strings.HasSuffix(content, "`") + if !pad && len(content) > 0 { + first, last := content[0], content[len(content)-1] + startsWS := first == ' ' || first == '\n' + endsWS := last == ' ' || last == '\n' + if startsWS && endsWS && strings.Trim(content, " \n") != "" { + pad = true + } + } + b.WriteString(delim) + if pad { + b.WriteByte(' ') + } + b.WriteString(content) + if pad { + b.WriteByte(' ') + } + b.WriteString(delim) +} + +const ( + edgeWS = iota + edgePunct + edgeWord +) + +func classifyRune(r rune) int { + if unicode.IsSpace(r) { + return edgeWS + } + if isPunctRune(r) { + return edgePunct + } + return edgeWord +} + +// escapeProse writes a text segment with canonical minimal escaping (§8.2). +// atBOL/atEOL report whether the segment starts/ends the whole rendered +// string; at internal segment edges the neighbor is a delimiter and is +// treated as punctuation. +func escapeProse(s string, atBOL, atEOL, inLabel bool) string { + rs := []rune(s) + var b strings.Builder + kindAt := func(i int) int { + if i < 0 { + if atBOL { + return edgeWS + } + return edgePunct + } + if i >= len(rs) { + if atEOL { + return edgeWS + } + return edgePunct + } + return classifyRune(rs[i]) + } + for i, r := range rs { + prev, next := kindAt(i-1), kindAt(i+1) + switch r { + case '\\': + if i+1 < len(rs) { + if isASCIIPunct(rs[i+1]) { + b.WriteString(`\\`) + } else { + b.WriteByte('\\') + } + } else if atEOL { + b.WriteByte('\\') + } else { + // a trailing backslash would escape the following delimiter + b.WriteString(`\\`) + } + case '`': + b.WriteString("\\`") + case '*': + if prev == edgeWS && next == edgeWS { + b.WriteByte('*') + } else { + b.WriteString(`\*`) + } + case '_': + canOpen := next != edgeWS && prev != edgeWord + canClose := prev != edgeWS && next != edgeWord + if canOpen || canClose { + b.WriteString(`\_`) + } else { + b.WriteByte('_') + } + case '~': + adjacent := (i > 0 && rs[i-1] == '~') || (i+1 < len(rs) && rs[i+1] == '~') || + (i == 0 && !atBOL) || (i == len(rs)-1 && !atEOL) + if adjacent { + b.WriteString(`\~`) + } else { + b.WriteByte('~') + } + case '[': + // always escaped: a bare '[' could assemble a false link with + // text from later segments, which no local lookahead can rule out + b.WriteString(`\[`) + case ']': + if inLabel { + b.WriteString(`\]`) + } else { + b.WriteByte(']') + } + case '<': + if tagShaped(rs[i:]) { + b.WriteString(`\<`) + } else { + b.WriteByte('<') + } + case '&': + if entityAhead(rs[i:]) { + b.WriteString(`\&`) + } else { + b.WriteByte('&') + } + default: + b.WriteRune(r) + } + } + return b.String() +} + +// tagShaped reports whether rs starts a tag-shaped sequence: '<', an optional +// '/', then at least one ASCII letter. This is the whole reserved syntax space +// of the tag namespace, not just the three names version 1 knows (§8.2). +// +// Anchoring the escape on the whitelist instead would leave literal +// `x` bytes in canonical output, and the day a version adds `sub` +// those bytes become markup — with nothing in the text string to say which +// version wrote them, and a stricter reading (§8.3 makes a malformed instance +// of a *known* tag an error) turning old valid documents invalid. Escaping the +// shape costs a backslash on text that looks like markup and buys a tag +// namespace a later version can extend without a text-rewriting migration. +// +// It is deliberately the exact complement of the parser's leniency: import +// keeps an unrecognized tag-shaped sequence literal and warns (§10), and +// export escapes exactly what import warns about. +func tagShaped(rs []rune) bool { + _, ok := tagShapedName(rs) + return ok +} + +// tagShapedName returns the name of the tag-shaped sequence at rs[0] and +// whether rs is tag-shaped at all. +func tagShapedName(rs []rune) (string, bool) { + j := 1 + if j < len(rs) && rs[j] == '/' { + j++ + } + start := j + if j >= len(rs) || !isASCIILetter(rs[j]) { + return "", false // a tag name starts with a letter, always + } + for j < len(rs) && (isASCIILetter(rs[j]) || rs[j] == '_') { + j++ + } + return string(rs[start:j]), true +} + +func entityAhead(rs []rune) bool { + _, _, ok := parseEntity(rs) + return ok +} + +// escapeAttr entity-encodes attribute values. Brackets and backticks are +// encoded too: raw ones would derail the link-label scan when the tag sits +// inside a link label (the scan runs before tag parsing). +func escapeAttr(s string) string { + r := strings.NewReplacer( + "&", "&", "<", "<", ">", ">", `"`, """, + "[", "[", "]", "]", "`", "`", + ) + return r.Replace(s) +} + +// escapeDest renders a link destination: bare with \-escaped specials, or +// angle-wrapped when it contains whitespace. +func escapeDest(dest string) string { + needsAngle := false + for _, r := range dest { + if unicode.IsSpace(r) { + needsAngle = true + break + } + } + var b strings.Builder + // brackets and backticks are escaped in both forms: a raw ']' (or a + // backtick pairing with one in prose) inside a destination would derail + // the enclosing label scan when the link nests in another label + if needsAngle { + b.WriteByte('<') + for _, r := range dest { + switch r { + case '\\', '<', '>', '&', '[', ']', '`': + b.WriteByte('\\') + } + b.WriteRune(r) + } + b.WriteByte('>') + return b.String() + } + // '<' is escaped too: a bare destination starting with '<' would + // otherwise be misread as the angle-wrapped form + for _, r := range dest { + switch r { + case '\\', '(', ')', '&', '<', '[', ']', '`': + b.WriteByte('\\') + } + b.WriteRune(r) + } + return b.String() +} + +// +// ---- parsing ---- +// + +// inlineError is a grammar error in a text string's inline markup (§12). +type inlineError struct { + Msg string + Snippet string +} + +func (e *inlineError) Error() string { + if e.Snippet == "" { + return e.Msg + } + return fmt.Sprintf("%s near %q", e.Msg, e.Snippet) +} + +func inlineErr(rs []rune, pos int, msg string) error { + end := pos + 24 + if end > len(rs) { + end = len(rs) + } + start := pos + if start > len(rs) { + start = len(rs) + } + return &inlineError{Msg: msg, Snippet: string(rs[start:end])} +} + +// inlineNotes collects what a caller may want to report as warnings about a +// text string it parsed successfully (§12). Nothing here makes a document +// invalid, so the parser records instead of failing, and a nil sink is the +// no-op case for callers that do not report. +type inlineNotes struct { + // unknownTags names the tag-shaped sequences the grammar does not + // recognize (§10), deduplicated in first-seen order: one occurrence of + // `x` is one fact about the text, not two. + unknownTags []string +} + +func (n *inlineNotes) unknownTag(name string) { + if n == nil { + return + } + for _, seen := range n.unknownTags { + if seen == name { + return + } + } + n.unknownTags = append(n.unknownTags, name) +} + +// parseInline parses §8 inline Markdown back into plain text and marks with +// UTF-16 code-unit ranges. +func parseInline(md string) (string, []*model.BlockContentTextMark, error) { + txt, marks, _, err := parseInlineNotes(md) + return txt, marks, err +} + +// parseInlineNotes is parseInline plus the notes worth surfacing as warnings. +func parseInlineNotes(md string) (string, []*model.BlockContentTextMark, *inlineNotes, error) { + rs := []rune(md) + notes := &inlineNotes{} + toks, err := tokenizeInline(rs, 0, notes) + if err != nil { + return "", nil, nil, err + } + ib := &inlineBuilder{} + if err := resolveTokens(toks, ib); err != nil { + return "", nil, nil, err + } + ib.applyInserts() + marks := canonicalizeMarks(ib.marks) + return text.UTF16ToStr(ib.out), marks, notes, nil +} + +// Resource bounds (deterministic local rules, recorded in SPEC §8): they keep +// parsing linear on the untrusted-document boundary. +const ( + // maxLinkDestLen bounds a link destination; longer candidates are not + // links (the '[' stays literal). Export drops Link/Object marks whose + // param exceeds it, keeping the round trip stable. + maxLinkDestLen = 2048 + // maxLinkDestWS bounds the whitespace tolerated around a destination. + maxLinkDestWS = 32 + // maxLinkNesting bounds link-label nesting (CommonMark caps labels + // similarly); deeper '[' stay literal. + maxLinkNesting = 32 + // maxEmojiParamLen bounds an emoji mark's replacement text; longer + // params are invalid and dropped (§8.3 step 1). + maxEmojiParamLen = 64 +) + +// inlineScanCtx holds per-text precomputed scan tables so link and code-span +// lookups are O(log n) instead of rescanning the tail per candidate. +type inlineScanCtx struct { + btRuns map[int][]int // backtick run length -> ascending start positions + brackets map[int]int // '[' position -> matching ']' position +} + +func newInlineScanCtx(rs []rune) *inlineScanCtx { + ctx := &inlineScanCtx{btRuns: map[int][]int{}, brackets: map[int]int{}} + for i := 0; i < len(rs); { + if rs[i] == '`' { + n := runLen(rs, i, '`') + ctx.btRuns[n] = append(ctx.btRuns[n], i) + i += n + } else { + i++ + } + } + // bracket pairing with the tokenizer's exact skip rules (escapes, code + // spans); LIFO pairing equals the per-'[' depth scan it replaces + var stack []int + for i := 0; i < len(rs); { + c := rs[i] + if c == '\\' && i+1 < len(rs) && isASCIIPunct(rs[i+1]) { + i += 2 + continue + } + if c == '`' { + n := runLen(rs, i, '`') + if end, ok := ctx.backtickClose(i+n, n); ok { + i = end + n + } else { + i += n + } + continue + } + switch c { + case '[': + stack = append(stack, i) + case ']': + if len(stack) > 0 { + ctx.brackets[stack[len(stack)-1]] = i + stack = stack[:len(stack)-1] + } + } + i++ + } + return ctx +} + +// backtickClose finds the next maximal backtick run of exactly n starting at +// or after from. +func (ctx *inlineScanCtx) backtickClose(from, n int) (int, bool) { + runs := ctx.btRuns[n] + idx := sort.SearchInts(runs, from) + if idx < len(runs) { + return runs[idx], true + } + return 0, false +} + +type tokenKind int + +const ( + tokText tokenKind = iota + tokDelim + tokCode + tokTag + tokLink +) + +type token struct { + kind tokenKind + txt string // tokText: literal (decoded) text + ch rune // tokDelim: delimiter char + n int // tokDelim: run length + canOpen, canClose bool + content string // tokCode + tagName string // tokTag + closing bool // tokTag + attrs map[string]string // tokTag + label []token // tokLink + dest string // tokLink +} + +func tokenizeInline(rs []rune, depth int, notes *inlineNotes) ([]token, error) { + ctx := newInlineScanCtx(rs) + var toks []token + var pending strings.Builder + flushText := func() { + if pending.Len() > 0 { + toks = append(toks, token{kind: tokText, txt: pending.String()}) + pending.Reset() + } + } + appendText := func(s string) { + pending.WriteString(s) + } + emit := func(t token) { + flushText() + toks = append(toks, t) + } + i := 0 + for i < len(rs) { + switch r := rs[i]; r { + case '\\': + if i+1 < len(rs) && isASCIIPunct(rs[i+1]) { + appendText(string(rs[i+1])) + i += 2 + } else { + appendText(`\`) + i++ + } + case '&': + if dec, size, ok := parseEntity(rs[i:]); ok { + appendText(dec) + i += size + } else { + appendText("&") + i++ + } + case '`': + n := runLen(rs, i, '`') + if end, ok := ctx.backtickClose(i+n, n); ok { + emit(token{kind: tokCode, content: stripCodePadding(string(rs[i+n : end]))}) + i = end + n + } else { + appendText(strings.Repeat("`", n)) + i += n + } + case '<': + tok, size, isTag, err := parseTag(rs, i) + if err != nil { + return nil, err + } + if !isTag { + // tag-shaped but not a tag this version parses: literal text, + // never an error (§10) — recorded so a caller can say so, + // because canonical output would have escaped these bytes + if name, shaped := tagShapedName(rs[i:]); shaped { + notes.unknownTag(name) + } + appendText("<") + i++ + break + } + if tok != nil { + emit(*tok) + } + i += size + case '[': + tok, size, ok, err := parseLink(rs, i, ctx, depth, notes) + if err != nil { + return nil, err + } + if ok { + emit(*tok) + i += size + } else { + appendText("[") + i++ + } + case '*', '_': + n := runLen(rs, i, r) + canOpen, canClose := delimFlanking(rs, i, n, r) + if canOpen || canClose { + emit(token{kind: tokDelim, ch: r, n: n, canOpen: canOpen, canClose: canClose}) + } else { + appendText(strings.Repeat(string(r), n)) + } + i += n + case '~': + n := runLen(rs, i, '~') + if n == 2 { + canOpen, canClose := delimFlanking(rs, i, n, '~') + if canOpen || canClose { + emit(token{kind: tokDelim, ch: '~', n: n, canOpen: canOpen, canClose: canClose}) + } else { + appendText("~~") + } + } else { + appendText(strings.Repeat("~", n)) + } + i += n + default: + // batch the plain run up to the next special character + j := i + 1 + for j < len(rs) && !isInlineSpecial(rs[j]) { + j++ + } + appendText(string(rs[i:j])) + i = j + } + } + flushText() + return toks, nil +} + +func isInlineSpecial(r rune) bool { + switch r { + case '\\', '&', '`', '<', '[', '*', '_', '~': + return true + } + return false +} + +func runLen(rs []rune, i int, ch rune) int { + n := 0 + for i+n < len(rs) && rs[i+n] == ch { + n++ + } + return n +} + +// delimFlanking computes flanking-lite open/close capability for a delimiter +// run: '*'/'~' need a non-space neighbor on the inside; '_' additionally must +// not sit inside a word (intraword underscores stay literal). +func delimFlanking(rs []rune, i, n int, ch rune) (canOpen, canClose bool) { + prev, next := edgeWS, edgeWS + if i > 0 { + prev = classifyRune(rs[i-1]) + } + if i+n < len(rs) { + next = classifyRune(rs[i+n]) + } + canOpen = next != edgeWS + canClose = prev != edgeWS + if ch == '_' { + canOpen = canOpen && prev != edgeWord + canClose = canClose && next != edgeWord + } + return canOpen, canClose +} + +func stripCodePadding(content string) string { + if len(content) < 2 { + return content + } + first, last := content[0], content[len(content)-1] + startsWS := first == ' ' || first == '\n' + endsWS := last == ' ' || last == '\n' + if startsWS && endsWS && strings.Trim(content, " \n") != "" { + return content[1 : len(content)-1] + } + return content +} + +var namedEntities = map[string]string{ + "lt": "<", "gt": ">", "amp": "&", "quot": `"`, "apos": "'", "nbsp": " ", +} + +// parseEntity decodes an HTML entity at the start of rs, returning the +// decoded text and consumed length. +func parseEntity(rs []rune) (string, int, bool) { + if len(rs) < 3 || rs[0] != '&' { + return "", 0, false + } + if rs[1] == '#' { + j := 2 + hex := false + if j < len(rs) && (rs[j] == 'x' || rs[j] == 'X') { + hex = true + j++ + } + start := j + var v int64 + for j < len(rs) && j-start < 7 { + d := digitVal(rs[j], hex) + if d < 0 { + break + } + v = v*int64(base(hex)) + int64(d) + j++ + } + if j == start || j >= len(rs) || rs[j] != ';' { + return "", 0, false + } + if v == 0 || v > 0x10FFFF || (v >= 0xD800 && v <= 0xDFFF) { + return "", 0, false + } + return string(rune(v)), j + 1, true + } + j := 1 + for j < len(rs) && j < 12 && isASCIILetter(rs[j]) { + j++ + } + if j >= len(rs) || rs[j] != ';' { + return "", 0, false + } + dec, ok := namedEntities[string(rs[1:j])] + if !ok { + return "", 0, false + } + return dec, j + 1, true +} + +func digitVal(r rune, hex bool) int { + switch { + case r >= '0' && r <= '9': + return int(r - '0') + case hex && r >= 'a' && r <= 'f': + return int(r-'a') + 10 + case hex && r >= 'A' && r <= 'F': + return int(r-'A') + 10 + } + return -1 +} + +func base(hex bool) int { + if hex { + return 16 + } + return 10 +} + +func decodeEntities(s string) string { + if !strings.ContainsRune(s, '&') { + return s + } + rs := []rune(s) + var b strings.Builder + for i := 0; i < len(rs); { + if rs[i] == '&' { + if dec, size, ok := parseEntity(rs[i:]); ok { + b.WriteString(dec) + i += size + continue + } + } + b.WriteRune(rs[i]) + i++ + } + return b.String() +} + +// parseTag parses a whitelisted inline tag at rs[i]. Returns isTag=false when +// the '<' does not start a whitelisted tag name (the '<' is then literal); +// once a whitelisted name is recognized, malformed syntax is an error (§12). +// A self-closing tag is zero-length and returns a nil token (dropped, §8.1). +func parseTag(rs []rune, i int) (*token, int, bool, error) { + j := i + 1 + closing := false + if j < len(rs) && rs[j] == '/' { + closing = true + j++ + } + nameStart := j + for j < len(rs) && isASCIILetter(rs[j]) { + j++ + } + name := string(rs[nameStart:j]) + if name != "u" && name != "font" && name != "mention" { + return nil, 0, false, nil + } + if j >= len(rs) || (rs[j] != '>' && rs[j] != '/' && !unicode.IsSpace(rs[j])) { + return nil, 0, false, nil + } + attrs := map[string]string{} + selfClose := false + for { + for j < len(rs) && unicode.IsSpace(rs[j]) { + j++ + } + if j >= len(rs) { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("unterminated <%s> tag", name)) + } + if rs[j] == '/' { + j++ + if j >= len(rs) || rs[j] != '>' { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("malformed <%s> tag", name)) + } + selfClose = true + j++ + break + } + if rs[j] == '>' { + j++ + break + } + if closing { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("closing tag with attributes", name)) + } + attrStart := j + // attribute names are snake_case like every other identifier the + // format defines (§8.1), so '_' is part of the name, not a terminator + for j < len(rs) && (isASCIILetter(rs[j]) || rs[j] == '_') { + j++ + } + attrName := string(rs[attrStart:j]) + if attrName == "" { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("malformed <%s> tag", name)) + } + for j < len(rs) && unicode.IsSpace(rs[j]) { + j++ + } + if j >= len(rs) || rs[j] != '=' { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("attribute %q in <%s> tag needs a quoted value", attrName, name)) + } + j++ + for j < len(rs) && unicode.IsSpace(rs[j]) { + j++ + } + if j >= len(rs) || (rs[j] != '"' && rs[j] != '\'') { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("attribute %q in <%s> tag needs a quoted value", attrName, name)) + } + quote := rs[j] + j++ + valStart := j + for j < len(rs) && rs[j] != quote { + j++ + } + if j >= len(rs) { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("unterminated attribute value in <%s> tag", name)) + } + if _, dup := attrs[attrName]; dup { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("duplicate attribute %q in <%s> tag", attrName, name)) + } + attrs[attrName] = decodeEntities(string(rs[valStart:j])) + j++ + } + if closing && selfClose { + return nil, 0, false, inlineErr(rs, i, fmt.Sprintf("malformed tag", name)) + } + if !closing { + if err := validateTagAttrs(name, attrs); err != "" { + return nil, 0, false, inlineErr(rs, i, err) + } + } + if selfClose { + // zero-length tag: dropped + return nil, j - i, true, nil + } + return &token{kind: tokTag, tagName: name, closing: closing, attrs: attrs}, j - i, true, nil +} + +// validateTagAttrs enforces the per-tag attribute rules (§8.1); returns an +// error message or "". +func validateTagAttrs(name string, attrs map[string]string) string { + switch name { + case "u": + if len(attrs) > 0 { + return "unexpected attribute on tag" + } + case "font": + for k := range attrs { + if k != "color" && k != "background" { + return fmt.Sprintf("unknown attribute %q on tag", k) + } + } + if len(attrs) == 0 { + return " tag needs a color or background attribute" + } + case "mention": + for k := range attrs { + if k != "object_id" { + return fmt.Sprintf("unknown attribute %q on tag", k) + } + } + if _, ok := attrs["object_id"]; !ok { + return " tag needs an object_id attribute" + } + } + return "" +} + +// scanLink matches the [label](dest) pattern at rs[i] without tokenizing the +// label, using the precomputed bracket table. +func scanLink(rs []rune, i int, ctx *inlineScanCtx) (labelEnd int, dest string, size int, ok bool) { + labelEnd, ok = ctx.brackets[i] + if !ok { + return 0, "", 0, false + } + k := labelEnd + 1 + if k >= len(rs) || rs[k] != '(' { + return 0, "", 0, false + } + k++ + ws := 0 + for k < len(rs) && unicode.IsSpace(rs[k]) { + k++ + ws++ + if ws > maxLinkDestWS { + return 0, "", 0, false + } + } + if k >= len(rs) { + return 0, "", 0, false + } + destLimit := k + maxLinkDestLen + var destOk bool + if rs[k] == '<' { + dest, k, destOk = scanAngleDest(rs, k+1, destLimit) + } else { + dest, k, destOk = scanBareDest(rs, k, destLimit) + } + if !destOk { + return 0, "", 0, false + } + ws = 0 + for k < len(rs) && unicode.IsSpace(rs[k]) { + k++ + ws++ + if ws > maxLinkDestWS { + return 0, "", 0, false + } + } + if k >= len(rs) || rs[k] != ')' { + return 0, "", 0, false + } + return labelEnd, dest, k + 1 - i, true +} + +// scanAngleDest reads an angle-wrapped destination up to the closing '>', +// decoding escapes and entities; limit bounds the scan (§8 resource bounds). +func scanAngleDest(rs []rune, k, limit int) (string, int, bool) { + var db strings.Builder + for { + if k >= len(rs) || k > limit { + return "", 0, false + } + c := rs[k] + if c == '>' { + return db.String(), k + 1, true + } + if c == '\\' && k+1 < len(rs) && isASCIIPunct(rs[k+1]) { + db.WriteRune(rs[k+1]) + k += 2 + continue + } + if c == '&' { + if dec, esize, eok := parseEntity(rs[k:]); eok { + db.WriteString(dec) + k += esize + continue + } + } + db.WriteRune(c) + k++ + } +} + +// scanBareDest reads a bare destination up to whitespace or the closing +// unbalanced ')', decoding escapes and entities; limit bounds the scan. +func scanBareDest(rs []rune, k, limit int) (string, int, bool) { + var db strings.Builder + parens := 0 + for { + if k >= len(rs) || k > limit { + return "", 0, false + } + c := rs[k] + if unicode.IsSpace(c) { + return db.String(), k, true + } + switch c { + case ')': + if parens == 0 { + return db.String(), k, true + } + parens-- + case '(': + parens++ + case '\\': + if k+1 < len(rs) && isASCIIPunct(rs[k+1]) { + db.WriteRune(rs[k+1]) + k += 2 + continue + } + case '&': + if dec, esize, eok := parseEntity(rs[k:]); eok { + db.WriteString(dec) + k += esize + continue + } + } + db.WriteRune(c) + k++ + } +} + +func parseLink(rs []rune, i int, ctx *inlineScanCtx, depth int, notes *inlineNotes) (*token, int, bool, error) { + if depth >= maxLinkNesting { + return nil, 0, false, nil + } + labelEnd, dest, size, ok := scanLink(rs, i, ctx) + if !ok { + return nil, 0, false, nil + } + labelToks, err := tokenizeInline(rs[i+1:labelEnd], depth+1, notes) + if err != nil { + return nil, 0, false, err + } + return &token{kind: tokLink, label: labelToks, dest: dest}, size, true, nil +} + +// +// ---- resolution ---- +// + +type entryKind int + +const ( + entryEmph entryKind = iota + entryTag +) + +type openEntry struct { + kind entryKind + ch rune + width int + markType model.BlockContentTextMarkType + tagName string + attrs map[string]string + start16 int +} + +func (e *openEntry) rawDelim() string { + return strings.Repeat(string(e.ch), e.width) +} + +type inlineBuilder struct { + out []uint16 + marks []*model.BlockContentTextMark + inserts []pendingInsert +} + +type pendingInsert struct { + pos int // in pre-insertion coordinates + seq int + s string +} + +func (ib *inlineBuilder) appendString(s string) { + ib.out = append(ib.out, text.StrToUTF16(s)...) +} + +// insertLiteral records literal text to splice at pos (an unmatched opening +// delimiter demoted back to text). Splices are deferred and applied in one +// batch by applyInserts — per-demotion slice rebuilds would be quadratic. +// All recorded positions (marks, open entries, inserts) share the +// pre-insertion coordinate space, so deferral is exact. +func (ib *inlineBuilder) insertLiteral(pos int, s string) { + ib.inserts = append(ib.inserts, pendingInsert{pos: pos, seq: len(ib.inserts), s: s}) +} + +// applyInserts splices all pending literals in one pass and shifts mark +// offsets. At equal positions, later-recorded inserts land first — a later +// insert at pos pushes earlier-inserted text right, matching the sequential +// semantics the deferral replaces. +func (ib *inlineBuilder) applyInserts() { + if len(ib.inserts) == 0 { + return + } + ins := ib.inserts + sort.SliceStable(ins, func(i, j int) bool { + if ins[i].pos != ins[j].pos { + return ins[i].pos < ins[j].pos + } + return ins[i].seq > ins[j].seq + }) + encoded := make([][]uint16, len(ins)) + total := 0 + for i := range ins { + encoded[i] = text.StrToUTF16(ins[i].s) + total += len(encoded[i]) + } + out := make([]uint16, 0, len(ib.out)+total) + prev := 0 + for i := range ins { + out = append(out, ib.out[prev:ins[i].pos]...) + out = append(out, encoded[i]...) + prev = ins[i].pos + } + out = append(out, ib.out[prev:]...) + ib.out = out + + // prefix sums over insert lengths for O(log n) shift lookups + positions := make([]int, len(ins)) + cum := make([]int, len(ins)+1) + for i := range ins { + positions[i] = ins[i].pos + cum[i+1] = cum[i] + len(encoded[i]) + } + shift := func(p int, inclusive bool) int32 { + // sum of insert lengths with pos < p (or <= p when inclusive) + idx := sort.SearchInts(positions, p) + if inclusive { + for idx < len(positions) && positions[idx] == p { + idx++ + } + } + return int32(cum[idx]) //nolint:gosec // UTF-16 offsets are bounded by the text length + } + for _, m := range ib.marks { + from, to := int(m.Range.From), int(m.Range.To) + m.Range.From += shift(from, true) + m.Range.To += shift(to, false) + } + ib.inserts = nil +} + +func (ib *inlineBuilder) addMark(typ model.BlockContentTextMarkType, param string, from, to int) { + if from >= to { + return + } + if markNeedsParam(typ) && param == "" { + return + } + ib.marks = append(ib.marks, &model.BlockContentTextMark{ + Range: &model.Range{From: int32(from), To: int32(to)}, //nolint:gosec // UTF-16 offsets are bounded by the text length + Type: typ, + Param: param, + }) +} + +// resolveTokens turns a token stream into text and marks with a delimiter +// stack. Unmatched emphasis delimiters demote to literal text; unmatched or +// misnested whitelisted tags are errors. +func resolveTokens(toks []token, ib *inlineBuilder) error { + var stack []openEntry + for ti := range toks { + t := &toks[ti] + switch t.kind { + case tokText: + ib.appendString(t.txt) + case tokCode: + start := len(ib.out) + ib.appendString(t.content) + ib.addMark(model.BlockContentTextMark_Keyboard, "", start, len(ib.out)) + case tokDelim: + resolveDelimRun(&stack, ib, t) + case tokTag: + if !t.closing { + stack = append(stack, openEntry{kind: entryTag, tagName: t.tagName, attrs: t.attrs, start16: len(ib.out)}) + break + } + idx := -1 + for i := len(stack) - 1; i >= 0; i-- { + if stack[i].kind == entryTag { + if stack[i].tagName != t.tagName { + return &inlineError{Msg: fmt.Sprintf("misnested tags: closes across <%s>", t.tagName, stack[i].tagName)} + } + idx = i + break + } + } + if idx < 0 { + return &inlineError{Msg: fmt.Sprintf("unmatched closing tag", t.tagName)} + } + for i := len(stack) - 1; i > idx; i-- { + ib.insertLiteral(stack[i].start16, stack[i].rawDelim()) + } + e := stack[idx] + stack = stack[:idx] + emitTagMarks(ib, e) + case tokLink: + start := len(ib.out) + if err := resolveTokens(t.label, ib); err != nil { + return err + } + if id, ok := parseObjectLink(t.dest); ok { + ib.addMark(model.BlockContentTextMark_Object, id, start, len(ib.out)) + } else { + ib.addMark(model.BlockContentTextMark_Link, t.dest, start, len(ib.out)) + } + } + } + for i := len(stack) - 1; i >= 0; i-- { + e := stack[i] + if e.kind == entryTag { + return &inlineError{Msg: fmt.Sprintf("unclosed <%s> tag", e.tagName)} + } + ib.insertLiteral(e.start16, e.rawDelim()) + } + return nil +} + +func emitTagMarks(ib *inlineBuilder, e openEntry) { + end := len(ib.out) + switch e.tagName { + case "u": + ib.addMark(model.BlockContentTextMark_Underscored, "", e.start16, end) + case "font": + ib.addMark(model.BlockContentTextMark_TextColor, e.attrs["color"], e.start16, end) + ib.addMark(model.BlockContentTextMark_BackgroundColor, e.attrs["background"], e.start16, end) + case "mention": + ib.addMark(model.BlockContentTextMark_Mention, e.attrs["object_id"], e.start16, end) + } +} + +// resolveDelimRun consumes an emphasis/strikethrough delimiter run against +// the stack: close the top entry while it matches, open with the remainder, +// demote what can do neither to literal text. +func resolveDelimRun(stack *[]openEntry, ib *inlineBuilder, t *token) { + n := t.n + for n > 0 { + if t.canClose && len(*stack) > 0 { + top := &(*stack)[len(*stack)-1] + if top.kind == entryEmph && top.ch == t.ch && top.width <= n { + ib.addMark(top.markType, "", top.start16, len(ib.out)) + n -= top.width + *stack = (*stack)[:len(*stack)-1] + continue + } + } + if t.canOpen { + e := openEntry{kind: entryEmph, ch: t.ch, start16: len(ib.out)} + switch { + case t.ch == '~': + e.width, e.markType = 2, model.BlockContentTextMark_Strikethrough + case n >= 2: + e.width, e.markType = 2, model.BlockContentTextMark_Bold + default: + e.width, e.markType = 1, model.BlockContentTextMark_Italic + } + *stack = append(*stack, e) + n -= e.width + continue + } + ib.appendString(strings.Repeat(string(t.ch), n)) + n = 0 + } +} + +// canonicalizeMarks merges same-type same-param overlapping/adjacent ranges +// and sorts marks into the canonical order (from asc, to desc, nesting +// priority, param). +func canonicalizeMarks(marks []*model.BlockContentTextMark) []*model.BlockContentTextMark { + if len(marks) == 0 { + return nil + } + type key struct { + typ model.BlockContentTextMarkType //nolint:unused // map-key equality is the use + param string //nolint:unused // map-key equality is the use + } + groups := make(map[key][]*model.BlockContentTextMark) + var order []key + for _, m := range marks { + k := key{m.Type, m.Param} + if _, seen := groups[k]; !seen { + order = append(order, k) + } + groups[k] = append(groups[k], m) + } + var out []*model.BlockContentTextMark + for _, k := range order { + g := groups[k] + sort.SliceStable(g, func(i, j int) bool { return g[i].Range.From < g[j].Range.From }) + merged := []*model.BlockContentTextMark{g[0]} + for _, m := range g[1:] { + last := merged[len(merged)-1] + if m.Range.From <= last.Range.To { + if m.Range.To > last.Range.To { + last.Range.To = m.Range.To + } + } else { + merged = append(merged, m) + } + } + out = append(out, merged...) + } + sort.SliceStable(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.Range.From != b.Range.From { + return a.Range.From < b.Range.From + } + if a.Range.To != b.Range.To { + return a.Range.To > b.Range.To + } + if markPriority[a.Type] != markPriority[b.Type] { + return markPriority[a.Type] < markPriority[b.Type] + } + return a.Param < b.Param + }) + return out +} + +func isASCIILetter(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') +} + +func isASCIIPunct(r rune) bool { + return strings.ContainsRune("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~", r) +} + +func isPunctRune(r rune) bool { + return isASCIIPunct(r) || unicode.IsPunct(r) || unicode.IsSymbol(r) +} diff --git a/pkg/lib/anyblockjson/inline_test.go b/pkg/lib/anyblockjson/inline_test.go new file mode 100644 index 0000000000..4fba5239b0 --- /dev/null +++ b/pkg/lib/anyblockjson/inline_test.go @@ -0,0 +1,382 @@ +package anyblockjson + +import ( + "math/rand" + "strings" + "testing" + "unicode/utf16" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func mark(t model.BlockContentTextMarkType, from, to int32, param string) *model.BlockContentTextMark { + return &model.BlockContentTextMark{Range: &model.Range{From: from, To: to}, Type: t, Param: param} +} + +const ( + mBold = model.BlockContentTextMark_Bold + mItalic = model.BlockContentTextMark_Italic + mStrike = model.BlockContentTextMark_Strikethrough + mCode = model.BlockContentTextMark_Keyboard + mLink = model.BlockContentTextMark_Link + mObject = model.BlockContentTextMark_Object + mMention = model.BlockContentTextMark_Mention + mUnder = model.BlockContentTextMark_Underscored + mColor = model.BlockContentTextMark_TextColor + mBg = model.BlockContentTextMark_BackgroundColor + mEmoji = model.BlockContentTextMark_Emoji +) + +// TestRenderInline_Golden checks canonical rendering and that every canonical +// form is byte-stable through parse ∘ render (§11.2). +func TestRenderInline_Golden(t *testing.T) { + tests := []struct { + name string + text string + marks []*model.BlockContentTextMark + want string + }{ + {"plain", "hello world", nil, "hello world"}, + {"bold", "hello world", []*model.BlockContentTextMark{mark(mBold, 0, 5, "")}, "**hello** world"}, + {"italic", "hello world", []*model.BlockContentTextMark{mark(mItalic, 6, 11, "")}, "hello *world*"}, + {"strike", "done", []*model.BlockContentTextMark{mark(mStrike, 0, 4, "")}, "~~done~~"}, + {"code", "code", []*model.BlockContentTextMark{mark(mCode, 0, 4, "")}, "`code`"}, + {"bold italic same range", "x", []*model.BlockContentTextMark{mark(mBold, 0, 1, ""), mark(mItalic, 0, 1, "")}, "***x***"}, + {"bold italic overlap", "abc", []*model.BlockContentTextMark{mark(mBold, 0, 2, ""), mark(mItalic, 1, 3, "")}, "**a*b****c*"}, + {"link", "docs here", []*model.BlockContentTextMark{mark(mLink, 0, 4, "https://x.io")}, "[docs](https://x.io) here"}, + {"object link", "docs here", []*model.BlockContentTextMark{mark(mObject, 0, 4, "bafy1")}, "[docs](anytype://object?objectId=bafy1) here"}, + {"mention", "ping Roman", []*model.BlockContentTextMark{mark(mMention, 5, 10, "bafyid")}, `ping Roman`}, + {"underline", "docs", []*model.BlockContentTextMark{mark(mUnder, 0, 4, "")}, "docs"}, + {"text color", "x", []*model.BlockContentTextMark{mark(mColor, 0, 1, "red")}, `x`}, + {"background", "x", []*model.BlockContentTextMark{mark(mBg, 0, 1, "yellow")}, `x`}, + {"coincident color and background", "x", + []*model.BlockContentTextMark{mark(mColor, 0, 1, "red"), mark(mBg, 0, 1, "yellow")}, + `x`}, + {"nested color and background", "abc", + []*model.BlockContentTextMark{mark(mColor, 0, 3, "red"), mark(mBg, 1, 2, "yellow")}, + `abc`}, + {"whitespace boundary shrink", " hi ", []*model.BlockContentTextMark{mark(mBold, 0, 4, "")}, " **hi** "}, + {"all whitespace mark dropped", "a b", []*model.BlockContentTextMark{mark(mBold, 1, 2, "")}, "a b"}, + {"same-type overlap truncated", "abcd", + []*model.BlockContentTextMark{mark(mLink, 0, 2, "u1"), mark(mLink, 1, 4, "u2")}, + "[ab](u1)[cd](u2)"}, + {"emoji materialized", "abc", []*model.BlockContentTextMark{mark(mEmoji, 1, 2, "😀")}, "a😀c"}, + {"emoji under bold", "abc", + []*model.BlockContentTextMark{mark(mBold, 0, 3, ""), mark(mEmoji, 1, 2, "😀")}, + "**a😀c**"}, + {"escape star", "2*3 = 6", nil, `2\*3 = 6`}, + {"star with spaces literal", "a * b", nil, "a * b"}, + {"escape backtick", "a`b", nil, "a\\`b"}, + {"escape tilde run", "~~x~~", nil, `\~\~x\~\~`}, + {"single tilde literal", "~x", nil, "~x"}, + {"escape bracket", "[note]", nil, `\[note]`}, + {"escape whitelisted tag", "", nil, `\`}, + // escaped on shape, not on the whitelist: reserved syntax space (§8.2) + {"escape unknown tag", "
x", nil, `\
x`}, + {"escape entity", "<", nil, `\<`}, + {"bare ampersand literal", "R&D", nil, "R&D"}, + {"escape underscore", "_x_", nil, `\_x\_`}, + {"intraword underscore literal", "snake_case", nil, "snake_case"}, + {"backslash before punct", `a\*b`, nil, `a\\\*b`}, + {"backslash before letter", `a\b`, nil, `a\b`}, + {"code span with backtick", "a`b", []*model.BlockContentTextMark{mark(mCode, 0, 3, "")}, "``a`b``"}, + {"code span starts with backtick", "`x", []*model.BlockContentTextMark{mark(mCode, 0, 2, "")}, "`` `x ``"}, + {"bold inside link label", "click here", + []*model.BlockContentTextMark{mark(mLink, 0, 10, "https://x.io"), mark(mBold, 6, 10, "")}, + "[click **here**](https://x.io)"}, + {"parens in url", "x", []*model.BlockContentTextMark{mark(mLink, 0, 1, "http://x/a(1)")}, `[x](http://x/a\(1\))`}, + {"space in url", "x", []*model.BlockContentTextMark{mark(mLink, 0, 1, "a b")}, "[x]()"}, + {"soft line break in bold", "a\nb", []*model.BlockContentTextMark{mark(mBold, 0, 3, "")}, "**a\nb**"}, + {"astral bold", "𝒜b", []*model.BlockContentTextMark{mark(mBold, 0, 2, "")}, "**𝒜**b"}, + {"escaped chars inside mention", "see *x*", + []*model.BlockContentTextMark{mark(mMention, 4, 7, "id1")}, + `see \*x\*`}, + {"bracket inside link label", "a[b]c", + []*model.BlockContentTextMark{mark(mLink, 0, 5, "u")}, + `[a\[b\]c](u)`}, + {"code under bold overlap", "abc", + []*model.BlockContentTextMark{mark(mCode, 0, 3, ""), mark(mBold, 1, 2, "")}, + "`a`**`b`**`c`"}, + {"adjacent same type merges", "ab", + []*model.BlockContentTextMark{mark(mBold, 0, 1, ""), mark(mBold, 1, 2, "")}, + "**ab**"}, + {"invalid ranges dropped", "ab", + []*model.BlockContentTextMark{mark(mBold, 3, 5, ""), mark(mBold, 1, 1, ""), mark(mLink, 0, 1, "")}, + "ab"}, + {"zero length text", "", []*model.BlockContentTextMark{mark(mBold, 0, 0, "")}, ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := renderInline(tc.text, tc.marks) + assert.Equal(t, tc.want, got) + + // canonical form must be byte-stable: parse it and render again + txt, marks, err := parseInline(got) + require.NoError(t, err) + again := renderInline(txt, marks) + assert.Equal(t, got, again, "Export ∘ Import must be byte-stable") + }) + } +} + +// TestRenderInline_ReservesTagSyntaxSpace pins the syntax space the tag +// namespace reserves (§8.2, §10). Escaping only the three tags version 1 +// happens to know would leave literal `x` bytes in canonical +// output that a version that adds `sub` reads as markup — and no reader can +// tell 1-literal from 2-markup from the text string alone. So the escape is +// anchored on the *shape* `x", `\x\`}, + {"unknown closing tag", "

", `\

`}, + {"known tag", "x", `\x\`}, + {"tag-shaped with no terminator", "aRoman`, "Roman", + []*model.BlockContentTextMark{mark(mMention, 0, 5, "id1")}}, + {"mention single quotes attr", `R`, "R", + []*model.BlockContentTextMark{mark(mMention, 0, 1, "id1")}}, + {"font attr order and spaces", `x`, "x", + []*model.BlockContentTextMark{mark(mColor, 0, 1, "r"), mark(mBg, 0, 1, "y")}}, + {"underline", "x", "x", []*model.BlockContentTextMark{mark(mUnder, 0, 1, "")}}, + {"zero-length tag dropped", `ab`, "ab", nil}, + {"self-closing tag dropped", `ab`, "ab", nil}, + {"entities", "<u> & A", " & A", nil}, + {"escapes", `\*x\* \[y] \~\~`, "*x* [y] ~~", nil}, + {"unmatched bold literal", "**unclosed", "**unclosed", nil}, + {"unmatched code literal", "`unclosed", "`unclosed", nil}, + {"link with space in dest literal", "[a](b c)", "[a](b c)", nil}, + {"angle dest", "[a]()", "a", []*model.BlockContentTextMark{mark(mLink, 0, 1, "b c")}}, + {"adjacent italic merges", "*a**b*", "ab", []*model.BlockContentTextMark{mark(mItalic, 0, 2, "")}}, + {"nested em in strong", "**a *b* c**", "a b c", + []*model.BlockContentTextMark{mark(mBold, 0, 5, ""), mark(mItalic, 2, 3, "")}}, + {"stars with spaces literal", "a * b * c", "a * b * c", nil}, + {"empty link param dropped", "[a]()", "a", nil}, + {"tilde run of three literal", "~~~x~~~", "~~~x~~~", nil}, + {"utf16 offsets astral", "𝒜 **b**", "𝒜 b", []*model.BlockContentTextMark{mark(mBold, 3, 4, "")}}, + {"soft break", "a\nb", "a\nb", nil}, + {"emphasis across soft break", "**a\nb**", "a\nb", []*model.BlockContentTextMark{mark(mBold, 0, 3, "")}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + text, marks, err := parseInline(tc.md) + require.NoError(t, err) + assert.Equal(t, tc.wantText, text) + assert.Equal(t, tc.wantMarks, marks) + + // render(parse(J)) is the canonical form; parsing it again must + // reproduce the same state (idempotence, §11.2) + canonical := renderInline(text, marks) + text2, marks2, err := parseInline(canonical) + require.NoError(t, err) + assert.Equal(t, text, text2) + assert.Equal(t, marks, marks2) + assert.Equal(t, canonical, renderInline(text2, marks2)) + }) + } +} + +func TestParseInline_Errors(t *testing.T) { + tests := []struct { + name string + md string + }{ + {"unclosed u tag", "x"}, + {"unmatched closing tag", "x"}, + {"mention without object_id", "x"}, + {"font without attrs", "x"}, + {"unknown font attr", `x`}, + {"unquoted attr value", `x`}, + {"misnested tags", `ab`}, + {"duplicate attr", `x`}, + {"unterminated tag", "`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, _, err := parseInline(tc.md) + require.Error(t, err) + }) + } +} + +// TestInline_UnmatchedDelimiterLiteralization: unmatched openers demote to +// literal text with correct mark offset shifting. +func TestInline_UnmatchedDelimiterLiteralization(t *testing.T) { + // the '*' opener never closes; the code span mark must shift right + text, marks, err := parseInline("*`c`") + require.NoError(t, err) + assert.Equal(t, "*c", text) + assert.Equal(t, []*model.BlockContentTextMark{mark(mCode, 1, 2, "")}, marks) +} + +// TestInline_PropertyRoundTrip generates random states and checks the §11 +// guarantees: canonical output always parses, and Export ∘ Import is +// byte-stable from the first canonical form on. +func TestInline_PropertyRoundTrip(t *testing.T) { + alphabet := []string{ + "a", "b", "c", " ", "\n", "*", "_", "~", "`", "[", "]", "(", ")", "<", ">", + "&", "\\", "\"", "'", "😀", "𝒜", "é", "u", "font", "mention", "lt;", "#x", + } + types := []model.BlockContentTextMarkType{ + mBold, mItalic, mStrike, mCode, mLink, mObject, mMention, mUnder, mColor, mBg, mEmoji, + } + params := []string{"", "red", "https://x.io/a(b)", "id1", "a b", "😀", "x\"y"} + rnd := rand.New(rand.NewSource(42)) + for i := 0; i < 20000; i++ { + var sb strings.Builder + for n := rnd.Intn(12); n > 0; n-- { + sb.WriteString(alphabet[rnd.Intn(len(alphabet))]) + } + txt := sb.String() + u16len := int32(len(utf16.Encode([]rune(txt)))) + var marks []*model.BlockContentTextMark + for n := rnd.Intn(6); n > 0; n-- { + from := rnd.Int31n(u16len + 1) + to := rnd.Int31n(u16len + 1) + marks = append(marks, mark(types[rnd.Intn(len(types))], from, to, params[rnd.Intn(len(params))])) + } + md1 := renderInline(txt, marks) + text1, marks1, err := parseInline(md1) + require.NoErrorf(t, err, "case %d: canonical output must parse: text=%q marks=%v md=%q", i, txt, marks, md1) + md2 := renderInline(text1, marks1) + require.Equalf(t, md1, md2, "case %d: not byte-stable: text=%q marks=%v", i, txt, marks) + text2, marks2, err := parseInline(md2) + require.NoError(t, err) + require.Equalf(t, text1, text2, "case %d", i) + require.Equalf(t, marks1, marks2, "case %d: marks not stable: md=%q", i, md1) + } +} + +// An Object mark's target is Anytype's deep link, and the form is exact: a +// single "object_id" parameter, nothing else (§8.1). Matching it by prefix +// took everything after "object_id=" as the id, so the platform's own +// two-parameter link (core/block/export/writer.go) produced the object id +// "&spaceId=" — byte-stable, and wrong forever. +func TestInline_ObjectDeepLinkStrictParse(t *testing.T) { + tests := []struct { + name string + dest string + wantType model.BlockContentTextMarkType + wantID string // for an Object mark; for a Link, the dest verbatim + }{ + {"canonical single parameter", "anytype://object?objectId=bafy1", mObject, "bafy1"}, + {"percent-encoded id decodes", "anytype://object?objectId=a%26b", mObject, "a&b"}, + + // every one of these was, or would be, mis-parsed as an object id + {"platform two-parameter link", "anytype://object?objectId=bafy1&spaceId=s1", mLink, "anytype://object?objectId=bafy1&spaceId=s1"}, + {"parameters reversed", "anytype://object?spaceId=s1&object_id=bafy1", mLink, "anytype://object?spaceId=s1&object_id=bafy1"}, + {"an extra parameter", "anytype://object?objectId=bafy1&mention=1", mLink, "anytype://object?objectId=bafy1&mention=1"}, + {"repeated parameter", "anytype://object?objectId=a&object_id=b", mLink, "anytype://object?objectId=a&object_id=b"}, + {"empty id", "anytype://object?objectId=", mLink, "anytype://object?objectId="}, + {"another host", "anytype://date?timestamp=123", mLink, "anytype://date?timestamp=123"}, + {"a path", "anytype://invite/?cid=x&key=y", mLink, "anytype://invite/?cid=x&key=y"}, + {"another scheme", "https://object?object_id=bafy1", mLink, "https://object?object_id=bafy1"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // given + md := "[Roman](" + tc.dest + ")" + + // when + txt, marks, err := parseInline(md) + + // then + require.NoError(t, err) + assert.Equal(t, "Roman", txt) + require.Len(t, marks, 1) + assert.Equal(t, tc.wantType, marks[0].Type) + assert.Equal(t, tc.wantID, marks[0].Param, + "a destination this reader does not recognize must survive verbatim, not be reinterpreted") + + // and the round trip is byte-stable, so nothing degrades on re-export + md2 := renderInline(txt, marks) + txt3, marks3, err := parseInline(md2) + require.NoError(t, err) + assert.Equal(t, txt, txt3) + assert.Equal(t, marks, marks3) + }) + } +} + +// An object id is percent-encoded on the way out, so it cannot introduce a +// second query parameter — otherwise an id containing "&spaceId=" would +// render a link that other tools resolve to a different space. +func TestInline_ObjectDeepLinkEncodesId(t *testing.T) { + tests := []struct { + name string + id string + wantDest string + }{ + {"an ordinary CID is unchanged", "bafyreiabc123", "anytype://object?objectId=bafyreiabc123"}, + {"an id smuggling a parameter", "a&spaceId=evil", "anytype://object?objectId=a%26spaceId%3Devil"}, + {"an id with a query terminator", "a?b#c", "anytype://object?objectId=a%3Fb%23c"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // given + marks := []*model.BlockContentTextMark{mark(mObject, 0, 1, tc.id)} + + // when + md := renderInline("x", marks) + + // then + assert.Contains(t, md, tc.wantDest) + + // and the id survives the round trip exactly + _, got, err := parseInline(md) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, mObject, got[0].Type) + assert.Equal(t, tc.id, got[0].Param) + }) + } +} diff --git a/pkg/lib/anyblockjson/internalkey_test.go b/pkg/lib/anyblockjson/internalkey_test.go new file mode 100644 index 0000000000..3fd702b786 --- /dev/null +++ b/pkg/lib/anyblockjson/internalkey_test.go @@ -0,0 +1,182 @@ +package anyblockjson + +// internalkey_test.go pins the key/spelling split (§2, §2e): `internal_key` +// is the ONLY thing called a key that is a stored id, `property` is the +// document-facing spelling, and a property definition may be identified by +// either (or by a `name` the spelling derives from). The split exists +// because one word carried both meanings — the envelope held stored bson +// ids while the same member name held slugs one level down — which is the +// §15 #14 disease measured over a 77-space corpus. + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// An entry identified by `internal_key` alone validates and imports, and the +// key is taken VERBATIM: a stored id is its own address (§3), so it never +// re-enters the slug ladder — not for a bundled key, not for a minted bson, +// and not even for a stored key that happens to look like a bundled slug. +// +// The last case is the one that pins the rule. `due_date` is the bundled +// table's slug for `dueDate`, and a term that reached the §3 ladder would be +// folded onto that bundled twin — which is exactly wrong for a member whose +// whole meaning is "this exact stored key": a space really can hold a shadow +// relation stored as `due_date` beside bundled `dueDate` (§3's identity-entry +// case), and `internal_key` is how a document addresses it with no legend. +func TestPropertyDefinitions_InternalKeyAloneIdentifiesVerbatim(t *testing.T) { + for name, tc := range map[string]struct { + internalKey string + format string + }{ + "a bundled stored key": {"dueDate", "date"}, + "a minted bson id": {"6a83296f61fab2265263ae34", "number"}, + "a shadow key shaped as a slug": {"due_date", "date"}, + } { + t.Run(name, func(t *testing.T) { + // given + doc := `{"version": 2, "kind": "object_type", "internal_key": "t", + "type_settings": {"property_definitions": [ + {"internal_key": "` + tc.internalKey + `", "format": "` + tc.format + `", "section": "featured"}]}}` + + // when + require.NoError(t, Validate([]byte(doc))) + _, snapshot, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("id")}) + + // then: without a resolver the resolved KEY passes through in + // place of an id, which is what makes the resolution observable + require.NoError(t, err) + assert.Equal(t, strList(tc.internalKey), + snapshot.Details.Fields["recommendedFeaturedRelations"], + "the stored key travels verbatim — no ladder, no fold, no rebinding") + }) + } +} + +// When an entry states both members, `property` outranks `internal_key`. On +// export's own output the two agree — both are written from one stored key — +// so the order only decides a DISAGREEING authored pair, and there the +// spelling wins because it is the member the document's own legend speaks +// for (§3 chain step 1). authoredKey is the one place the order lives. +func TestPropertyDefinitions_PropertyOutranksInternalKey(t *testing.T) { + // given a pair that disagrees on purpose + doc := `{"version": 2, "kind": "object_type", "internal_key": "t", + "property_internal_keys": {"budget": "6a83296f61fab2265263ae34"}, + "type_settings": {"property_definitions": [ + {"property": "budget", "internal_key": "somethingElse", "format": "number", "section": "featured"}]}}` + + // when + _, snapshot, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("id")}) + + // then: the spelling resolved through the legend, the internal_key lost + require.NoError(t, err) + assert.Equal(t, strList("6a83296f61fab2265263ae34"), + snapshot.Details.Fields["recommendedFeaturedRelations"]) +} + +// The identity anyOf admits each of the three members alone and refuses an +// entry with none — in BOTH homes of the shape (§2e). +func TestPropertyDefinitions_IdentityIsPropertyOrInternalKeyOrName(t *testing.T) { + entryDoc := func(entry string) string { + return `{"version": 2, "kind": "object_type", "internal_key": "t", + "type_settings": {"property_definitions": [` + entry + `]}}` + } + t.Run("each identity member alone is enough", func(t *testing.T) { + for _, entry := range []string{ + `{"property": "budget", "format": "number"}`, + `{"internal_key": "6a83296f61fab2265263ae34", "format": "number"}`, + `{"name": "Budget", "format": "number"}`, + } { + assert.NoError(t, Validate([]byte(entryDoc(entry))), entry) + } + }) + t.Run("an entry with no identity at all is refused", func(t *testing.T) { + err := Validate([]byte(entryDoc(`{"format": "number"}`))) + require.Error(t, err) + }) + t.Run("the dictionary home says the same", func(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte( + `{"version":2,"properties":[{"internal_key":"6a83296f61fab2265263ae34","format":"number"}]}`)) + assert.NoError(t, err) + _, err = UnmarshalPropertyDictionary([]byte( + `{"version":2,"properties":[{"format":"number"}]}`)) + require.Error(t, err) + }) +} + +// A dictionary entry's `internal_key` is verbatim too — same rule, third +// home: the fold ladder that recovers a stored key from a `property` +// spelling must not touch a member that IS the stored key. +func TestPropertyDictionary_InternalKeyIsVerbatim(t *testing.T) { + d, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"properties":[ + {"internal_key":"due_date","format":"date"}, + {"internal_key":"6a83296f61fab2265263ae34","name":"Budget","format":"number"}]}`)) + require.NoError(t, err) + require.Len(t, d.Properties, 2) + assert.Equal(t, "due_date", string(d.Properties[0].Key), + "a slug-shaped stored key stays itself — the fold onto bundled dueDate is for spellings") + assert.Equal(t, "6a83296f61fab2265263ae34", string(d.Properties[1].Key)) +} + +// Export writes BOTH members on a dictionary entry: `property` in the +// spelling every document uses, `internal_key` the stored key verbatim — +// fidelity an author never has to produce (§2f). +func TestPropertyDictionary_ExportWritesBothIdentityMembers(t *testing.T) { + out, err := MarshalPropertyDictionary(&PropertyDictionary{Properties: []PropertyDefinition{ + {Key: "dueDate", Name: "End", Format: model.RelationFormat_date}, + {Key: "6a83296f61fab2265263ae34", Name: "Budget", Format: model.RelationFormat_number}, + }}) + require.NoError(t, err) + assert.Contains(t, string(out), `"property": "Due date"`) + assert.Contains(t, string(out), `"internal_key": "dueDate"`) + assert.Contains(t, string(out), `"property": "6a83296f61fab2265263ae34"`, + "a bson id has no slug and must never be given one (§2f)") + assert.Contains(t, string(out), `"internal_key": "6a83296f61fab2265263ae34"`) +} + +// One property, one slot — whichever member names it. Two entries that state +// one identity through the two different members are still two definitions +// of one property, refused with the first occurrence named (§2e, §2f). +func TestPropertyDictionary_DuplicateAcrossTheTwoIdentityMembers(t *testing.T) { + _, err := UnmarshalPropertyDictionary([]byte(`{"version":2,"properties":[ + {"property":"6a83296f61fab2265263ae34","format":"number"}, + {"internal_key":"6a83296f61fab2265263ae34","format":"text"}]}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/properties/1/property") + assert.Contains(t, err.Error(), "already defined at /properties/0") +} + +// property_settings refuses the identity pair the way it refused `key`: a +// relation document's stored key is the envelope `internal_key`, and its +// spelling is derived, never stated — admitting either member would be a +// second spelling of a fact another surface owns (§2d). +func TestRelationSettings_RefusesTheIdentityPair(t *testing.T) { + for member, wantHome := range map[string]string{ + "property": "internal_key", + "internal_key": "envelope", + } { + t.Run(member, func(t *testing.T) { + err := Validate([]byte(`{"version":2,"kind":"property","id":"o1","internal_key":"b", + "property_settings":{"format":"number","` + member + `":"x"}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/property_settings/"+member) + assert.Contains(t, strings.ToLower(err.Error()), wantHome, + "the refusal names where the fact lives") + }) + } +} + +// An unwritable `internal_key` member is refused with the member's own path +// and a readable reason — the same writable-key rule every stored-key slot +// carries (§3), stated where the fault is instead of as a bare schema bound. +func TestPropertyDefinitions_UnwritableInternalKeyIsRefusedByName(t *testing.T) { + err := Validate([]byte(`{"version": 2, "kind": "object_type", "internal_key": "t", + "type_settings": {"property_definitions": [{"internal_key": "a\nb", "format": "text"}]}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/type_settings/property_definitions/0/internal_key") +} diff --git a/pkg/lib/anyblockjson/invalidutf8_test.go b/pkg/lib/anyblockjson/invalidutf8_test.go new file mode 100644 index 0000000000..e3932a9d54 --- /dev/null +++ b/pkg/lib/anyblockjson/invalidutf8_test.go @@ -0,0 +1,88 @@ +package anyblockjson + +// invalidutf8_test.go — a display name that is not valid UTF-8 is not a +// spelling. +// +// The writer maps every invalid byte to U+FFFD; the collision plan compares +// raw Go strings. So two names differing ONLY in their invalid bytes looked +// distinct to the plan, took no suffix, and then rendered as one member +// name — a JSON object holds a member once, so one value replaced the other +// and Validate saw nothing wrong, because by then the collision had already +// happened. +// +// Zero occurrences in the 77-space corpus; the retired normalization +// grammar dropped U+FFFD as a matter of course, so the exposure arrived +// with raw names. Hardening, pinned. + +import ( + "encoding/json" + "testing" + "unicode/utf8" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +func TestInvalidUTF8IsNotASpelling(t *testing.T) { + // two names a reader would call different and the writer cannot: the + // invalid byte is the only thing between them, and it is the byte that + // does not survive + const ( + keyA = "6a7663db61fab21cd4b900a1" + keyB = "6a7663db61fab21cd4b900b2" + nameA = "Region \xff" + nameB = "Region \xfe" + ) + require.False(t, utf8.ValidString(nameA)) + require.False(t, utf8.ValidString(nameB)) + + t.Run("neither name is a writable key", func(t *testing.T) { + assert.False(t, isWritablePropertyKey(nameA)) + assert.Contains(t, unwritableKeyReason("property key", nameA), "not valid UTF-8") + }) + + t.Run("both keys are written verbatim and both values survive", func(t *testing.T) { + // given + vocab := nameVocab{names: map[string]string{keyA: nameA, keyB: nameB}} + snap := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("o1"), + keyA: pbtypes.String("value of A"), + keyB: pbtypes.String("value of B"), + }}} + opts := Options{Keys: vocab} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + + // then — the stored key is always its own address, so each holder + // keeps its own member and neither value is lost + require.NoError(t, Validate(data), "I1:\n%s", data) + var doc struct { + Properties map[string]string `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "value of A", doc.Properties[keyA]) + assert.Equal(t, "value of B", doc.Properties[keyB]) + assert.Len(t, doc.Properties, 2, "no member was written twice and silently collapsed") + + // I2's substance, and the fixpoint + _, back, err := Unmarshal(data, opts) + require.NoError(t, err) + assert.Equal(t, "value of A", back.Details.Fields[keyA].GetStringValue()) + assert.Equal(t, "value of B", back.Details.Fields[keyB].GetStringValue()) + again, err := Marshal(model.SmartBlockType_Page, back, opts) + require.NoError(t, err) + assert.Equal(t, string(data), string(again)) + }) + + t.Run("a legend entry is refused rather than written unreadable", func(t *testing.T) { + reason, refused := legendEntryRefusal(nameA, keyA, true) + assert.True(t, refused) + assert.Contains(t, reason, "not valid UTF-8") + }) +} diff --git a/pkg/lib/anyblockjson/json.go b/pkg/lib/anyblockjson/json.go new file mode 100644 index 0000000000..94bdba10f5 --- /dev/null +++ b/pkg/lib/anyblockjson/json.go @@ -0,0 +1,1153 @@ +package anyblockjson + +// json.go holds the shared serialization infrastructure: the ordered-JSON +// writer that produces the §4 canonical byte form, the enum name tables, the +// proto value bridges, date formatting, and id helpers. + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// omap is a JSON object with explicit key order — the canonical form fixes +// key order (§4), which encoding/json maps cannot express. +type omap struct { + keys []string + vals []any +} + +func (m *omap) set(k string, v any) { + m.keys = append(m.keys, k) + m.vals = append(m.vals, v) +} + +// sortedNestedOmap renders a two-level string map into nested omaps, both +// levels sorted by key (§4 canon). Returns nil for an empty map so +// setNonEmpty omits the slot. +func sortedNestedOmap(m map[string]map[string]string) *omap { + if len(m) == 0 { + return nil + } + out := &omap{} + for _, outer := range sortedStringKeys(m) { + inner := &omap{} + for _, k := range sortedStringKeys(m[outer]) { + inner.set(k, m[outer][k]) + } + out.set(outer, inner) + } + return out +} + +func sortedStringKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// setNonEmpty appends k only when v is not an empty/default value (§4: +// canonical form omits empty strings, arrays, objects and default scalars). +func (m *omap) setNonEmpty(k string, v any) { + switch x := v.(type) { + case string: + if x == "" { + return + } + case bool: + if !x { + return + } + case int: + if x == 0 { + return + } + case int32: + if x == 0 { + return + } + case int64: + if x == 0 { + return + } + case float64: + if x == 0 { + return + } + case []any: + if len(x) == 0 { + return + } + case *omap: + if x == nil || len(x.keys) == 0 { + return + } + case nil: + return + } + m.set(k, v) +} + +func (m *omap) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + if err := encodeJSONValue(&buf, m); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func encodeJSONValue(buf *bytes.Buffer, v any) error { + switch x := v.(type) { + case nil: + buf.WriteString("null") + case *omap: + buf.WriteByte('{') + for i, k := range x.keys { + if i > 0 { + buf.WriteByte(',') + } + writeJSONString(buf, k) + buf.WriteByte(':') + if err := encodeJSONValue(buf, x.vals[i]); err != nil { + return err + } + } + buf.WriteByte('}') + case []any: + buf.WriteByte('[') + for i, e := range x { + if i > 0 { + buf.WriteByte(',') + } + if err := encodeJSONValue(buf, e); err != nil { + return err + } + } + buf.WriteByte(']') + case string: + writeJSONString(buf, x) + default: + // numbers and booleans contain no HTML-escapable characters + b, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("encode value: %w", err) + } + buf.Write(b) + } + return nil +} + +// writeJSONString escapes like encoding/json but without HTML escaping, so +// inline markup tags stay readable. +func writeJSONString(buf *bytes.Buffer, s string) { + buf.WriteByte('"') + for _, r := range s { + switch r { + case '"': + buf.WriteString(`\"`) + case '\\': + buf.WriteString(`\\`) + case '\n': + buf.WriteString(`\n`) + case '\r': + buf.WriteString(`\r`) + case '\t': + buf.WriteString(`\t`) + case '\u2028': + buf.WriteString(`\u2028`) + case '\u2029': + buf.WriteString(`\u2029`) + default: + if r < 0x20 { + fmt.Fprintf(buf, `\u%04x`, r) + } else { + buf.WriteRune(r) + } + } + } + buf.WriteByte('"') +} + +// marshalCanonical renders the document omap in the canonical byte form: +// UTF-8, LF, two-space indent, trailing newline (§4). +func marshalCanonical(doc *omap) ([]byte, error) { + compact, err := doc.MarshalJSON() + if err != nil { + return nil, err + } + var out bytes.Buffer + if err := json.Indent(&out, compact, "", " "); err != nil { + return nil, fmt.Errorf("indent document: %w", err) + } + out.WriteByte('\n') + return out.Bytes(), nil +} + +// +// ---- enum name tables ---- +// + +type enumNames[T comparable] struct { + toName map[T]string + toVal map[string]T +} + +func newEnumNames[T comparable](pairs map[T]string) enumNames[T] { + e := enumNames[T]{toName: pairs, toVal: make(map[string]T, len(pairs))} + for v, n := range pairs { + e.toVal[n] = v + } + return e +} + +func (e enumNames[T]) name(v T) string { return e.toName[v] } +func (e enumNames[T]) value(n string) T { return e.toVal[n] } +func (e enumNames[T]) has(n string) bool { _, ok := e.toVal[n]; return ok } + +var kindNames = newEnumNames(map[model.SmartBlockType]string{ + model.SmartBlockType_AccountOld: "account_old", + model.SmartBlockType_Page: "page", + model.SmartBlockType_ProfilePage: "profile_page", + model.SmartBlockType_Home: "home", + model.SmartBlockType_Archive: "archive", + model.SmartBlockType_Widget: "widget", + model.SmartBlockType_File: "file", + model.SmartBlockType_Template: "template", + model.SmartBlockType_BundledTemplate: "bundled_template", + // the three definition kinds say "property" where the store says + // "relation": the product calls these things properties, and the format + // already did everywhere else — the block type is `featured_properties`, + // the shape is `propertyDefinition`, the legend `property_internal_keys`. + // One word for one concept (§15 #14); the model constants are the store's + // own names and stay. + model.SmartBlockType_BundledRelation: "bundled_property", + model.SmartBlockType_SubObject: "sub_object", + model.SmartBlockType_BundledObjectType: "bundled_object_type", + model.SmartBlockType_AnytypeProfile: "anytype_profile", + model.SmartBlockType_Date: "date", + // the space's own object holds the space's SETTINGS — its name, icon, + // homepage — not the space itself, and `space_settings` says that where + // `workspace` said something the product no longer calls anything. One + // per space (77 in a 77-space corpus), machine-written and never + // authored, so the rename costs nothing but is a wire value: after the + // freeze it would cost a version. + model.SmartBlockType_Workspace: "space_settings", + model.SmartBlockType_STRelation: "property", + model.SmartBlockType_STType: "object_type", + model.SmartBlockType_STRelationOption: "property_option", + model.SmartBlockType_SpaceView: "space_view", + model.SmartBlockType_Identity: "identity", + model.SmartBlockType_Participant: "participant", + model.SmartBlockType_MissingObject: "missing_object", + model.SmartBlockType_FileObject: "file_object", + model.SmartBlockType_NotificationObject: "notification", + model.SmartBlockType_DevicesObject: "devices", + model.SmartBlockType_ChatObjectDeprecated: "chat_object", + model.SmartBlockType_ChatDerivedObject: "chat", + model.SmartBlockType_AccountObject: "account", + model.SmartBlockType_DiscussionObject: "discussion", + model.SmartBlockType_TechSpaceObject: "tech_space", + model.SmartBlockType_TechSpaceVirtualObject: "tech_space_virtual", +}) + +// textStyleNames maps text styles to JSON block types. Header4 is absent on +// purpose: deprecated Header4 blocks export as heading3 (§5). +var textStyleNames = newEnumNames(map[model.BlockContentTextStyle]string{ + model.BlockContentText_Paragraph: "paragraph", + model.BlockContentText_Header1: "heading_1", + model.BlockContentText_Header2: "heading_2", + model.BlockContentText_Header3: "heading_3", + model.BlockContentText_Quote: "quote", + model.BlockContentText_Code: "code", + model.BlockContentText_Title: "title", + model.BlockContentText_Checkbox: "checkbox", + model.BlockContentText_Marked: "bulleted_list_item", + model.BlockContentText_Numbered: "numbered_list_item", + model.BlockContentText_Toggle: "toggle", + model.BlockContentText_Description: "description", + model.BlockContentText_Callout: "callout", + model.BlockContentText_ToggleHeader1: "toggle_heading_1", + model.BlockContentText_ToggleHeader2: "toggle_heading_2", + model.BlockContentText_ToggleHeader3: "toggle_heading_3", +}) + +var fileTypeNames = newEnumNames(map[model.BlockContentFileType]string{ + model.BlockContentFile_File: "file", + model.BlockContentFile_Image: "image", + model.BlockContentFile_Video: "video", + model.BlockContentFile_Audio: "audio", + model.BlockContentFile_PDF: "pdf", +}) + +var alignNames = newEnumNames(map[model.BlockAlign]string{ + model.Block_AlignLeft: "left", + model.Block_AlignCenter: "center", + model.Block_AlignRight: "right", + model.Block_AlignJustify: "justify", +}) + +var verticalAlignNames = newEnumNames(map[model.BlockVerticalAlign]string{ + model.Block_VerticalAlignTop: "top", + model.Block_VerticalAlignMiddle: "middle", + model.Block_VerticalAlignBottom: "bottom", +}) + +var fileStyleNames = newEnumNames(map[model.BlockContentFileStyle]string{ + model.BlockContentFile_Auto: "auto", + model.BlockContentFile_Link: "link", + model.BlockContentFile_Embed: "embed", +}) + +var cardStyleNames = newEnumNames(map[model.BlockContentLinkCardStyle]string{ + model.BlockContentLink_Text: "text", + model.BlockContentLink_Card: "card", + model.BlockContentLink_Inline: "inline", +}) + +var iconSizeNames = newEnumNames(map[model.BlockContentLinkIconSize]string{ + model.BlockContentLink_SizeNone: "none", + model.BlockContentLink_SizeSmall: "small", + model.BlockContentLink_SizeMedium: "medium", +}) + +// linkDescriptionNames: proto "Added" is the manually-set description (§5). +var linkDescriptionNames = newEnumNames(map[model.BlockContentLinkDescription]string{ + model.BlockContentLink_None: "none", + model.BlockContentLink_Added: "manual", + model.BlockContentLink_Content: "content", +}) + +var divStyleNames = newEnumNames(map[model.BlockContentDivStyle]string{ + model.BlockContentDiv_Line: "line", + model.BlockContentDiv_Dots: "dots", +}) + +var processorNames = newEnumNames(map[model.BlockContentLatexProcessor]string{ + model.BlockContentLatex_Latex: "latex", + model.BlockContentLatex_Mermaid: "mermaid", + model.BlockContentLatex_Chart: "chart", + model.BlockContentLatex_Youtube: "youtube", + model.BlockContentLatex_Vimeo: "vimeo", + model.BlockContentLatex_Soundcloud: "soundcloud", + model.BlockContentLatex_GoogleMaps: "google_maps", + model.BlockContentLatex_Miro: "miro", + model.BlockContentLatex_Figma: "figma", + model.BlockContentLatex_Twitter: "twitter", + model.BlockContentLatex_OpenStreetMap: "open_street_map", + model.BlockContentLatex_Reddit: "reddit", + model.BlockContentLatex_Facebook: "facebook", + model.BlockContentLatex_Instagram: "instagram", + model.BlockContentLatex_Telegram: "telegram", + model.BlockContentLatex_GithubGist: "github_gist", + model.BlockContentLatex_Codepen: "codepen", + model.BlockContentLatex_Bilibili: "bilibili", + model.BlockContentLatex_Excalidraw: "excalidraw", + model.BlockContentLatex_Kroki: "kroki", + model.BlockContentLatex_Graphviz: "graphviz", + model.BlockContentLatex_Sketchfab: "sketchfab", + model.BlockContentLatex_Image: "image", + model.BlockContentLatex_Drawio: "drawio", + model.BlockContentLatex_Spotify: "spotify", +}) + +// sourceProcessors carry source code in text; all others carry a URL (§5.2). +var sourceProcessors = map[model.BlockContentLatexProcessor]bool{ + model.BlockContentLatex_Latex: true, + model.BlockContentLatex_Mermaid: true, + model.BlockContentLatex_Chart: true, + model.BlockContentLatex_Graphviz: true, + model.BlockContentLatex_Kroki: true, + model.BlockContentLatex_Excalidraw: true, + model.BlockContentLatex_Drawio: true, +} + +var widgetLayoutNames = newEnumNames(map[model.BlockContentWidgetLayout]string{ + model.BlockContentWidget_Link: "link", + model.BlockContentWidget_Tree: "tree", + model.BlockContentWidget_List: "list", + model.BlockContentWidget_CompactList: "compact_list", + model.BlockContentWidget_View: "view", +}) + +var viewTypeNames = newEnumNames(map[model.BlockContentDataviewViewType]string{ + model.BlockContentDataviewView_Table: "table", + model.BlockContentDataviewView_List: "list", + model.BlockContentDataviewView_Gallery: "gallery", + model.BlockContentDataviewView_Kanban: "kanban", + model.BlockContentDataviewView_Calendar: "calendar", + model.BlockContentDataviewView_Graph: "graph", +}) + +var cardSizeNames = newEnumNames(map[model.BlockContentDataviewViewSize]string{ + model.BlockContentDataviewView_Small: "small", + model.BlockContentDataviewView_Medium: "medium", + model.BlockContentDataviewView_Large: "large", +}) + +var listSizeNames = newEnumNames(map[model.BlockContentDataviewViewListSize]string{ + model.BlockContentDataviewView_Compact: "compact", + model.BlockContentDataviewView_Regular: "regular", +}) + +var sortDirectionNames = newEnumNames(map[model.BlockContentDataviewSortType]string{ + model.BlockContentDataviewSort_Asc: "asc", + model.BlockContentDataviewSort_Desc: "desc", + model.BlockContentDataviewSort_Custom: "custom", +}) + +var emptyPlacementNames = newEnumNames(map[model.BlockContentDataviewSortEmptyType]string{ + model.BlockContentDataviewSort_Start: "start", + model.BlockContentDataviewSort_End: "end", +}) + +var conditionNames = newEnumNames(map[model.BlockContentDataviewFilterCondition]string{ + model.BlockContentDataviewFilter_Equal: "equal", + model.BlockContentDataviewFilter_NotEqual: "not_equal", + model.BlockContentDataviewFilter_Greater: "greater", + model.BlockContentDataviewFilter_Less: "less", + model.BlockContentDataviewFilter_GreaterOrEqual: "greater_or_equal", + model.BlockContentDataviewFilter_LessOrEqual: "less_or_equal", + model.BlockContentDataviewFilter_Like: "contains", + model.BlockContentDataviewFilter_NotLike: "not_contains", + model.BlockContentDataviewFilter_In: "in", + model.BlockContentDataviewFilter_NotIn: "not_in", + model.BlockContentDataviewFilter_Empty: "empty", + model.BlockContentDataviewFilter_NotEmpty: "not_empty", + model.BlockContentDataviewFilter_AllIn: "all_in", + model.BlockContentDataviewFilter_NotAllIn: "not_all_in", + model.BlockContentDataviewFilter_ExactIn: "exact_in", + model.BlockContentDataviewFilter_NotExactIn: "not_exact_in", + model.BlockContentDataviewFilter_Exists: "exists", +}) + +// countingPresets take a day count from the filter's `value` rather than +// naming a fixed period: getDateRange reads it as a NUMBER OF DAYS for these +// two and for no others (pkg/lib/database/quickoptions.go — the exactDate +// default reads the same field, as the timestamp it is). Without a value the +// count is 0, which silently means "today" — but only where the range reaches +// the query at all, which takes a date property and one of the six conditions +// below (transformDateFilter, datePresetConditions). Everywhere else the +// preset is inert and the count is never read, which is why the validation +// rule this set feeds is scoped and export writes the count regardless. +var countingPresets = map[model.BlockContentDataviewFilterQuickOption]struct{}{ + model.BlockContentDataviewFilter_NumberOfDaysAgo: {}, + model.BlockContentDataviewFilter_NumberOfDaysNow: {}, +} + +func countingPreset(q model.BlockContentDataviewFilterQuickOption) bool { + _, ok := countingPresets[q] + return ok +} + +// countingPresetNames is the same set by name, for validation. +var countingPresetNames = map[string]struct{}{ + "number_of_days_ago": {}, + "number_of_days_now": {}, +} + +// datePresetConditions are the conditions that apply a preset's day range at +// all — the condition half of transformDateFilter's gate. It computes the +// range for every DATE filter (a filter of any other format it returns before +// computing anything, which is the other half), then substitutes the range +// into the filter for these six and no others +// (pkg/lib/database/quickoptions.go): on any other condition — the +// presence-only leaves above all — it returns the filter unchanged, so the +// preset is inert and its day count is never read. That is why a counting +// preset without a count is an error here and nothing at all there. +var datePresetConditions = map[string]struct{}{ + "equal": {}, + "in": {}, + "less": {}, + "greater": {}, + "less_or_equal": {}, + "greater_or_equal": {}, +} + +var datePresetNames = newEnumNames(map[model.BlockContentDataviewFilterQuickOption]string{ + model.BlockContentDataviewFilter_Yesterday: "yesterday", + model.BlockContentDataviewFilter_Today: "today", + model.BlockContentDataviewFilter_Tomorrow: "tomorrow", + model.BlockContentDataviewFilter_LastWeek: "last_week", + model.BlockContentDataviewFilter_CurrentWeek: "current_week", + model.BlockContentDataviewFilter_NextWeek: "next_week", + model.BlockContentDataviewFilter_LastMonth: "last_month", + model.BlockContentDataviewFilter_CurrentMonth: "current_month", + model.BlockContentDataviewFilter_NextMonth: "next_month", + model.BlockContentDataviewFilter_NumberOfDaysAgo: "number_of_days_ago", + model.BlockContentDataviewFilter_NumberOfDaysNow: "number_of_days_now", + model.BlockContentDataviewFilter_LastYear: "last_year", + model.BlockContentDataviewFilter_CurrentYear: "current_year", + model.BlockContentDataviewFilter_NextYear: "next_year", +}) + +var aggregationNames = newEnumNames(map[model.BlockContentDataviewRelationFormulaType]string{ + model.BlockContentDataviewRelation_Count: "count", + model.BlockContentDataviewRelation_CountValue: "count_value", + model.BlockContentDataviewRelation_CountDistinct: "count_distinct", + model.BlockContentDataviewRelation_CountEmpty: "count_empty", + model.BlockContentDataviewRelation_CountNotEmpty: "count_not_empty", + model.BlockContentDataviewRelation_PercentEmpty: "percent_empty", + model.BlockContentDataviewRelation_PercentNotEmpty: "percent_not_empty", + model.BlockContentDataviewRelation_MathSum: "sum", + model.BlockContentDataviewRelation_MathAverage: "average", + model.BlockContentDataviewRelation_MathMedian: "median", + model.BlockContentDataviewRelation_MathMin: "min", + model.BlockContentDataviewRelation_MathMax: "max", + model.BlockContentDataviewRelation_Range: "range", +}) + +// FormatName returns the format's canonical JSON name for a property format +// ("text", "select", "objects", …) — the one vocabulary shared by documents +// and API surfaces. It is the exported form of formatName, so +// it applies the same shorttext→"text" fold. Unknown formats return "". +func FormatName(f model.RelationFormat) string { + return formatName(f) +} + +// FormatByName is FormatName's inverse: it maps a §3 format name back to the +// internal relation format. ok is false for names outside the vocabulary. +// "text" maps to longtext (the map's side of the fold); where an existing +// property's stored format matters, the import path resolves it instead. +func FormatByName(name string) (model.RelationFormat, bool) { + if !formatNames.has(name) { + return 0, false + } + return formatNames.value(name), true +} + +// formatNames follows the public REST API vocabulary (§3). Text has exactly +// one name: the editor offers a single Text format, so the stored +// longtext/shorttext split stays out of this serialization — shorttext has +// no name of its own and folds into "text" via formatName. The map must +// remain a bijection (newEnumNames inverts it, and a duplicated name would +// invert nondeterministically), which is why the fold lives outside it. +// +// It is TOTAL over model.RelationFormat, shorttext's fold aside, and that is +// a load-bearing property rather than tidiness: a relation document states +// its format on the envelope as a required NAME (§2d), so a stored format +// this map cannot name is a relation object Marshal cannot export. "map" +// (RelationFormat_map) is in the vocabulary for exactly that reason — the +// API does not serve it, but 72 production relation documents carry format +// 102 (every one the bundled templatePlaceholders relation), and the §3 note +// that names exist for internal formats (emoji, objects, properties) already +// covers it. TestFormatNames_TotalOverModelEnum pins the totality, so a +// format added to the model without a name here fails a test instead of an +// export. +var formatNames = newEnumNames(map[model.RelationFormat]string{ + model.RelationFormat_longtext: "text", + model.RelationFormat_number: "number", + model.RelationFormat_status: "select", + model.RelationFormat_tag: "multi_select", + model.RelationFormat_date: "date", + model.RelationFormat_file: "files", + model.RelationFormat_checkbox: "checkbox", + model.RelationFormat_url: "url", + model.RelationFormat_email: "email", + model.RelationFormat_phone: "phone", + model.RelationFormat_emoji: "emoji", + model.RelationFormat_object: "objects", + model.RelationFormat_relations: "properties", + model.RelationFormat_map: "map", +}) + +// filterTemplatePrefix marks a dynamic filter value: a placeholder the +// client substitutes for a real object id before it issues the query +// (anytype-ts Dataview.valueTemplateMapper). The tokens are built as +// sprintf("_filter_template_%d_", FilterValueTemplate) — _filter_template_2_ +// is the current user, resolving to _participant__, and +// _filter_template_1_ is the object hosting an inline dataview, resolving to +// its id. +// +// They are stored verbatim in the filter's value and are opaque to the +// middleware: nothing in Go resolves them, so a query evaluated server-side +// compares against the literal string and matches nothing. They are not +// object ids and must never be remapped as such. +const filterTemplatePrefix = "_filter_template_" + +func isFilterTemplate(v string) bool { + return strings.HasPrefix(v, filterTemplatePrefix) +} + +// layoutNames maps the object layout enum to the names this format uses. +// Layout is *stored* as a number (its bundled relation's format is `number`), +// but a bare integer would be the one opaque enum in an otherwise +// self-describing format — every other enum here is a name (§3). +var layoutNames = newEnumNames(map[model.ObjectTypeLayout]string{ + model.ObjectType_basic: "basic", + model.ObjectType_profile: "profile", + model.ObjectType_todo: "todo", + model.ObjectType_set: "set", + model.ObjectType_objectType: "object_type", + // the wire names for the three relation-flavored layouts say "property", + // like the kinds above — only the NAME moves, the model constants they + // map from are the store's + model.ObjectType_relation: "property", + model.ObjectType_file: "file", + model.ObjectType_dashboard: "dashboard", + model.ObjectType_image: "image", + model.ObjectType_note: "note", + model.ObjectType_space: "space", + model.ObjectType_bookmark: "bookmark", + model.ObjectType_relationOptionsList: "property_options_list", + model.ObjectType_relationOption: "property_option", + model.ObjectType_collection: "collection", + model.ObjectType_audio: "audio", + model.ObjectType_video: "video", + model.ObjectType_date: "date", + model.ObjectType_spaceView: "space_view", + model.ObjectType_participant: "participant", + model.ObjectType_pdf: "pdf", + model.ObjectType_chatDeprecated: "chat_deprecated", + model.ObjectType_chatDerived: "chat_derived", + model.ObjectType_tag: "tag", + model.ObjectType_notification: "notification", + model.ObjectType_missingObject: "missing_object", + model.ObjectType_devices: "devices", + model.ObjectType_discussion: "discussion", +}) + +// propertyVocabulary is one stored property key's name-over-number contract +// (§3): the stored value is a number whose meaning is a proto enum, and the +// format writes the NAME — a bare integer would be an opaque enum in an +// otherwise self-describing format. All four surfaces that touch such a key +// ask this one struct, so they cannot disagree about what a name means: +// export substitutes the name for an in-vocabulary number (and refuses to +// write a stored string the vocabulary does not name — there is no way to +// write it, I1); import maps a known name back to its number; validation +// refuses an unknown name as an ERROR, because the typo would otherwise +// import as a raw string onto a number-format detail, where every consumer +// reads it with an int getter and silently sees the enum's zero; and a raw +// number outside the vocabulary passes every surface unchanged, because a +// stored value round-trips as its number rather than being lost. +type propertyVocabulary struct { + what string // the concept a refusal names: "layout", "align", … + has func(string) bool // is this string a vocabulary name + value func(string) float64 // name → stored number (only for names has() admits) + name func(float64) string // stored number → name; "" outside the vocabulary + names func() []string // the vocabulary, sorted, for refusals that state it +} + +// vocabularyOf adapts an enumNames table to the property contract. The name +// direction reads the number the way every consumer of these details does — +// int32 of the float — GUARDED the way relationFormatName is: int32(NaN) is 0 +// on this machine, and without the guard a NaN stored on a layout key exports +// as the enum's zero's name, a false claim that then imports as a permanent +// silent rewrite. A fraction, an infinity or an out-of-int32 number likewise +// has no name and round-trips as the number it is. +func vocabularyOf[T ~int32](e enumNames[T], what string) propertyVocabulary { + return propertyVocabulary{ + what: what, + has: e.has, + value: func(n string) float64 { return float64(e.value(n)) }, + name: func(n float64) string { + if math.IsNaN(n) || math.IsInf(n, 0) || n != math.Trunc(n) || + n < math.MinInt32 || n > math.MaxInt32 { + return "" + } + return e.name(T(int32(n))) + }, + names: func() []string { + out := make([]string, 0, len(e.toVal)) + for n := range e.toVal { + out = append(out, n) + } + sort.Strings(out) + return out + }, + } +} + +// quotedNames renders the vocabulary for a refusal — 'a', 'b', 'c' — in the +// same quoting the schema's own enum errors use, so the two refusal channels +// read as one. +func (v propertyVocabulary) quotedNames() string { + names := v.names() + for i, n := range names { + names[i] = "'" + n + "'" + } + return strings.Join(names, ", ") +} + +var layoutVocabulary = vocabularyOf(layoutNames, "layout") + +// alignVocabulary spells a model.BlockAlign — the enum a block's `align`, +// a view column's `align` and the layoutAlign DETAIL all store. One concept, +// one spelling (§15 #14): the four names were already the format's alignment +// vocabulary twice over before the property joined. +var alignVocabulary = vocabularyOf(alignNames, "align") + +// originNames maps model.ObjectOrigin — how an object entered its space — +// to the format's names: the proto's own identifiers, snake_cased where they +// are camelCase, because there is no established public vocabulary to defer +// to (the REST API stores the same number). TOTAL over the proto enum, +// pinned by TestNamedEnum_VocabulariesTotalOverModelEnums: a member added to +// the proto without a name here would export as a bare integer again. +var originNames = newEnumNames(map[model.ObjectOrigin]string{ + model.ObjectOrigin_none: "none", + model.ObjectOrigin_clipboard: "clipboard", + model.ObjectOrigin_dragAndDrop: "drag_and_drop", + model.ObjectOrigin_import: "import", + model.ObjectOrigin_webclipper: "webclipper", + model.ObjectOrigin_sharingExtension: "sharing_extension", + model.ObjectOrigin_usecase: "usecase", + model.ObjectOrigin_builtin: "builtin", + model.ObjectOrigin_bookmark: "bookmark", + model.ObjectOrigin_api: "api", +}) + +var originVocabulary = vocabularyOf(originNames, "origin") + +// importTypeNames maps model.ImportType — which importer brought an +// import/usecase-originated object in. The names are the proto identifiers +// lowercased; `pb` stays `pb` (the protobuf export format, the store's own +// name for it) rather than gaining an invented alias. Note the enum's ZERO +// is notion — the sharpest reason this key had to be named or die: an +// accepted-then-zeroed string here did not read as "unset", it read as a +// false claim that the object came from Notion. +var importTypeNames = newEnumNames(map[model.ImportType]string{ + model.Import_Notion: "notion", + model.Import_Markdown: "markdown", + model.Import_External: "external", + model.Import_Pb: "pb", + model.Import_Html: "html", + model.Import_Txt: "txt", + model.Import_Csv: "csv", + model.Import_Obsidian: "obsidian", +}) + +var importTypeVocabulary = vocabularyOf(importTypeNames, "import type") + +// imageKindNames maps model.ImageKind — what an image was uploaded FOR — to +// the format's names: the proto identifiers snake_cased. TOTAL over the +// proto enum, pinned below. +// +// This is the fourth of the five 2026-08 bare-integer enums to be named, and +// it is named on the same measured ground the others were left as numbers: +// imageKind occurs on 4,079 file objects across the 77-space corpus — 4,053 +// automatically_added, 23 icon, 3 basic-or-cover — where widgetLayout is on +// 13 documents and headerRelationsLayout on 51. A reader of an export saw a +// bare 3 and had no way to learn what it meant. +// +// The two small ones were once recorded as 13 and ZERO, and the zero was +// wrong: headerRelationsLayout is on 51 documents and holds two distinct +// values (44 ones, 7 zeros), which is what typesettings.go already says +// about it. The decision to leave it bare therefore rests on VOLUME alone +// now, not on "nothing writes it" — 51 documents against imageKind's 4,079 +// — and it is the weakest of the five verdicts on that account. +// +// Note the enum's ZERO is `basic`, and the app never STORES it: +// makeInitialDetails returns early for Basic, so the key is absent rather +// than 0 on an ordinary upload. The name exists anyway because a total +// vocabulary is what keeps a future writer of 0 from exporting a bare +// integer, and because absent and basic must not be forced to differ. +var imageKindNames = newEnumNames(map[model.ImageKind]string{ + model.ImageKind_Basic: "basic", + model.ImageKind_Cover: "cover", + model.ImageKind_Icon: "icon", + model.ImageKind_AutomaticallyAdded: "automatically_added", +}) + +var imageKindVocabulary = vocabularyOf(imageKindNames, "image kind") + +// viewTypeVocabulary is not a property vocabulary — no stored detail key +// maps to it — but §2a's default_view member shares the reading, and the +// guarded adapter is how both enum members stopped naming NaN. +var viewTypeVocabulary = vocabularyOf(viewTypeNames, "view type") + +// namedEnumProperties maps each stored property key whose number the format +// names onto its vocabulary (§3). The three layout keys hold an +// ObjectTypeLayout. The remaining layout-ish bundled keys are left as +// numbers deliberately: layoutWidth is a fraction, not an enum, and +// widgetLayout/headerRelationsLayout hold enums almost nothing writes — 13 +// and 51 occurrences across 28,831 real exported documents, against +// imageKind's 4,079. +var namedEnumProperties = map[string]propertyVocabulary{ + "recommendedLayout": layoutVocabulary, + "layout": layoutVocabulary, + "resolvedLayout": layoutVocabulary, + // layoutAlign is the object's own page alignment — the one key of the + // five 2026-08 bare-integer enums a user can set (readonly false in the + // bundled table), which is why it is NAMED rather than deprecated: it + // survives the §2a admission on type documents as "the type object's own + // page display, set by a person where non-zero", and the app writes it + // as a model.BlockAlign (participant/profile editors stamp AlignCenter; + // the align UI sets the rest). Before this entry, `layout_align: + // "center"` VALIDATED and stored the string on a number detail — every + // int getter answered 0, left — while the reader of an export saw a bare + // 1 beside a named `layout` and had no way to learn what it meant. + "layoutAlign": alignVocabulary, + // origin and importType are the object's PROVENANCE — how it entered the + // space it was exported from — and they are NAMED rather than deprecated + // on the format's own precedent: the §2a admission dropped `origin` from + // TYPE documents as install provenance precisely because "on ordinary + // objects origin is real provenance and stays", and §2f drops both only + // on bundled-identical property documents. The corpus agrees it is real: + // all TEN origin values occur across 15,943 documents (import 6,463 · + // bookmark 2,444 · api 2,293 · webclipper 2,080 · usecase 1,110 · + // clipboard 449 · none 425 · builtin 333 · drag_and_drop 301 · + // sharing_extension 45) — a reader can tell an object a person clipped + // from one a pipeline made, which is not the class of syncStatus but the + // class of createdDate (which the import pipeline deliberately preserves + // as OriginalCreatedTimestamp) and creator (written as attribution). + // + // Deprecation was weighed: heart's own import pipeline re-stamps both on + // every snapshot (objectcreator.injectImportDetails), so nothing + // downstream of an app import acts on the carried value. But the format + // is a READ surface first, and a transient key must describe a MOMENT + // rather than the object — origin describes the object's history. The + // pair travels together: objectorigin.go writes importType only beside + // an import/usecase origin. + "origin": originVocabulary, + "importType": importTypeVocabulary, + // imageKind records what an image was uploaded FOR — a cover, an icon, + // or added automatically by a pipeline. It is NAMED rather than + // deprecated even though heart has one writer and no reader, because + // the format is a read surface first: a person looking at a file object + // wants to know why the image is there. + // + // Deprecation was weighed and is still arguable. The behaviour a client + // actually runs on is `isHiddenDiscovery`, which travels independently + // and is in perfect lockstep with the automatically_added member — 4,053 + // of 4,053 in the corpus — so the one live consumer (the client's + // subscription filter, which hides auto-added images) survives without + // this key. The two anytype-ts filters that DO read imageKind, in the + // icon and cover pickers, are both commented out. What would be lost is + // the 26 documents where the key says icon or cover and nothing else + // does, and even those are recoverable from whichever object references + // the image through icon_image or cover_id. + // + // It stays because naming costs one entry and drops nothing, while + // dropping 4,079 documents' worth of a stored, user-visible-in-principle + // fact is a decision the freeze does not need to take. + "imageKind": imageKindVocabulary, +} + +// namedEnumProperty answers whether a stored key is written by name, and +// with which vocabulary. +func namedEnumProperty(key string) (propertyVocabulary, bool) { + v, ok := namedEnumProperties[key] + return v, ok +} + +// formatName is the export-side name of a stored format: the canonical name +// from formatNames, with legacy shorttext folded into "text" (§3). +func formatName(f model.RelationFormat) string { + if f == model.RelationFormat_shorttext { + f = model.RelationFormat_longtext + } + return formatNames.name(f) +} + +// +// ---- proto value bridges ---- +// + +// protoValueToJSON converts a types.Value tree into JSON values, with omaps +// (alphabetical keys) for structs so the output stays canonical. +func protoValueToJSON(v *types.Value) any { + switch k := v.GetKind().(type) { + case *types.Value_NullValue: + return nil + case *types.Value_NumberValue: + return k.NumberValue + case *types.Value_StringValue: + return k.StringValue + case *types.Value_BoolValue: + return k.BoolValue + case *types.Value_ListValue: + out := make([]any, 0, len(k.ListValue.Values)) + for _, e := range k.ListValue.Values { + out = append(out, protoValueToJSON(e)) + } + return out + case *types.Value_StructValue: + return protoStructToJSON(k.StructValue) + } + return nil +} + +func protoStructToJSON(s *types.Struct) *omap { + m := &omap{} + if s == nil { + return m + } + keys := make([]string, 0, len(s.Fields)) + for k := range s.Fields { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + m.set(k, protoValueToJSON(s.Fields[k])) + } + return m +} + +// jsonToProtoValue converts decoded JSON (float64 numbers or json.Number) +// into a types.Value tree. +func jsonToProtoValue(v any) *types.Value { + switch x := v.(type) { + case nil: + return &types.Value{Kind: &types.Value_NullValue{}} + case bool: + return &types.Value{Kind: &types.Value_BoolValue{BoolValue: x}} + case float64: + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: x}} + case json.Number: + f, err := x.Float64() + if err != nil { + return &types.Value{Kind: &types.Value_NullValue{}} + } + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: f}} + case string: + return &types.Value{Kind: &types.Value_StringValue{StringValue: x}} + case []any: + vals := make([]*types.Value, 0, len(x)) + for _, e := range x { + vals = append(vals, jsonToProtoValue(e)) + } + return &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: vals}}} + case map[string]any: + return &types.Value{Kind: &types.Value_StructValue{StructValue: jsonMapToProtoStruct(x)}} + } + return &types.Value{Kind: &types.Value_NullValue{}} +} + +func jsonMapToProtoStruct(m map[string]any) *types.Struct { + s := &types.Struct{Fields: make(map[string]*types.Value, len(m))} + for k, v := range m { + s.Fields[k] = jsonToProtoValue(v) + } + return s +} + +// +// ---- dates ---- +// + +// The range of unix seconds RFC 3339 can represent: its year is four digits, +// so a timestamp outside years 0000–9999 has no form in it. The bound is not a +// matter of taste — outside it, Format produces a string (`57482-01-22…`, +// `-0044-03-15…`) that parseDate cannot read back, so a caller that writes one +// anyway has silently changed the value's type on the way home. +// +// It is reachable from ordinary data: a millisecond timestamp stored where +// seconds belong (1751791445000) lands in year 57482, and that mistake is +// common enough to be a corruption class rather than a curiosity. +var ( + minDateSec = time.Date(0, time.January, 1, 0, 0, 0, 0, time.UTC).Unix() + maxDateSec = time.Date(9999, time.December, 31, 23, 59, 59, 0, time.UTC).Unix() +) + +// formatDate renders unix seconds in the full UTC RFC 3339 form (§3), +// reporting false when the value has no representation there. Callers must +// handle false rather than write the string anyway: parseDate cannot read it +// back, and the round trip would turn a date into a string. +func formatDate(sec int64) (string, bool) { + if sec < minDateSec || sec > maxDateSec { + return "", false + } + return time.Unix(sec, 0).UTC().Format(time.RFC3339), true +} + +// formatDateValue is formatDate for a stored property value, which is a +// float64. It range-checks before converting: a float too large for an int64 +// converts to an implementation-defined value in Go, so checking after would +// be checking the wrong number. +func formatDateValue(f float64) (string, bool) { + if math.IsNaN(f) || f < float64(minDateSec) || f > float64(maxDateSec) { + return "", false + } + return formatDate(int64(f)) +} + +// parseDate accepts RFC 3339 (with offsets and fractional seconds truncated) +// and date-only strings (UTC midnight), per §3. +func parseDate(s string) (int64, bool) { + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02"} { + if t, err := time.Parse(layout, s); err == nil { + return t.Unix(), true + } + } + return 0, false +} + +// +// ---- ids ---- +// + +// maxIdLen bounds every id this format writes: the block charset (§4) and the +// table inner charset (§6.1) both stop at 64 characters. +const maxIdLen = 64 + +// sanitizeBlockId maps a stored id onto the schema's block charset +// [A-Za-z0-9_-]{1,64}. Stored ids are not required to match it — legacy +// accounts hold dots and slashes, and a caller's GenerateId may derive ids from +// file paths — and writing one verbatim produces a document Validate rejects. +// Every replacement is ASCII, so the length bound can be applied to bytes. +func sanitizeBlockId(s string) string { + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + if isLabelRune(r) || r == '-' { + b.WriteRune(r) + } else { + b.WriteByte('_') + } + } + out := b.String() + if out == "" { + out = "b" + } + if len(out) > maxIdLen { + out = out[:maxIdLen] + } + return out +} + +// uniqueLabel returns base, or base with a "_2", "_3", … suffix — whichever is +// the first one taken() rejects. The result stays inside maxIdLen, so +// disambiguating never pushes an id past the charset bound. +func uniqueLabel(base string, taken func(string) bool) string { + if !taken(base) { + return base + } + for n := 2; ; n++ { + suffix := "_" + strconv.Itoa(n) + trimmed := base + if len(trimmed)+len(suffix) > maxIdLen { + trimmed = trimmed[:maxIdLen-len(suffix)] + } + if candidate := trimmed + suffix; !taken(candidate) { + return candidate + } + } +} + +// isValidTableInnerId reports whether s matches the schema's tableInnerId +// pattern ^[A-Za-z0-9_]{1,64}$ — the local-label charset, which excludes '-' +// because that separates a derived cell id (§6.1). +func isValidTableInnerId(s string) bool { + return !isInvalidLocalLabel(s) +} + +// defaultGenerateId mints ids shaped like the editor's (24 hex chars). +func defaultGenerateId() string { + var b [12]byte + if _, err := rand.Read(b[:]); err != nil { + panic(fmt.Errorf("random id: %w", err)) + } + return hex.EncodeToString(b[:]) +} + +// mintedSuffixLabels is the doc-local relabeler (§9a): it labels an id with +// its last size characters ONLY when the id matches a machine-minted shape +// (isMintedLocalId). The rule is deliberately inverted from "relabel unless +// the label charset is dirty": a non-opaque id usually carries meaning — +// `dataview` is a documented constant, `title`/`header`/`featuredRelations` +// are structural, imported documents carry human-readable ids — and +// relabeling those destroys information for zero benefit, while a false +// negative on a minted id merely costs a few tokens. +// +// The census counts EVERY id — the ids that never relabel included, with an +// id no longer than size counting as itself — so a label can neither equal +// another served id nor be an ambiguous suffix of one. disallow rejects +// candidates the caller reserves on top of that (every id that stays full, +// via the fullIds avoid-set, plus charset rules). +func mintedSuffixLabels(ids []string, size int, disallow func(candidate string) bool) map[string]string { + counts := make(map[string]int, len(ids)) + for _, id := range ids { + if r := []rune(id); len(r) > size { + counts[string(r[len(r)-size:])]++ + } else { + counts[id]++ + } + } + out := make(map[string]string, len(ids)) + for _, id := range ids { + if !isMintedLocalId(id) { + continue + } + suffix := id[len(id)-size:] // minted shapes are ASCII and longer than size + if counts[suffix] == 1 && (disallow == nil || !disallow(suffix)) { + out[id] = suffix + } + } + return out +} + +// IsCompactLabelShaped reports whether s has the exact shape of a served +// compact label: compactIdMinLen lowercase-hex characters. Every label the +// relabeler mints is the 5-char hex tail of a minted id (isMintedLocalId), +// so this is the serving layer's cheap tell for "this id probably came off a +// default read" where no owned-id baseline exists to check against. +func IsCompactLabelShaped(s string) bool { + return isHexLower(s, compactIdMinLen) +} + +// isMintedLocalId recognises the machine-minted doc-local id shapes — the +// only ids relabeling may touch. Worked out from the actual minting sites: +// +// - 24-char lowercase hex: bson.NewObjectId().Hex() — every editor-minted +// block, table-row and table-column id (core/block/simple, the table +// editor) — and this package's own defaultGenerateId (12 random bytes). +// - RFC-4122 UUID (8-4-4-4-12 lowercase hex): uuid.New().String() — +// dataview view ids. +// +// Derived cell ids (`rowId-colId`) are reserved by the census even though a +// cell carries no id in the flat form: a cell's suffix IS its column's, so +// unless both are counted the column wins the bucket alone and compacts to a +// label its own cells share in the live object. Anything unrecognised stays full: a false +// negative costs a few tokens, a false positive destroys a meaningful +// identifier. +func isMintedLocalId(id string) bool { + return isHexLower(id, 24) || isUuidShaped(id) +} + +// isHexLower reports whether s is exactly n lowercase-hex characters. +func isHexLower(s string, n int) bool { + if len(s) != n { + return false + } + for i := 0; i < n; i++ { + c := s[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +// isUuidShaped reports the 8-4-4-4-12 lowercase-hex UUID shape. +func isUuidShaped(s string) bool { + if len(s) != 36 { + return false + } + for i := 0; i < 36; i++ { + c := s[i] + switch i { + case 8, 13, 18, 23: + if c != '-' { + return false + } + default: + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + } + return true +} diff --git a/pkg/lib/anyblockjson/keycandidates_test.go b/pkg/lib/anyblockjson/keycandidates_test.go new file mode 100644 index 0000000000..f6f5a5fe19 --- /dev/null +++ b/pkg/lib/anyblockjson/keycandidates_test.go @@ -0,0 +1,182 @@ +package anyblockjson + +// keycandidates_test.go — what the importer does with the two LISTS a +// space-backed vocabulary hands it: the candidates for a spelling, and the +// declared type's own property keys. Both are read as counts — "how many live +// entities answer to this spelling", "does the type single one of them out" — +// and a count is exactly the thing a bookkeeping slip in the producer can +// falsify without corrupting anything else. The vocabulary here repeats every +// entry on purpose, because the shipped vocabulary's own guard against that +// (storeresolver's addClaimant) is one append away from being lost again and +// ScopedKeyVocabulary is a public interface Options.Keys accepts from anyone. +// +// The second half pins WHERE an ambiguity is reported. The refusal used to +// name `/property_internal_keys`, a member that is absent in precisely the +// documents that trigger it — the missing legend is the fix being asked for, +// so pointing at it tells a reader to look at nothing. + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" +) + +// dupVocab is nameVocab with every list answer doubled — one entity reported +// twice, in both namespaces and in the type's property scope. +type dupVocab struct{ nameVocab } + +func doubled(keys []string) []string { + out := make([]string, 0, len(keys)*2) + for _, key := range keys { + out = append(out, key, key) + } + return out +} + +func (v dupVocab) PropertyKeyCandidates(spelling string) []string { + return doubled(v.nameVocab.PropertyKeyCandidates(spelling)) +} + +func (v dupVocab) TypeKeyCandidates(spelling string) []string { + return doubled(v.nameVocab.TypeKeyCandidates(spelling)) +} + +func (v dupVocab) TypePropertyKeys(typeKey string) []string { + return doubled(v.nameVocab.TypePropertyKeys(typeKey)) +} + +// A repeated candidate is one entity, not two. Every arm here would round-trip +// under a vocabulary that counts correctly; the point is that the importer +// must not turn a producer's slip into a refused document, because a refusal +// is the one outcome the reader cannot recover from — it has no row count to +// compare the list against. +func TestKeyCandidates_ARepeatedCandidateIsNotAnAmbiguity(t *testing.T) { + const ( + keyA = "6a7663db61fab21cd4b9c001" + keyB = "6a7663db61fab21cd4b9c002" + typeKey = "6a7663db61fab21cd4b9c003" + typeName = "Sprint" + ) + + t.Run("a property listed twice still binds its value", func(t *testing.T) { + // given + vocab := dupVocab{nameVocab{names: map[string]string{keyA: "Projects"}}} + doc := `{"version":2,"id":"o1","properties":{"Projects":"kept"}}` + + // when + _, back, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + + // then + require.NoError(t, err, "one entity named twice is still one entity") + assert.Equal(t, "kept", back.Details.Fields[keyA].GetStringValue()) + }) + + t.Run("a type listed twice still binds the envelope", func(t *testing.T) { + // given + vocab := dupVocab{nameVocab{typeNames: map[string]string{typeKey: typeName}}} + doc := `{"version":2,"id":"o1","type":"` + typeName + `"}` + want := []string{domain.TypeKey(typeKey).URL()} + + // when + _, back, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + + // then + require.NoError(t, err, "the type namespace has no wider scope to recover in") + assert.Equal(t, want, back.ObjectTypes) + }) + + t.Run("a genuinely shared name is still resolved inside a doubled scope", func(t *testing.T) { + // given — two live properties really do bear the name, and the type + // names its own one TWICE: the intersection has to be counted by + // distinct key, or the type stops being able to single out the + // property it declares + vocab := dupVocab{nameVocab{ + names: map[string]string{keyA: "Projects", keyB: "Projects"}, + typeNames: map[string]string{typeKey: typeName}, + typeProps: map[string][]string{typeKey: {keyA}}, + }} + doc := `{"version":2,"id":"o1","type":"` + typeName + `","properties":{"Projects":"resolved"}}` + + // when + _, back, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + + // then + require.NoError(t, err) + assert.Equal(t, "resolved", back.Details.Fields[keyA].GetStringValue()) + assert.Nil(t, back.Details.Fields[keyB], "the scope picked one, and only one") + }) +} + +// Where the loud refusal points. The message asks for a legend entry, which +// is right — the legend is what settles a shared name — but the POINTER has +// to name the slot that spelled the term, because that is the only place in +// the document a reader can go and look. +func TestKeyCandidates_TheAmbiguityNamesTheOffendingSlot(t *testing.T) { + const ( + keyA = "6a7663db61fab21cd4b9c011" + keyB = "6a7663db61fab21cd4b9c022" + ) + shared := nameVocab{names: map[string]string{keyA: "Projects", keyB: "Projects"}} + + refusal := func(t *testing.T, doc string, vocab nameVocab) Issue { + t.Helper() + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + require.Error(t, err) + var invalid *ValidationError + require.True(t, errors.As(err, &invalid), "a refusal, not a decode failure: %v", err) + require.Len(t, invalid.Issues, 1) + return invalid.Issues[0] + } + + t.Run("a properties member names its own slot", func(t *testing.T) { + // given — no legend, which is what makes the term ambiguous in the + // first place, so pointing at the legend points at nothing + doc := `{"version":2,"id":"o1","type":"Page","properties":{"Projects":"?"}}` + want := "/properties/Projects" + + // when + got := refusal(t, doc, shared) + + // then + assert.Equal(t, want, got.Path) + assert.Contains(t, got.Message, `"Projects"`) + assert.Contains(t, got.Message, memberPropertyInternalKeys, + "the message still asks for the legend — that is the repair") + }) + + t.Run("a block slot names the block section", func(t *testing.T) { + // given — these slots are built without a pointer of their own, so + // the coarse-but-true section is the honest answer, exactly as the + // empty-key refusal beside it already reports + doc := `{"version":2,"id":"o1","type":"Page","blocks":[{"id":"dv","type":"dataview",` + + `"properties":[{"property":"Projects","format":"text"}]}]}` + want := "/blocks" + + // when + got := refusal(t, doc, shared) + + // then + assert.Equal(t, want, got.Path) + }) + + t.Run("the type envelope names /type", func(t *testing.T) { + // given + sharedTypes := nameVocab{typeNames: map[string]string{ + "6a7663db61fab21cd4b9c077": "Meeting", + "6a7663db61fab21cd4b9c088": "Meeting", + }} + doc := `{"version":2,"id":"o1","type":"Meeting"}` + want := "/type" + + // when + got := refusal(t, doc, sharedTypes) + + // then + assert.Equal(t, want, got.Path, "the type namespace has carried its slot pointer all along") + assert.Contains(t, got.Message, memberTypeInternalKeys) + }) +} diff --git a/pkg/lib/anyblockjson/keyslotadmission_test.go b/pkg/lib/anyblockjson/keyslotadmission_test.go new file mode 100644 index 0000000000..c08f352c8b --- /dev/null +++ b/pkg/lib/anyblockjson/keyslotadmission_test.go @@ -0,0 +1,311 @@ +package anyblockjson + +// keyslotadmission_test.go — a key slot has to name something, at every slot +// and through every door. +// +// Three slots enforced it and thirteen did not. `/properties`, +// `type_properties[].key` and `type_properties[].object_types[]` refused an +// empty key; the property block, a link block's `properties`, a dataview's +// `properties[].property` (spelled `key` then), `group_by`, `cover_property`, `end_property`, +// `columns[].property`, `sorts[].property`, `filters[].property`, the +// envelope `type` and `template_for` all took one — from a plain document, no +// vocabulary needed — and then LOST the slot on the way back out, in silence. +// A column and a sort vanish; a property block and a link's shown-property +// list come back nameless; a filter re-exports as a node that filters on +// nothing; and `"type": ""` costs the object its TYPE. +// +// Export had two matching leaks: a filter and a property block whose stored +// key was empty were written as nameless nodes, where the sort and the column +// beside them have always been dropped. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// keySlotCase is one slot, spelled twice: once with a real key, once with the +// empty one. The `named` half is the control — without it a fixture that is +// malformed for some unrelated reason refuses for the wrong reason and the +// test says nothing. +type keySlotCase struct { + slot string + named string + empty string + path string +} + +func keySlotCases() []keySlotCase { + dv := func(inner string) string { + return `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview",` + inner + `}]}` + } + view := func(inner string) string { + return dv(`"views":[{"id":"v1","type":"table",` + inner + `}]`) + } + return []keySlotCase{ + {"envelope type", `{"version":2,"id":"o1","type":"page"}`, + `{"version":2,"id":"o1","type":""}`, "/type"}, + {"template_for", + `{"version":2,"kind":"template","id":"o1","type":"template","template_for":"page"}`, + `{"version":2,"kind":"template","id":"o1","type":"template","template_for":""}`, + "/template_for"}, + {"type_properties key", + `{"version":2,"kind":"object_type","id":"o1","type":"object_type","type_settings":{"property_definitions": [{"property":"prio","format":"text"}]}}`, + `{"version":2,"kind":"object_type","id":"o1","type":"object_type","type_settings":{"property_definitions": [{"property":"","format":"text"}]}}`, + "/type_settings/property_definitions/0/property"}, + {"type_properties object_types", + `{"version":2,"kind":"object_type","id":"o1","type":"object_type","type_settings":{"property_definitions": [{"property":"who","format":"objects","object_types":["page"]}]}}`, + `{"version":2,"kind":"object_type","id":"o1","type":"object_type","type_settings":{"property_definitions": [{"property":"who","format":"objects","object_types":[""]}]}}`, + "/type_settings/property_definitions/0/object_types/0"}, + {"relation object_types", + `{"version":2,"kind":"property","id":"o1","internal_key":"who","property_settings":{"format":"objects","object_types":["page"]}}`, + `{"version":2,"kind":"property","id":"o1","internal_key":"who","property_settings":{"format":"objects","object_types":[""]}}`, + "/property_settings/object_types/0"}, + {"properties member", `{"version":2,"id":"o1","properties":{"prio":"x"}}`, + `{"version":2,"id":"o1","properties":{"":"x"}}`, "/properties/"}, + {"property block key", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"property","property":"prio"}]}`, + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"property","property":""}]}`, + "/blocks/0/property"}, + {"link block properties", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"link","object_id":"t1","properties":["prio"]}]}`, + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"link","object_id":"t1","properties":[""]}]}`, + "/blocks/0/properties/0"}, + {"dataview properties key", + dv(`"properties":[{"property":"prio","format":"text"}],"views":[{"id":"v1","type":"table"}]`), + dv(`"properties":[{"property":"","format":"text"}],"views":[{"id":"v1","type":"table"}]`), + "/blocks/0/properties/0/property"}, + {"view group_by", + dv(`"views":[{"id":"v1","type":"kanban","group_by":"tag"}]`), + dv(`"views":[{"id":"v1","type":"kanban","group_by":""}]`), + "/blocks/0/views/0/group_by"}, + {"view cover_property", + dv(`"views":[{"id":"v1","type":"gallery","cover_property":"prio"}]`), + dv(`"views":[{"id":"v1","type":"gallery","cover_property":""}]`), + "/blocks/0/views/0/cover_property"}, + {"view end_property", + dv(`"views":[{"id":"v1","type":"calendar","end_property":"prio"}]`), + dv(`"views":[{"id":"v1","type":"calendar","end_property":""}]`), + "/blocks/0/views/0/end_property"}, + {"view column property", view(`"columns":[{"property":"prio"}]`), + view(`"columns":[{"property":""}]`), "/blocks/0/views/0/columns/0/property"}, + {"sort property", view(`"sorts":[{"property":"prio","direction":"asc"}]`), + view(`"sorts":[{"property":"","direction":"asc"}]`), "/blocks/0/views/0/sorts/0/property"}, + {"filter property", view(`"filters":[{"property":"prio","condition":"equal","value":"x"}]`), + view(`"filters":[{"property":"","condition":"equal","value":"x"}]`), + "/blocks/0/views/0/filters/0/property"}, + {"nested filter property", + view(`"filters":[{"operator":"or","filters":[{"property":"prio","condition":"equal","value":"x"}]}]`), + view(`"filters":[{"operator":"or","filters":[{"property":"","condition":"equal","value":"x"}]}]`), + "/blocks/0/views/0/filters/0/filters/0/property"}, + } +} + +// The document door: every key slot refuses the empty spelling, and the same +// document with a real spelling is accepted — so the refusal is about the +// key, not about the fixture. +func TestValidate_EveryKeySlotRefusesTheEmptySpelling(t *testing.T) { + for _, tc := range keySlotCases() { + t.Run(tc.slot, func(t *testing.T) { + require.NoError(t, Validate([]byte(tc.named)), + "the control must be a document this format accepts:\n%s", tc.named) + + err := Validate([]byte(tc.empty)) + require.Error(t, err, "accepted an empty key at %s:\n%s", tc.slot, tc.empty) + assert.Contains(t, issuePaths(t, err), tc.path, + "the refusal has to name the slot it is about (§12): %v", err) + + // and Unmarshal agrees with Validate (§12): what one refuses the + // other refuses, which is the half that used to fail — the + // document imported clean and the slot was simply gone + _, _, err = Unmarshal([]byte(tc.empty), Options{GenerateId: seqIds("g")}) + assert.Error(t, err, "Unmarshal accepted what Validate refuses") + }) + } +} + +// A filter with no `property` member at all — the shape export used to write +// for a view whose relation was deleted. It carried the same meaning as the +// empty spelling and had the same silence. +func TestValidate_AFilterHasToNameItsProperty(t *testing.T) { + doc := `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","views":[` + + `{"id":"v1","type":"table","filters":[{"condition":"equal","value":"x"}]}]}]}` + + err := Validate([]byte(doc)) + require.Error(t, err, "a filter that filters on nothing:\n%s", doc) + var ve *ValidationError + require.ErrorAs(t, err, &ve) + assert.Equal(t, "/blocks/0/views/0/filters/0", ve.Issues[0].Path, + "the FIRST issue is the one an agent acts on (§12): %v", err) + assert.Contains(t, ve.Issues[0].Message, "a filter has to name the property it filters on") + + _, _, err = Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + assert.Error(t, err) +} + +// The resolution door, which needs no document fault at all: a vocabulary +// answering ("", true) — accepted from anyone (§3) — used to land the empty +// key in the model at nine of these slots. +func TestUnmarshal_EveryKeySlotRefusesAnEmptyResolution(t *testing.T) { + // `want` is the substring the refusal has to carry. The four slots that + // already refused name the SLOT (their pointer is exact); the ten this + // change reaches name the SPELLING, because the fault is the reader's + // vocabulary and their pointer can only be the coarse `/blocks`. The four + // are here to state the uniformity, not because they are load-bearing — + // reverting the change leaves them green and fails the other ten. + for _, tc := range []struct{ slot, doc, want string }{ + {"envelope type", `{"version":2,"id":"o1","type":"prio"}`, + "/type: resolved type key is empty"}, + {"template_for", + `{"version":2,"kind":"template","id":"o1","type":"template","template_for":"prio"}`, + "/template_for: resolved type key is empty"}, + {"type_properties key", + `{"version":2,"kind":"object_type","id":"o1","type":"object_type","type_settings":{"property_definitions": [{"property":"prio","format":"text"}]}}`, + "/type_settings/property_definitions/0/property: resolved property key is empty"}, + {"type_properties object_types", + `{"version":2,"kind":"object_type","id":"o1","type":"object_type","type_settings":{"property_definitions": [{"property":"who","format":"objects","object_types":["prio"]}]}}`, + "/type_settings/property_definitions/0/object_types/0: resolved type key is empty"}, + {"relation object_types", + `{"version":2,"kind":"property","id":"o1","internal_key":"who","property_settings":{"format":"objects","object_types":["prio"]}}`, + "/property_settings/object_types/0: resolved type key is empty"}, + {"properties member", `{"version":2,"id":"o1","properties":{"prio":"x"}}`, + "/properties/prio: resolved property key is empty"}, + {"property block key", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"property","property":"prio"}]}`, + "the property block `property` spelling \"prio\""}, + {"link block properties", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"link","object_id":"t1","properties":["prio"]}]}`, + "the link block `properties` spelling \"prio\""}, + {"dataview properties key", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","properties":[{"property":"prio","format":"text"}],"views":[{"id":"v1","type":"table"}]}]}`, + "the dataview `properties` spelling \"prio\""}, + {"view group_by", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","views":[{"id":"v1","type":"kanban","group_by":"prio"}]}]}`, + "the view `group_by` spelling \"prio\""}, + {"view cover_property", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","views":[{"id":"v1","type":"gallery","cover_property":"prio"}]}]}`, + "the view `cover_property` spelling \"prio\""}, + {"view end_property", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","views":[{"id":"v1","type":"calendar","end_property":"prio"}]}]}`, + "the view `end_property` spelling \"prio\""}, + {"view column property", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","views":[{"id":"v1","type":"table","columns":[{"property":"prio"}]}]}]}`, + "the view column `property` spelling \"prio\""}, + {"sort property", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","views":[{"id":"v1","type":"table","sorts":[{"property":"prio","direction":"asc"}]}]}]}`, + "the sort `property` spelling \"prio\""}, + {"filter property", + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","views":[{"id":"v1","type":"table","filters":[{"property":"prio","condition":"equal","value":"x"}]}]}]}`, + "the filter `property` spelling \"prio\""}, + } { + t.Run(tc.slot, func(t *testing.T) { + // the document itself is fine — Validate, which takes no + // vocabulary, accepts it. The fault is entirely in the reader. + require.NoError(t, Validate([]byte(tc.doc)), "%s", tc.doc) + + _, snap, err := Unmarshal([]byte(tc.doc), + Options{GenerateId: seqIds("g"), Keys: emptyAnswerVocabulary{}}) + require.Error(t, err, + "the empty key landed in the model at %s; the slot is lost on the way back out", tc.slot) + assert.Nil(t, snap, "a refused document hands back no object") + assert.Contains(t, err.Error(), "a key slot has to name something") + assert.Contains(t, err.Error(), tc.want, + "the refusal has to say which slot, or which spelling, it is about") + }) + } +} + +// emptyAnswerVocabulary answers "this spelling is a slug for the relation ”" +// — the shape a buggy or half-built resolver produces, and one nothing in +// KeyVocabulary's preconditions forbids. +type emptyAnswerVocabulary struct{ BundledKeyVocabulary } + +func (emptyAnswerVocabulary) PropertyKey(slug string) (string, bool) { + if slug == "prio" { + return "", true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (emptyAnswerVocabulary) TypeKey(slug string) (string, bool) { + if slug == "prio" { + return "", true + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// The export door. A stored relation key can be empty — that is real data — +// and export wrote a filter and a property block that named nothing, which +// the rule above now refuses. Both are dropped, with a warning, exactly as +// the sort and the column beside them already were. +func TestExport_ANamelessFilterAndPropertyBlockAreDropped(t *testing.T) { + // given — one dataview view holding a named filter and a nameless one, + // plus a nameless property block. The named filter is the control: the + // view must survive, so this cannot pass by dropping everything. + dv := &model.BlockContentDataview{ + RelationLinks: []*model.RelationLink{{Key: "tag", Format: model.RelationFormat_tag}}, + Views: []*model.BlockContentDataviewView{{ + Id: "v1", Name: "All", Type: model.BlockContentDataviewView_Table, + Filters: []*model.BlockContentDataviewFilter{ + {Id: "f1", RelationKey: "tag", + Condition: model.BlockContentDataviewFilter_Equal, Value: str("x")}, + {Id: "f2", RelationKey: "", + Condition: model.BlockContentDataviewFilter_Equal, Value: str("y")}, + }, + }}, + } + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "o1", ChildrenIds: []string{"dv1", "rel1", "rel2"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: dv}}, + {Id: "rel1", Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{Key: "tag"}}}, + {Id: "rel2", Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{Key: ""}}}, + }, + Details: fields(map[string]*types.Value{"id": str("o1"), "name": str("Board")}), + } + var warned []Issue + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, + Options{OnWarning: func(i Issue) { warned = append(warned, i) }}) + + // then — I1: what Marshal writes, its own Validate and Unmarshal accept + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "emitted:\n%s", data) + + // the named ones survive, the nameless ones are gone and said so + view := backView(t, back) + require.Len(t, view.Filters, 1, "emitted:\n%s", data) + assert.Equal(t, "tag", view.Filters[0].RelationKey) + var blockKeys []string + for _, b := range back.Blocks { + if c, ok := b.Content.(*model.BlockContentOfRelation); ok { + blockKeys = append(blockKeys, c.Relation.Key) + } + } + assert.Equal(t, []string{"tag"}, blockKeys) + joined := warningsAt(warned, "") + assert.Contains(t, joined, "a filter names no property and is dropped") + assert.Contains(t, joined, "a property block names no property and is dropped") +} + +// issuePaths lists the pointers a ValidationError names. +func issuePaths(t *testing.T, err error) []string { + t.Helper() + var ve *ValidationError + require.ErrorAs(t, err, &ve) + out := make([]string, 0, len(ve.Issues)) + for _, i := range ve.Issues { + out = append(out, i.Path) + } + return out +} diff --git a/pkg/lib/anyblockjson/keyslotbounds_test.go b/pkg/lib/anyblockjson/keyslotbounds_test.go new file mode 100644 index 0000000000..38b9179904 --- /dev/null +++ b/pkg/lib/anyblockjson/keyslotbounds_test.go @@ -0,0 +1,231 @@ +package anyblockjson + +// keyslotbounds_test.go — the writable-key rule (§3: non-empty, +// control-character-free, inside the 128-character bound) at every PROPERTY +// key slot outside /properties, through all three doors. +// +// $defs/propertyDefinition carried the bound and the pattern all along; the +// sibling slots — a dataview's properties[], a view's +// group_by/cover_property/end_property, columns, sorts, filters, a link +// block's properties[], the property block — carried only minLength, so a +// million-character key and raw NUL/CR/ESC bytes validated clean, imported +// clean, and persisted into RelationLink.Key. Export emitted such stored +// keys verbatim into the same slots, so the schema half and the export half +// land in one change: a schema-only bound made Marshal emit what its own +// Validate rejects (§11, I1) — that was tried and reverted. + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// hostileKeys are stored-key strings no spelling can carry. Built from runes +// so no source literal has to hold a raw control byte. +func hostileKeys() map[string]string { + return map[string]string{ + "an over-long key": strings.Repeat("k", maxPropertyKeyLen+1), + "a control-character key": "bad" + string(rune(0x07)) + "key", + "a NUL-and-ESC key": "bad" + string(rune(0x00)) + "key" + string(rune(0x1b)), + // DEL slips past the schema's pattern (its lower-plane character + // class stops at 0x1f, matching $defs/propertyDefinition) — the + // restated writable-key rule is what has to catch it, exactly as it + // does for /properties and the definition slots + "a DEL-carrying key": "bad" + string(rune(0x7f)) + "key", + } +} + +// The document door: every property-key slot refuses an over-long and a +// control-character spelling, path-addressed, and Unmarshal agrees. +func TestValidate_EveryKeySlotBoundsTheSpelling(t *testing.T) { + for _, tc := range keySlotCases() { + switch tc.slot { + case "envelope type", "template_for", "type_properties object_types", "relation object_types": + continue // type-key slots: any non-empty stored key round-trips (§3) + case "properties member", "type_properties key": + continue // bounded before this change; the seven sibling slots are the point + } + for name, hostile := range hostileKeys() { + t.Run(tc.slot+" refuses "+name, func(t *testing.T) { + // given — the good fixture with its key swapped for the + // hostile one, JSON-escaped the way a document would carry it + enc, err := json.Marshal(hostile) + require.NoError(t, err) + doc := strings.ReplaceAll(tc.named, `"prio"`, string(enc)) + doc = strings.ReplaceAll(doc, `"tag"`, string(enc)) + require.NotEqual(t, tc.named, doc, + "the fixture must actually carry the hostile key") + require.NoError(t, Validate([]byte(tc.named)), + "the control must be a document this format accepts:\n%s", tc.named) + + // when + err = Validate([]byte(doc)) + + // then + require.Error(t, err, "accepted %s at %s", name, tc.slot) + assert.Contains(t, issuePaths(t, err), tc.path, + "the refusal has to name the slot it is about (§12): %v", err) + + // and Unmarshal agrees with Validate (§12) + _, _, err = Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + assert.Error(t, err, "Unmarshal accepted what Validate refuses") + }) + } + } +} + +// The resolution door: a vocabulary resolving a legal spelling onto an +// unwritable stored key used to land it in RelationLink.Key at the block +// slots, where /properties has refused it all along. +func TestUnmarshal_KeySlotsRefuseAnUnwritableResolution(t *testing.T) { + // given + doc := `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview",` + + `"views":[{"id":"v1","type":"table","sorts":[{"property":"prio","direction":"asc"}]}]}]}` + require.NoError(t, Validate([]byte(doc)), + "the document itself is fine; the fault is the reader's vocabulary") + + // when + _, snap, err := Unmarshal([]byte(doc), + Options{GenerateId: seqIds("g"), Keys: unwritableAnswerVocabulary{}}) + + // then + require.Error(t, err, "the unwritable key landed in the model; export can only drop it later") + assert.Nil(t, snap, "a refused document hands back no object") + assert.Contains(t, err.Error(), "carries a control character") +} + +// unwritableAnswerVocabulary resolves the spelling "prio" onto a stored key +// carrying a control byte — a shape nothing in KeyVocabulary's preconditions +// forbids. +type unwritableAnswerVocabulary struct{ BundledKeyVocabulary } + +func (unwritableAnswerVocabulary) PropertyKey(slug string) (string, bool) { + if slug == "prio" { + return "bad" + string(rune(0x00)) + "key", true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +// The export door: a stored key no spelling can carry is dropped from every +// reference slot with a warning, the way the empty key already is, instead +// of being emitted verbatim into a document Marshal's own Validate rejects. +func TestExport_UnwritableKeysAreDroppedFromEveryReferenceSlot(t *testing.T) { + // given — every slot holds one good key (the control: the view must + // survive, so this cannot pass by dropping everything) and one hostile + hostile := "bad" + string(rune(0x00)) + "key" + string(rune(0x1b)) + overlong := strings.Repeat("k", maxPropertyKeyLen+1) + dv := &model.BlockContentDataview{ + RelationLinks: []*model.RelationLink{ + {Key: "tag", Format: model.RelationFormat_tag}, + {Key: hostile, Format: model.RelationFormat_longtext}, + {Key: overlong, Format: model.RelationFormat_longtext}, + }, + Views: []*model.BlockContentDataviewView{{ + Id: "v1", Name: "All", Type: model.BlockContentDataviewView_Kanban, + GroupRelationKey: hostile, + CoverRelationKey: overlong, + EndRelationKey: hostile, + Sorts: []*model.BlockContentDataviewSort{ + {Id: "s1", RelationKey: "tag"}, + {Id: "s2", RelationKey: hostile}, + }, + Filters: []*model.BlockContentDataviewFilter{ + {Id: "f1", RelationKey: "tag", + Condition: model.BlockContentDataviewFilter_Equal, Value: str("x")}, + {Id: "f2", RelationKey: overlong, + Condition: model.BlockContentDataviewFilter_Equal, Value: str("y")}, + }, + Relations: []*model.BlockContentDataviewRelation{ + {Key: "tag"}, + {Key: hostile}, + }, + }}, + } + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "o1", ChildrenIds: []string{"dv1", "l1", "rel1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: dv}}, + {Id: "l1", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: "t1", Relations: []string{"tag", hostile}}}}, + {Id: "rel1", Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{Key: hostile}}}, + }, + Details: fields(map[string]*types.Value{"id": str("o1"), "name": str("Board")}), + } + var warned []Issue + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, + Options{OnWarning: func(i Issue) { warned = append(warned, i) }}) + + // then — I1: what Marshal writes, its own Validate and Unmarshal accept + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "emitted:\n%s", data) + assert.NotContains(t, string(data), overlong, + "the over-long key must not be emitted anywhere") + + // the named slots survive; the unwritable ones are gone and said so + view := backView(t, back) + require.Len(t, view.Sorts, 1, "emitted:\n%s", data) + assert.Equal(t, "tag", view.Sorts[0].RelationKey) + require.Len(t, view.Filters, 1) + assert.Equal(t, "tag", view.Filters[0].RelationKey) + require.Len(t, view.Relations, 1) + assert.Equal(t, "tag", view.Relations[0].Key) + assert.Empty(t, view.GroupRelationKey) + assert.Empty(t, view.CoverRelationKey) + assert.Empty(t, view.EndRelationKey) + for _, b := range back.Blocks { + switch c := b.Content.(type) { + case *model.BlockContentOfDataview: + for _, rl := range c.Dataview.RelationLinks { + assert.True(t, isWritablePropertyKey(rl.Key), + "an unwritable key persisted into RelationLink.Key: %q", rl.Key) + } + case *model.BlockContentOfLink: + assert.Equal(t, []string{"tag"}, c.Link.Relations) + case *model.BlockContentOfRelation: + t.Errorf("a property block naming an unwritable key survived: %q", c.Relation.Key) + } + } + assert.NotEmpty(t, warned, "every dropped slot owes a warning") +} + +// The bundle index's widget `properties[]` is the same key-slot family in a +// different grammar: the schema bounds it the same way, and the widget-object +// lift refuses an unwritable key the way it refuses an out-of-range limit — +// the document then travels whole, where the link block's own slot rule +// applies. +func TestIndex_WidgetPropertiesAreBoundedTheSameWay(t *testing.T) { + for name, hostile := range hostileKeys() { + if strings.Contains(name, "DEL") { + continue // the index grammar has no restatement pass; the schema's bound is its rule + } + t.Run("the index refuses "+name, func(t *testing.T) { + // given + enc, err := json.Marshal(hostile) + require.NoError(t, err) + doc := `{"version":2,"widgets":[{"target":"page-home","properties":[` + string(enc) + `]}]}` + control := `{"version":2,"widgets":[{"target":"page-home","properties":["Due date"]}]}` + _, err = UnmarshalIndex([]byte(control)) + require.NoError(t, err, "the control must be an index this format accepts") + + // when + _, err = UnmarshalIndex([]byte(doc)) + + // then + require.Error(t, err, "accepted %s at the index widget properties slot", name) + assert.Contains(t, issuePaths(t, err), "/widgets/0/properties/0", + "the refusal has to name the slot it is about (§12): %v", err) + }) + } +} diff --git a/pkg/lib/anyblockjson/keyvocab.go b/pkg/lib/anyblockjson/keyvocab.go new file mode 100644 index 0000000000..83539dfa82 --- /dev/null +++ b/pkg/lib/anyblockjson/keyvocab.go @@ -0,0 +1,363 @@ +package anyblockjson + +// keyvocab.go — the wire vocabulary for type and property keys. +// +// The format speaks ONE key vocabulary, the display NAME — NFC-normalized, +// otherwise verbatim — everywhere a type or property is named: envelope +// `type`/`template_for`, `properties` map keys, +// `property_definitions[].property`, dataview +// `properties[].property`/`group_by`/`cover_property`/`end_property`/sort +// and filter `property`/column `property`, the `property` block's +// `property`, and a link block's `properties`. `dueDate` is "Due date" on +// the wire; bundled, API-created and UI-created keys are indistinguishable +// to a reader, and there is no derived identifier anywhere in the format — +// the api slug stays the API surface's affair. +// +// **The reverse is a TABLE, both directions — never a string transform.** +// That is proven, not cautionary: "Creation date" says a different word +// than `createdDate`, so no derivation in either direction exists, and a +// spelling that IS the stored key (a shared bundled name has no wire form) +// must pass through rather than be "restored". +// TestKeyVocabulary_ReverseIsATableNotACaseTransform pins both. +// +// The DEFAULT vocabulary is the bundled name table (bundledname.go), which +// ships with every reader — so a document written by a full node still +// resolves its bundled keys in a package-only reader, offline, with no +// store (§3 chain step 3). A node-backed caller supplies a wider vocabulary +// that also knows the space's own names (chain step 2); v2 does, via +// storeresolver. + +import ( + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" +) + +// KeyVocabulary translates between the STORED keys the snapshot carries and +// the SPELLINGS the document writes — display names, under §3's rule — in +// both directions. The method names still say "slug" from the era when the +// spelling was a derived identifier; they are kept because the contract +// they name (spelling ↔ stored key) is unchanged, and every implementor +// would churn for a word. Implementations owe three things, and none of +// them is implied by the one before it. +// +// 1. **Inversion.** Whatever `…Slug` emits, `…Key` must invert, or a document +// does not round-trip. +// +// 2. **No shadowing of the bundled table.** No answer, in either direction, +// may bind a spelling that the bundled table binds to a DIFFERENT key: +// `…Slug(key)` must not return a slug the table inverts to something other +// than `key`, and `…Key(slug)` must not answer for a slug the table binds +// elsewhere. +// +// 3. **A live stored key outranks the vocabulary's own name binding.** This +// is §3 chain step 2 (verbatim-first) stated as an obligation on the +// implementation: when a term is the stored key of a live entity, +// `…Key(term)` must answer "not a spelling" (ok == false) even when some +// other live entity is NAMED by that exact string, and `…Slug(key)` must +// not emit a spelling that some live stored key answers to. +// storeresolver has implemented this from the start and the interface +// never said so — `keyMaps.key`'s `if m.storedKey[term] { return "", +// false }` is the accept half; the emit half is `grant`'s vetting of +// every label against the stored-key set plus `PropertySlug`'s +// bundled-arm re-check. +// Without it, a document naming a relation by its stored key lands on +// whichever OTHER relation happens to be named by that string — the +// document names one entity and the reader writes another, with no error +// — and the emit half labels a value with an address that resolves to +// somebody else's row. Two live entities can always be told apart by +// their stored keys; if the name layer is allowed to outrank them, +// nothing can. +// +// The second rule is what makes §11's round-trip guarantee true for a reader +// export never met, and a vocabulary can satisfy the first completely while +// breaking it. The legend is the reason: a document owes a +// `type_internal_keys`/`property_internal_keys` entry only for a spelling the READER's chain +// cannot invert, and export can only ask the chains it can see — the bundled +// table, which ships with every reader, and the vocabulary it is running +// under (recordTypeKey / termInverts; that second half was missing, and a +// conforming vocabulary lost a type because of it). A third reader's +// vocabulary is not one of them. So a stored key both visible chains invert — +// `task`, spelled `task` — is written with NO legend entry, and a reader whose +// vocabulary answers `TypeKey("task") == "69bbfc…"` resolves it through that +// answer instead. A template for the bundled Task type comes back as a +// template for an unrelated custom type, silently. The property namespace has +// the same shape: a bundled `description` reads back as whatever custom key +// claimed the spelling. +// +// storeresolver, the vocabulary the product wires, keeps both halves by +// construction — a name shared with the bundled table is answered as a +// CANDIDATE set (never bound to one holder), and the emit side yields to +// live stored keys — so this is a rule for hand-written implementations, +// which Options.Keys accepts from anyone. +// TestKeyVocabulary_ShadowingSlugBreaksInversion pins what happens when it +// is broken. +// +// What NO rule here can prevent, and the legend therefore must: the third +// rule lets a live name binding win once the stored key stops being live, +// which is what a UI delete does (storeresolver's corpse policy). A fully +// conforming vocabulary then binds a spelling that objects still carry as a +// stored key, so export writes the identity entry for it — +// TestKeyVocabulary_VocabularyInForceIsAReaderToo and +// TestCorpseStoredKeyStillNamesItsObjects. +// +// **What no rule above requires, and every shipped implementation does.** +// The three obligations are about correctness — a spelling that inverts, +// and inverts to the right key. What it LOOKS like is a separate question, +// and §3 answers it: a key is spelled by its display name, NFC and +// verbatim. PropertyLabel and TypeLabel are that rule, exported so a +// vocabulary can apply it to whatever its source of truth stores; +// storeresolver calls them, and an implementation that answers with a raw +// stored key is still correct, merely less readable. +type KeyVocabulary interface { + // PropertySlug is the wire spelling of a stored relation key. Returning + // the input unchanged is always valid ("no slug for this key"). + PropertySlug(key string) string + // PropertyKey inverts PropertySlug. ok is false when the term is not a + // known spelling — the caller then treats it as a stored key verbatim, + // which is the §3 verbatim-first rule (an exact stored key always wins + // over the name tables). + PropertyKey(slug string) (key string, ok bool) + + TypeSlug(key string) string + TypeKey(slug string) (key string, ok bool) +} + +// ScopedKeyVocabulary is the OPTIONAL capability a space-backed vocabulary +// adds beyond KeyVocabulary, discovered by type assertion (the TypeResolver +// pattern, §2d). It exists because raw-name addressing admits questions the +// four-method interface cannot ask: +// +// - **A shared name.** Two live properties may bear one name, and +// PropertyKey then refuses to answer (an ambiguous address is never +// resolved by guess). The importer, which knows the document's declared +// type, asks for the full candidate list and resolves WITHIN THE TYPE: +// a name unambiguous among the type's own properties — the overwhelming +// case, measured at 1 ambiguous type in 1,753 — is resolved; a name the +// type cannot place raises a loud error asking for the legend, never a +// phantom key. +// - **A term about to be stored verbatim.** The importer warns when a +// verbatim term is not any live entity's stored key (the +// stale-or-guessed-name phantom) and when it extends a live name with +// trailing text (the glued-annotation hazard). Both diagnoses need the +// space's stored-key set and name list, which only the vocabulary has. +// +// storeresolver implements it; the bundled-only default does not (bundled +// names are unique by CI guard, so neither question arises offline). +// +// **Every list this interface returns is a SET.** No key appears twice in a +// candidate list or in a type's property list. This is not tidiness: the +// importer reads these lists as COUNTS — "how many live entities answer to +// this spelling", "does the declared type single one of them out" — and a +// count is exactly what a bookkeeping slip in the implementation can falsify +// while leaving every key in the list correct. One entity listed twice reads +// as two, and the import REFUSES a document that the matching export had just +// written, with nothing in the document at fault and nothing in the list a +// reader could check the count against. A refusal is also the one outcome the +// reader cannot work around: a wrong resolution can be overridden with a +// legend entry, a refused import stops. The importer takes the distinct keys +// defensively (distinctKeys in import.go), because Options.Keys accepts an +// implementation from anyone, but an implementation still owes the set. +type ScopedKeyVocabulary interface { + // PropertyKeyCandidates returns every live property key whose exact + // document spelling is the term — the space's claimants plus the + // bundled table's binding — as a sorted set, no key twice. It says + // nothing about stored keys: verbatim-first is the caller's step, asked + // before this one. + PropertyKeyCandidates(spelling string) []string + // TypeKeyCandidates is the type namespace's half, under the same set and + // ordering contract. + TypeKeyCandidates(spelling string) []string + // TypePropertyKeys returns the stored property keys the type declares — + // the disambiguating scope for a shared property name. A set, in the + // type's own order: it is intersected with a candidate list and the + // survivors are counted, so a property the type names through two of its + // lists must still be counted once. + TypePropertyKeys(typeKey string) []string + // PropertyTermFacts / TypeTermFacts diagnose one term for the + // verbatim-resolution warnings. + PropertyTermFacts(term string) KeyTermFacts + TypeTermFacts(term string) KeyTermFacts +} + +// KeyTermFacts is what a space-backed vocabulary knows about one term that +// is about to resolve verbatim. +type KeyTermFacts struct { + // LiveStoredKey: the term is a live entity's stored key — verbatim + // resolution is then simply chain step 2, nothing to warn about. + LiveStoredKey bool + // ExtendsName: a live entity's display name the term extends with + // trailing text past a word boundary ("" when none) — the eval's one + // real raw-name failure shape, an annotation glued onto a copied name. + ExtendsName string +} + +// BundledKeyVocabulary is the package default: the bundled name table +// (bundledname.go), both directions, plus the forgiving fold on the accept +// side, and nothing else. Custom keys pass through unchanged: a +// package-only reader has no space to ask about names. +type BundledKeyVocabulary struct{} + +func (BundledKeyVocabulary) PropertySlug(key string) string { + if bundle.HasRelation(domain.RelationKey(key)) { + return bundledPropertySpelling(key) + } + return key +} + +func (BundledKeyVocabulary) PropertyKey(slug string) (string, bool) { + if key, ok := bundledPropertyKeyBySpelling(slug); ok { + return key, true + } + // the forgiving fold, single candidate only — the layer that keeps every + // pre-change derived-slug spelling (`created_date`) resolving in a + // package-only reader with no compatibility table: ToSnake only inserts + // `_` and lowercases, so the old slug sits in its stored key's fold class + if candidates := BundledPropertyKeysByFold(slug); len(candidates) == 1 { + return candidates[0], true + } + return slug, false +} + +func (BundledKeyVocabulary) TypeSlug(key string) string { + if bundle.HasObjectTypeByKey(domain.TypeKey(key)) { + return bundledTypeSpelling(key) + } + return key +} + +func (BundledKeyVocabulary) TypeKey(slug string) (string, bool) { + if key, ok := bundledTypeKeyBySpelling(slug); ok { + return key, true + } + if candidates := BundledTypeKeysByFold(slug); len(candidates) == 1 { + return candidates[0], true + } + return slug, false +} + +// keys returns the vocabulary in force — the caller's, or the bundled table. +func (o Options) keys() KeyVocabulary { + if o.Keys != nil { + return o.Keys + } + return BundledKeyVocabulary{} +} + +// propertySlug / propertyKey / typeSlug / typeKey are the raw vocabulary +// lookups; empty terms pass through untouched so no site has to special-case +// them. They are NOT the key-slot boundary: every slot goes through the +// exporter's claim step (propertySlug/typeSlug on *exporter — the term +// ledgers and the legends they owe) and the importer's legend-first read +// (propertyKey/typeKey on *importer). Options carries only the vocabulary; +// the document's own statements live with the codec halves. +func (o Options) propertySlug(key string) string { + if key == "" { + return key + } + return o.keys().PropertySlug(key) +} + +func (o Options) propertyKey(slug string) string { + if slug == "" { + return slug + } + // §3: a spelling resolves under its canonical NFC form (nfcTerm) — the + // read half of the rule PropertyLabel writes by. Idempotent, so the + // importer door normalizing first costs nothing here. + key, _ := o.keys().PropertyKey(nfcTerm(slug)) + return key +} + +// legendPropertyKey is propertyKey with §3 chain step 1 in front of it: the +// legend Options.Legend carries, which for a fragment entry point IS the +// enclosing document's `property_internal_keys`. Same precedence as +// importer.propertyKey, and stated once rather than twice for exactly that +// reason — the two doors into a type's property list must not disagree about +// what a spelling means. +func (o Options) legendPropertyKey(slug string) string { + if key, ok := legendLookup(o.Legend.PropertyKeys, slug); ok { + return key + } + return o.propertyKey(slug) +} + +// legendTypeKey is legendPropertyKey on the type namespace. +func (o Options) legendTypeKey(slug string) string { + if key, ok := legendLookup(o.Legend.TypeKeys, slug); ok { + return key + } + return o.typeKey(slug) +} + +// legendLookup answers a legend for one spelling under §3's normalization +// rule: the exact bytes first (a legend may bind a non-NFC spelling — export +// writes an identity entry for a stored key it spells verbatim), then the +// canonical NFC form, against a legend whose own non-NFC entries also answer +// for their NFC form (nfcExpandLegend). Values are stored keys and pass +// byte-verbatim. +func legendLookup(m map[string]string, slug string) (string, bool) { + if len(m) == 0 { + return "", false + } + expanded := nfcExpandLegend(m) + if key, ok := expanded[slug]; ok && key != "" { + return key, true + } + if n := nfcTerm(slug); n != slug { + if key, ok := expanded[n]; ok && key != "" { + return key, true + } + } + return "", false +} + +// nfcExpandLegend returns a legend that also answers for the NFC form of any +// non-NFC-spelled member (§3: a slot may spell one name in either byte +// form). Exact entries always win — an NFC-form shadow never displaces an +// entry the legend states at that exact spelling — and two non-NFC entries +// collapsing onto one unclaimed NFC form leave it unbound: an ambiguous +// address is never resolved by guess (the twin warning names the pair). The +// common all-NFC legend comes back untouched, unallocated. +func nfcExpandLegend[V any](m map[string]V) map[string]V { + var nonCanonical []string + for k := range m { + if nfcTerm(k) != k { + nonCanonical = append(nonCanonical, k) + } + } + if len(nonCanonical) == 0 { + return m + } + claims := map[string]int{} + for _, k := range nonCanonical { + claims[nfcTerm(k)]++ + } + out := make(map[string]V, len(m)+len(nonCanonical)) + for k, v := range m { + out[k] = v + } + for _, k := range nonCanonical { + n := nfcTerm(k) + if _, exact := m[n]; exact || claims[n] > 1 { + continue + } + out[n] = m[k] + } + return out +} + +func (o Options) typeSlug(key string) string { + if key == "" { + return key + } + return o.keys().TypeSlug(key) +} + +func (o Options) typeKey(slug string) string { + if slug == "" { + return slug + } + // §3's canonical form, as in propertyKey above + key, _ := o.keys().TypeKey(nfcTerm(slug)) + return key +} diff --git a/pkg/lib/anyblockjson/keyvocab_test.go b/pkg/lib/anyblockjson/keyvocab_test.go new file mode 100644 index 0000000000..d75a94c5a4 --- /dev/null +++ b/pkg/lib/anyblockjson/keyvocab_test.go @@ -0,0 +1,642 @@ +package anyblockjson + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/iancoleman/strcase" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +// TestKeyVocabulary_ReverseIsATableNotACaseTransform is the guard rail on the +// one decision that looks most "simplifiable" in this whole layer: the +// reverse direction of the vocabulary is a TABLE built from the bundle, in +// both directions, and NOT a string transform. Under raw names that is even +// more obviously so — "Creation date" says a different word than +// `createdDate`, so no derivation in either direction exists — but the two +// keys below pin the subtler halves: a key and its name can differ by more +// than case ever could, and a spelling that IS the stored key (a shared +// name has no wire form) must pass through rather than be "restored". +func TestKeyVocabulary_ReverseIsATableNotACaseTransform(t *testing.T) { + vocab := BundledKeyVocabulary{} + + t.Run("createdDate: the name says a different word than the key", func(t *testing.T) { + // given + const key = "createdDate" + require.True(t, bundle.HasRelation(key)) + + // when + spelling := vocab.PropertySlug(key) + back, ok := vocab.PropertyKey(spelling) + + // then + assert.Equal(t, "Creation date", spelling) + require.True(t, ok) + assert.Equal(t, key, back, "the table must invert what no transform can") + assert.Equal(t, "creationDate", strcase.ToLowerCamel(spelling), + "and this is what a case transform would have produced instead — a key that does not exist") + }) + + t.Run("fileId: a shared name has no wire form, so the key spells itself", func(t *testing.T) { + // given — nine hidden transients share the name "Underlying file id" + const key = "fileId" + require.True(t, bundle.HasRelation(key)) + + // when + spelling := vocab.PropertySlug(key) + back, ok := vocab.PropertyKey(spelling) + + // then + assert.Equal(t, "fileId", spelling, "an ambiguous name is not a spelling") + require.True(t, ok, "its own fold class still answers — to itself") + assert.Equal(t, key, back, "so the verbatim key remains its own address") + + _, ok = vocab.PropertyKey("Underlying file id") + assert.False(t, ok, "and the shared name binds nothing") + }) + + t.Run("every bundled key round-trips through the table", func(t *testing.T) { + for _, key := range []string{"dueDate", "iconEmoji", "lastModifiedDate", "setOf", "name", "_final_score"} { + back, ok := vocab.PropertyKey(vocab.PropertySlug(key)) + require.True(t, ok, key) + assert.Equal(t, key, back) + } + for _, key := range []string{"objectType", "relationOption", "spaceView", "diaryEntry", "chatDerived", "page"} { + back, ok := vocab.TypeKey(vocab.TypeSlug(key)) + require.True(t, ok, key) + assert.Equal(t, key, back) + } + }) +} + +// typeSlugShadowsBundled is the second KeyVocabulary precondition as a +// predicate: does this vocabulary bind a spelling the bundled table binds to +// a DIFFERENT key? Written out rather than described, so "conforming" is +// something a test can assert instead of something a reader has to check by +// eye. +func typeSlugShadowsBundled(v KeyVocabulary, key string) bool { + slug := v.TypeSlug(key) + if other, ok := (BundledKeyVocabulary{}).TypeKey(slug); ok && other != key { + return true // the emit direction + } + if back, ok := v.TypeKey(slug); ok { + if other, isBundled := (BundledKeyVocabulary{}).TypeKey(slug); isBundled && back != other { + return true // the accept direction + } + } + return false +} + +// TestKeyVocabulary_ShadowingSlugBreaksInversion pins the precondition +// KeyVocabulary states beyond "…Key inverts …Slug": no answer may bind a +// spelling the bundled table binds elsewhere. +// +// The vocabulary here is a strict inverse pair — it satisfies the weaker +// contract completely — and the type is still lost. Stating that honestly +// takes TWO parties, which is the precondition's whole point: a WRITER +// holding this vocabulary is covered, because export records the legend +// entry for every term its own vocabulary would bind elsewhere +// (recordTypeKey / termInverts), and the second arm below shows that +// document surviving. What no writer can cover is a READER it never met. +// The document in the first arm is written by the package default, owes no +// entry by any rule anyone can compute, and the shadowing reader still binds +// "Task" to `69bbfc…` — a template for the bundled Task type comes back as a +// template for an unrelated custom type, with no error anywhere. +// +// This is not a live defect. storeresolver, the only vocabulary the product +// wires, refuses both halves: keyMaps.roundTrips will not SPELL a key with a +// slug the bundled table binds elsewhere, and the bundledKey check in +// keyMaps.key will not BIND one — its comment records 12 re-pointed objects +// in a 36 808-object sweep as the cost of missing the second half. A +// hand-written Options.Keys can still do it, so the rule is written on the +// interface and held here, and the conforming twin below shows the same +// document surviving. +func TestKeyVocabulary_ShadowingSlugBreaksInversion(t *testing.T) { + shadowing := typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "Task"}} + require.True(t, typeSlugShadowsBundled(shadowing, customTypeKey), + "the fixture has to break the precondition, or this test proves nothing") + + snap := typedSnapshot("ot-template", "ot-task") + + t.Run("a shadowing READER re-points a document no writer could have warned it about", func(t *testing.T) { + // given — written by the package default: "Task" is the bundled + // table's own spelling of the bundled key, and no vocabulary this + // writer holds says otherwise + data, err := Marshal(model.SmartBlockType_Template, snap, Options{}) + require.NoError(t, err) + assert.NotContains(t, string(data), "type_internal_keys", + "nothing here owes an entry — which is exactly why the reader is on its own") + + // when + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: shadowing}) + + // then + require.NoError(t, err) + assert.Equal(t, []string{"ot-template", "ot-" + customTypeKey}, back.ObjectTypes, + "a shadowing vocabulary re-points the template's target type, silently — the failure the precondition forbids") + }) + + t.Run("a shadowing WRITER says so in the legend, and its own reader is safe", func(t *testing.T) { + // when + data, err := Marshal(model.SmartBlockType_Template, snap, Options{Keys: shadowing}) + require.NoError(t, err) + + // then — the term "Task" is written for the stored key `task`, and + // this vocabulary would bind it elsewhere, so the entry is owed (§3) + // even though the bundled table inverts it + assert.Equal(t, map[string]string{"Task": "task"}, decodeEnvelope(t, data).TypeKeys) + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: shadowing}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-template", "ot-task"}, back.ObjectTypes, + "the legend is chain step 1, ahead of the reader's vocabulary") + }) + + t.Run("a conforming vocabulary needs neither", func(t *testing.T) { + // the same document, the same shape of vocabulary, one conforming answer + conforming := typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "tsk7"}} + require.False(t, typeSlugShadowsBundled(conforming, customTypeKey)) + data, err := Marshal(model.SmartBlockType_Template, snap, Options{Keys: conforming}) + require.NoError(t, err) + assert.NotContains(t, string(data), "type_internal_keys") + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("h"), Keys: conforming}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-template", "ot-task"}, back.ObjectTypes, + "a conforming vocabulary round-trips the same document (§11.1, equivalent resolvers)") + }) +} + +// The vocabulary I1's round-trip axis reads with has to conform, or that axis +// measures the wrong thing — it would fail for the reason above rather than +// for a regression in the codec. +func TestKeyVocabulary_RoundTripVocabConforms(t *testing.T) { + for key := range roundTripTypeSlugs { + assert.False(t, typeSlugShadowsBundled(roundTripVocab{}, key), + "roundTripVocab must not shadow the bundled table for %q", key) + } +} + +func TestKeyVocabulary_CustomKeysPassThrough(t *testing.T) { + vocab := BundledKeyVocabulary{} + + // a package-only reader has no space to ask about names, so a custom key + // is spelled — and read back — verbatim (§3 chain step 5: a term no table + // answers for IS the stored key, always its own address) + for _, key := range []string{"68b1c0aa4e1f0d0011223344", "myLegacyKey", "customStatus"} { + assert.Equal(t, key, vocab.PropertySlug(key)) + back, ok := vocab.PropertyKey(key) + assert.False(t, ok) + assert.Equal(t, key, back) + } +} + +// TestDocumentSpellsNames pins the §3 surface rule at the format level: a +// document names properties and types by display name, reading it back +// restores the stored keys, and the legacy derived-slug spellings still +// resolve through the fold on the way in. Revert any of the boundary sites +// (export.go, dataview.go, typeproperties.go, import.go) and one of these +// fails. +func TestDocumentSpellsNames(t *testing.T) { + t.Run("properties are written as display names; legacy slugs read back as stored keys", func(t *testing.T) { + // given + doc := `{"version": 2, "id": "o1", "properties": { + "name": "A page", "plural_name": "Pages", "due_date": "2025-07-06T08:44:05Z", + "customDate": "whatever"}}` + + // when — the document's legacy slug spellings bind to stored keys + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + // then + assert.Contains(t, snap.Details.Fields, "pluralName") + assert.Contains(t, snap.Details.Fields, "dueDate") + assert.NotContains(t, snap.Details.Fields, "due_date") + assert.Contains(t, snap.Details.Fields, "customDate", "custom keys pass through") + + // and the export spells them back — as the display names, which is + // the wire vocabulary now; the legacy slugs the input carried keep + // resolving through the fold but are never written again + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"Plural name"`) + assert.Contains(t, string(data), `"Due date"`) + assert.NotContains(t, string(data), `"pluralName"`) + assert.NotContains(t, string(data), `"plural_name"`) + assert.Contains(t, string(data), `"customDate"`) + }) + + t.Run("a dataview's key slots follow the same vocabulary", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "blocks": [{"id": "dv", "type": "dataview", + "properties": [{"property": "due_date", "format": "date"}], + "views": [{"id": "v1", "type": "table", "group_by": "due_date", + "sorts": [{"property": "due_date"}], + "filters": [{"property": "due_date", "condition": "not_empty"}], + "columns": [{"property": "due_date"}]}]}]}` + + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + var dv *model.BlockContentDataview + for _, b := range snap.Blocks { + if c, ok := b.Content.(*model.BlockContentOfDataview); ok { + dv = c.Dataview + } + } + require.NotNil(t, dv) + require.Len(t, dv.RelationLinks, 1) + assert.Equal(t, "dueDate", dv.RelationLinks[0].Key) + require.Len(t, dv.Views, 1) + assert.Equal(t, "dueDate", dv.Views[0].GroupRelationKey) + require.Len(t, dv.Views[0].Sorts, 1) + assert.Equal(t, "dueDate", dv.Views[0].Sorts[0].RelationKey) + require.Len(t, dv.Views[0].Filters, 1) + assert.Equal(t, "dueDate", dv.Views[0].Filters[0].RelationKey) + require.Len(t, dv.Views[0].Relations, 1) + assert.Equal(t, "dueDate", dv.Views[0].Relations[0].Key) + }) + + t.Run("the envelope type follows the same vocabulary", func(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "type": "object_type"}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + require.Len(t, snap.ObjectTypes, 1) + assert.Equal(t, "ot-objectType", snap.ObjectTypes[0]) + }) +} + +// collapsingVocab spells two stored keys as ONE spelling — under raw names +// an ordinary shape, not a pathology: names are not unique, and a conforming +// space vocabulary grants a shared name to every claimant (collisions are +// resolved per document, not per space). No bundled fixture can produce it, +// because the bundled table refuses a shared name outright. +type collapsingVocab struct{ a, b, slug string } + +func (v collapsingVocab) PropertySlug(key string) string { + if key == v.a || key == v.b { + return v.slug + } + return key +} + +func (v collapsingVocab) PropertyKey(slug string) (string, bool) { + if slug == v.slug { + return v.a, true + } + return slug, false +} + +func (v collapsingVocab) TypeSlug(key string) string { return key } +func (v collapsingVocab) TypeKey(s string) (string, bool) { return s, false } + +// TestBuildPropertiesKeepsBothValuesWhenSlugsCollapse is the data-loss guard +// behind buildProperties. Two stored keys spelling one JSON key would +// overwrite each other in the properties map — one value gone, no error, no +// warning. The census's collision plan (planKeyTerms) degrades EVERY +// claimant of a contested spelling instead — here both keys are readable, +// so both take their stored keys — and the outcome may not depend on Go's +// map iteration order: the plan runs over the sorted census, or the +// canonical form is a coin flip on exactly the spaces that hold a shadow. +// Revert the plan and one value lands under the shared spelling by claim +// order (intermittently, which is the point). +func TestBuildPropertiesKeepsBothValuesWhenSlugsCollapse(t *testing.T) { + // given + snapshot := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + "aaaKey": pbtypes.String("value of A"), + "zzzKey": pbtypes.String("value of Z"), + }}, + } + vocab := collapsingVocab{a: "aaaKey", b: "zzzKey", slug: "shared_slug"} + + // when: repeated, because map iteration order is randomized per run + for i := 0; i < 32; i++ { + data, err := Marshal(model.SmartBlockType_Page, snapshot, Options{Keys: vocab}) + + // then + require.NoError(t, err) + var doc struct { + Properties map[string]string `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Len(t, doc.Properties, 2, "no value may be lost to a collapsed spelling") + assert.Equal(t, "value of A", doc.Properties["aaaKey"], + "every claimant of a contested spelling degrades — the shared spelling is written for nobody") + assert.Equal(t, "value of Z", doc.Properties["zzzKey"], "each keeps its honest stored key") + assert.NotContains(t, doc.Properties, "shared_slug") + } +} + +// TestBuildPropertiesRefusesASlugAnotherStoredKeyOwns is the second arm of the +// same collapse: the contested spelling is not another holder's SLUG but +// another stored key on this very object. Emitting it would bind the value to +// that key on the way back (chain step 1 — an exact stored key always wins). +func TestBuildPropertiesRefusesASlugAnotherStoredKeyOwns(t *testing.T) { + // given + snapshot := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + "aaaKey": pbtypes.String("value of A"), + "shared_slug": pbtypes.String("value of the squatter"), + }}, + } + vocab := collapsingVocab{a: "aaaKey", b: "", slug: "shared_slug"} + + // when + data, err := Marshal(model.SmartBlockType_Page, snapshot, Options{Keys: vocab}) + + // then + require.NoError(t, err) + var doc struct { + Properties map[string]string `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Len(t, doc.Properties, 2) + assert.Equal(t, "value of the squatter", doc.Properties["shared_slug"]) + assert.Equal(t, "value of A", doc.Properties["aaaKey"], "the slug is not emitted — its spelling is taken") +} + +// TestObjectTypesIsAKeySlot pins the vocabulary decision for +// typeProperties[].objectTypes (§2a's target-type restriction). It NAMES +// types, so it is a type-key slot and speaks the one vocabulary — the same +// answer the envelope `type` gets. It was the last untranslated key slot in +// the format; revert the typeSlugs/typeKeys calls in typeproperties.go and +// this fails in both directions. +func TestObjectTypesIsAKeySlot(t *testing.T) { + t.Run("import inverts the spelling to the stored type key", func(t *testing.T) { + // given + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "owner", "name": "Owner", "format": "objects", + "object_types": ["object_type", "wikiPerson"]}]}}` + r := &recordingPropertyResolver{} + + // when + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), ResolveProperties: r}) + + // then + require.NoError(t, err) + require.Len(t, r.defs, 1) + assert.Equal(t, []string{"objectType", "wikiPerson"}, r.defs[0].ObjectTypes, + "the legacy slug inverts through the fold; an unknown term passes through (chain step 5, verbatim)") + }) + + t.Run("export spells the display name", func(t *testing.T) { + snapshot := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + "recommendedRelations": pbtypes.StringList([]string{"rel-owner"}), + }}, + ObjectTypes: []string{"ot-objectType"}, + } + resolver := &staticPropertyResolver{def: PropertyDefinition{ + Key: "owner", Name: "Owner", Format: model.RelationFormat_object, + ObjectTypes: []string{"objectType", "wikiPerson"}, + }} + + data, err := Marshal(model.SmartBlockType_STType, snapshot, Options{ResolveProperties: resolver}) + + require.NoError(t, err) + var doc struct { + TypeSettings struct { + PropertyDefinitions []TypeProperty `json:"property_definitions"` + } `json:"type_settings"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + require.Len(t, doc.TypeSettings.PropertyDefinitions, 1) + assert.Equal(t, []string{"Type", "wikiPerson"}, doc.TypeSettings.PropertyDefinitions[0].ObjectTypes) + }) +} + +// TestBuildRecommendedListsInvertsItsKeySlots: the PATCH-types channel writes +// the SAME §2a array a type document carries, so it must invert the same key +// slots through the same vocabulary. It took a bare resolver and inverted +// nothing, which made one type's property list mean two different things +// depending on which endpoint wrote it. +func TestBuildRecommendedListsInvertsItsKeySlots(t *testing.T) { + // given + r := &recordingPropertyResolver{} + props := []TypeProperty{{ + Property: "due_date", + Name: "Due date", + Format: "date", + ObjectTypes: []string{"object_type", "wikiPerson"}, + Section: "featured", + }} + + // when + lists, err := BuildRecommendedLists(props, Options{ResolveProperties: r}) + + // then + require.NoError(t, err) + require.Len(t, r.defs, 1) + assert.Equal(t, domain.RelationKey("dueDate"), r.defs[0].Key, + "the resolver receives the def import would hand it — stored spellings") + assert.Equal(t, []string{"objectType", "wikiPerson"}, r.defs[0].ObjectTypes) + require.NotEmpty(t, lists) + assert.Equal(t, "recommendedFeaturedRelations", lists[0].DetailKey) + assert.Equal(t, []string{"dueDate"}, lists[0].Ids) +} + +// TestImportRefusesTwoSpellingsOfOneStoredKey is the ACCEPT-half mirror of +// the export collapse guard above. `build()` ranged doc.Properties, so when +// two spellings canonicalized onto one stored key the last writer won — over +// a Go map. Same request, different stored object, run to run (48 identical +// POSTs: iconEmoji A:6 / B:42). The refusal belongs in the codec because the +// API layer's canonicalizeDocumentKeys — which refuses first, and better — +// is skipped by the type-create channel and by every direct package caller +// (cmd/anyblockroundtrip, cmd/anyblockrecover, cmd/internal/anyblockbatch). +// Revert either the sort or the boundBy branch and this fails (the sort one +// intermittently, which is the point of the 32 iterations). +func TestImportRefusesTwoSpellingsOfOneStoredKey(t *testing.T) { + // The original repro was `icon_emoji` beside `iconEmoji`; that exact pair + // is now refused one step earlier, because §2b lifted the icon keys out of + // `properties` altogether — and `pluralName`, which the repro moved to, + // is likewise refused earlier ON TYPE DOCUMENTS since the §2a lift. The + // shape both stood for is unchanged and still reachable through every + // bundled twin on a kind with no lift for it, so the repro stays on the + // same pair one kind over: a page carrying pluralName is unusual but + // legal, and both spellings still land on one detail. + t.Run("the POST /types repro: plural_name beside pluralName", func(t *testing.T) { + // given — a stored key the bundled table resolves elsewhere: + // `plural_name` inverts to `pluralName`, which is also a literal + // stored key, so both spellings land on one detail + doc := `{"version": 2, "id": "t1", "internal_key": "k", + "properties": {"name": "T", "plural_name": "A", "pluralName": "B"}}` + + for i := 0; i < 32; i++ { + // when + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + + // then + require.Error(t, err, "a collapse must never be resolved by map order") + var ve *ValidationError + require.ErrorAs(t, err, &ve) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/properties/plural_name", ve.Issues[0].Path, "the same path on every run") + assert.Contains(t, ve.Issues[0].Message, `"pluralName" and "plural_name" both address property "pluralName"`) + } + }) + + t.Run("a vocabulary spelling over a BSON key collapses the same way", func(t *testing.T) { + // given — the ordinary space shape: a BSON stored key addressed by + // the name its vocabulary grants it. Naming both spellings + // in one document addresses one property twice. + const bsonKey = "68b1c0aa4e1f0d0011223344" + vocab := collapsingVocab{a: bsonKey, slug: "severity"} + doc := `{"version": 2, "id": "o1", "properties": {"severity": "high", "` + bsonKey + `": "low"}}` + + // when + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + + // then + var ve *ValidationError + require.ErrorAs(t, err, &ve) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/properties/severity", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, `"`+bsonKey+`" and "severity"`) + }) + + t.Run("distinct keys are untouched", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"name": "T", "plural_name": "A", "due_date": "2025-07-06T08:44:05Z"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Contains(t, snap.Details.Fields, "pluralName") + assert.Contains(t, snap.Details.Fields, "dueDate") + }) +} + +// corpseVocabulary is the vocabulary a space grows by DELETING things, and it +// is fully conforming: a strict inverse pair in both namespaces, and no +// answer touches a spelling the bundled table binds. A UI-deleted type or +// property vacates its stored key (storeresolver's corpse policy — +// loadKeyMaps filters `isUninstalled != true` out of the name namespace), so +// the key stops being a live stored key while every object that used it +// still carries it; the freed spelling is then another live entity's. +// `initiative` is that spelling here, in both namespaces, and it binds to +// the BSON key. +type corpseVocabulary struct{} + +// corpsePropKey is a space-minted (bson) relation key, the property +// namespace's customTypeKey. +const corpsePropKey = "6a32d4856761631534b22f85" + +func (corpseVocabulary) PropertySlug(key string) string { + if key == corpsePropKey { + return "initiative" + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (corpseVocabulary) PropertyKey(slug string) (string, bool) { + if slug == "initiative" { + return corpsePropKey, true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (corpseVocabulary) TypeSlug(key string) string { + if key == customTypeKey { + return "initiative" + } + return BundledKeyVocabulary{}.TypeSlug(key) +} + +func (corpseVocabulary) TypeKey(slug string) (string, bool) { + if slug == "initiative" { + return customTypeKey, true + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// TestKeyVocabulary_VocabularyInForceIsAReaderToo is the §3 legend rule +// stated the way export has to apply it: a term owes an entry when ANY +// resolver a conforming reader may run binds it to a different stored key — +// not only when the BUNDLED table does. +// +// Export used to ask the bundled table alone (recordTypeKey / +// recordPropertyKey), which is the wrong table for the reader most likely to +// read the document: the writer's own space. Both symptoms below come from +// that one question, and both need a vocabulary that CONFORMS — the +// preconditions on KeyVocabulary do not forbid this shape, and storeresolver +// grows it on its own the moment a user deletes a type whose freed spelling +// another entity's name then takes. +func TestKeyVocabulary_VocabularyInForceIsAReaderToo(t *testing.T) { + require.False(t, typeSlugShadowsBundled(corpseVocabulary{}, customTypeKey), + "the fixture must CONFORM, or it proves only that a broken vocabulary breaks") + + t.Run("the type namespace: a stored key written verbatim, re-pointed in silence", func(t *testing.T) { + // given: the object is typed with the DELETED type's stored key, + // which the space no longer reserves — while the live type's api key + // is that very spelling + snap := typedSnapshot("ot-initiative") + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: corpseVocabulary{}}) + require.NoError(t, err) + + // then: `initiative` is written verbatim, and the identity entry is + // what says so — the bundled table is silent on the term, so nothing + // else in the document can + doc := decodeEnvelope(t, data) + assert.Equal(t, "initiative", doc.Type) + assert.Equal(t, map[string]string{"initiative": "initiative"}, doc.TypeKeys) + require.NoError(t, Validate(data)) + + // and the writer's own reader binds it back to the type it came from + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: corpseVocabulary{}}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-initiative"}, back.ObjectTypes, + "without the entry this is ot-"+customTypeKey+" — a different type, no error anywhere") + + // as does a package-only reader, which is what an archive gets + _, back, err = Unmarshal(data, Options{GenerateId: seqIds("h")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-initiative"}, back.ObjectTypes) + }) + + t.Run("the property namespace: Marshal emits what its own Unmarshal refuses", func(t *testing.T) { + // given: the object holds BOTH the BSON key and the stored key whose + // spelling this vocabulary hands to it + snap := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + "initiative": pbtypes.String("value of the deleted property"), + corpsePropKey: pbtypes.String("value of the live one"), + }}, + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: corpseVocabulary{}}) + require.NoError(t, err) + + // then: the census contests the spelling (the stored key + // `initiative` owns its own term, verbatim-first), so the bson + // claimant degrades through the ladder — its key is unreadable, so + // it takes the suffixed form — and every written term owes its + // entry, because this reader binds `initiative` elsewhere + doc := decodeEnvelope(t, data) + assert.Equal(t, "value of the deleted property", doc.Properties["initiative"]) + assert.Equal(t, "value of the live one", doc.Properties["initiative (b22f85)"], + "an unreadable claimant of a contested spelling takes ` ()`") + assert.Equal(t, map[string]string{ + "initiative": "initiative", + "initiative (b22f85)": corpsePropKey, + }, doc.PropertyKeys, + "the identity entry for the stored key, and the suffix's inverse — "+ + "the bundled table binds neither term, so neither is safe from a reader that later does") + require.NoError(t, Validate(data)) + + // and both values come home, on the stored keys they left on. Without + // the entry the two spellings both address corpsePropKey, and + // Unmarshal refuses the document Marshal has just written — I1. + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: corpseVocabulary{}}) + require.NoError(t, err) + assert.Equal(t, "value of the deleted property", + back.Details.Fields["initiative"].GetStringValue()) + assert.Equal(t, "value of the live one", + back.Details.Fields[corpsePropKey].GetStringValue()) + }) +} diff --git a/pkg/lib/anyblockjson/kind_test.go b/pkg/lib/anyblockjson/kind_test.go new file mode 100644 index 0000000000..b3c893b87e --- /dev/null +++ b/pkg/lib/anyblockjson/kind_test.go @@ -0,0 +1,66 @@ +package anyblockjson + +// kind: "chat" is the authorable name for ChatDerivedObject (§2). A chat is a +// standalone object: its identity is the envelope's "internal_key", like a type's. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func TestImport_ChatKind(t *testing.T) { + doc := `{"version": 2, "id": "chat-wiki", "kind": "chat", "internal_key": "wikiChat", + "icon": {"format": "emoji", "emoji": "💬"}, + "properties": {"name": "Wiki"}}` + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + assert.Equal(t, model.SmartBlockType_ChatDerivedObject, sbType) + assert.Equal(t, "wikiChat", snap.Key, "identity lives in key, like a type") + assert.Equal(t, "chat-wiki", snap.Details.Fields["id"].GetStringValue()) + assert.Equal(t, "Wiki", snap.Details.Fields["name"].GetStringValue()) + assert.Equal(t, "💬", snap.Details.Fields["iconEmoji"].GetStringValue(), + "the typed envelope field is where an icon is written now (§2b)") +} + +func TestRoundtrip_ChatKind(t *testing.T) { + snapshot := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "chat-wiki", Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + }, + Details: fields(map[string]*types.Value{ + "id": str("chat-wiki"), + "name": str("Wiki"), + }), + Key: "wikiChat", + } + data, err := Marshal(model.SmartBlockType_ChatDerivedObject, snapshot, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(data), `"kind": "chat"`) + assert.NotContains(t, string(data), "chat_derived") + + sbType, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_ChatDerivedObject, sbType) + assert.Equal(t, "wikiChat", back.Key) +} + +// the old name is gone from the vocabulary, and the deprecated sibling kind +// stays distinct from it +func TestValidate_ChatDerivedNameRejected(t *testing.T) { + _, _, err := Unmarshal([]byte(`{"version": 2, "kind": "chat_derived", "internal_key": "k"}`), + Options{GenerateId: seqIds("g")}) + require.Error(t, err) + assert.Contains(t, err.Error(), "kind") + + sbType, _, err := Unmarshal([]byte(`{"version": 2, "kind": "chat_object", "internal_key": "k"}`), + Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_ChatObjectDeprecated, sbType) +} diff --git a/pkg/lib/anyblockjson/label.go b/pkg/lib/anyblockjson/label.go new file mode 100644 index 0000000000..35f0670d40 --- /dev/null +++ b/pkg/lib/anyblockjson/label.go @@ -0,0 +1,76 @@ +package anyblockjson + +// label.go — the spelling a document writes for a key the BUNDLED table does +// not speak for (§3): the entity's display name, NFC-normalized, otherwise +// verbatim. +// +// This used to be a ladder — the stored `apiObjectKey` when legal, +// re-spelled by the name inside one fold class, else the slug normalized, +// else the name normalized through an identifier grammar — and every rung +// existed to repair what normalization broke: `#`, `☕` and `C++` normalize +// to nothing or to `c`, so the ladder carried an empty-normalization +// fallback; `50% done` and `All` broke the grammar, so it carried a +// leading-`_` escape. Raw naming has no normalization step, so none of +// those faults can arise and none of the machinery survives: a name is a +// legal key exactly as written (the writable-key rule admits spaces, +// punctuation and every script), `api_object_key` is never read by the +// format at all, and the only inputs that still yield no label are the ones +// no rule could spell — an empty name, a name over the writable bound, a +// name carrying control characters. Each of those degrades to the stored +// key verbatim, which is always its own address. +// +// The name is carried exactly as the space holds it — edge whitespace and +// invisible characters included. The format warns about those (§12) but +// does not trim: a cleanup belongs where a user creates or renames the +// entity, applied once, not at the export seam on every write. The fold +// layer forgives the near-misses either way. + +import ( + "golang.org/x/text/unicode/norm" +) + +// PropertyLabel is the spelling a document writes for one space-minted +// PROPERTY key: NFC(name), else nothing. An empty answer means the key has +// no label but itself — the stored key is written verbatim, which is always +// its own address (§3 chain step 5). A name that merely repeats the stored +// key is no label either, for the same reason a spelling that repeated it +// never said anything: the verbatim key already says exactly that. +// +// `id` and `type` are refused: §2 refuses both SPELLINGS in `properties` +// before any resolution, so minting one would produce a label the exporter +// throws away with a warning. The type namespace does not share that +// reservation — its home surface is a value, not a member name — which is +// the one place the two namespaces differ and why TypeLabel is a separate +// function rather than a flag. +func PropertyLabel(key, name string) string { + label := TypeLabel(key, name) + if label == detailKeyId || label == detailKeyType { + return "" + } + return label +} + +// nfcTerm is §3's canonical form of one key spelling — NFC, otherwise +// verbatim. PropertyLabel/TypeLabel are the write half of the rule (a label +// is minted NFC); this is the read half's step: a slot's term resolves under +// its canonical form, so the precomposed and the decomposed bytes of one +// name land on one key instead of splitting into two visually +// indistinguishable properties. Stored keys — legend VALUES, the argument of +// every `…Slug` call — are never passed through it: a stored key's bytes are +// its address, whatever their normal form. +func nfcTerm(s string) string { + return norm.NFC.String(s) +} + +// TypeLabel is PropertyLabel for the type namespace. +func TypeLabel(key, name string) string { + label := norm.NFC.String(name) + // isWritablePropertyKey is the format's own shape rule (§3) — non-empty, + // no control characters, inside the schema's 128-character bound. A name + // longer than that is refused rather than truncated: a truncation + // invents a spelling nobody chose, and the stored key is right there. + if label == key || !isWritablePropertyKey(label) { + return "" + } + return label +} diff --git a/pkg/lib/anyblockjson/label_test.go b/pkg/lib/anyblockjson/label_test.go new file mode 100644 index 0000000000..cba3219dc8 --- /dev/null +++ b/pkg/lib/anyblockjson/label_test.go @@ -0,0 +1,206 @@ +package anyblockjson + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +// The label rule (§3): a space-minted key is spelled by its display name, +// NFC-normalized, otherwise verbatim. There is no normalization step left to +// fail, so the old repair machinery — the empty-normalization fallback, the +// leading-`_` escape for digit-initial and keyword names — is deleted, not +// reimplemented, and this table pins that the inputs which used to NEED it +// are now plain spellings. +// +// How this can fail: reintroduce any transform (a case fold, a separator +// collapse, a grammar escape) and the verbatim rows below stop being +// verbatim; drop the writable-key check and a control-character name becomes +// a spelling Validate rejects (I1). +func TestPropertyLabel(t *testing.T) { + const bson = "6a7663db61fab21cd4b9e101" + + for name, tc := range map[string]struct{ display, want, why string }{ + "a name is the label, verbatim": { + display: "Publish Date", want: "Publish Date", + why: "no case fold, no separator collapse"}, + "spaces and punctuation survive": { + display: "Manual export & import", want: "Manual export & import", why: ""}, + "a one-symbol name is a legal key exactly as written": { + display: "#", want: "#", + why: "under a NORMALIZED spelling this normalized to nothing and needed a fallback"}, + "an emoji name likewise": { + display: "☕", want: "☕", why: ""}, + "a digit-initial name needs no escape": { + display: "50% done", want: "50% done", + why: "the leading-underscore escape existed for an identifier grammar keys no longer live in"}, + "a grammar keyword is just a name": { + display: "All", want: "All", why: ""}, + "non-Latin scripts are kept, never transliterated": { + display: "Дата выполнения", want: "Дата выполнения", why: ""}, + "CJK likewise": { + display: "作業内容", want: "作業内容", why: ""}, + "edge whitespace is carried, not trimmed": { + display: "Email 📧 ", want: "Email 📧 ", + why: "the format warns about it (§12) but a cleanup belongs at authoring time, not at the export seam"}, + "an empty name has no label but the stored key": { + display: "", want: "", why: ""}, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.want, PropertyLabel(bson, tc.display), tc.why) + }) + } + + t.Run("a name that repeats the stored key is not a label", func(t *testing.T) { + // the verbatim key already says exactly that, and the legend rule + // would owe an identity entry either way + assert.Equal(t, "", PropertyLabel("website", "website")) + }) + + // the schema's propertyNames bound (§3). Truncating would invent a + // spelling nobody chose and could collide two long names onto one + // label; the stored key is right there. + t.Run("a name past the key bound is refused, never truncated", func(t *testing.T) { + assert.Equal(t, "", PropertyLabel(bson, strings.Repeat("a", maxPropertyKeyLen+1))) + assert.Len(t, PropertyLabel(bson, strings.Repeat("a", maxPropertyKeyLen)), maxPropertyKeyLen) + }) + + t.Run("a control character has no written form", func(t *testing.T) { + assert.Equal(t, "", PropertyLabel(bson, "a\nb")) + }) + + // §2 refuses `id` and `type` as property SPELLINGS before any + // resolution, so minting one produces a label export throws away with a + // warning. The type namespace has no such reservation — its home + // surface is a value, not a member name. + t.Run("the two reserved property spellings are never minted", func(t *testing.T) { + assert.Equal(t, "", PropertyLabel(bson, "id")) + assert.Equal(t, "", PropertyLabel(bson, "type")) + assert.Equal(t, "id", TypeLabel(bson, "id")) + assert.Equal(t, "type", TypeLabel(bson, "type")) + }) + + // "Type" with a capital T is NOT the refused spelling: the refusal is + // byte-exact (§2 refuses the member names `id` and `type`), and raw + // naming means the capitalized name no longer lowercases into the + // refused form on its way to the wire. + t.Run("the refusal is byte-exact, not case-folded", func(t *testing.T) { + assert.Equal(t, "Type", PropertyLabel(bson, "Type")) + assert.Equal(t, "Id", PropertyLabel(bson, "Id")) + }) + + // Pitfall stated in §3: two visually identical names can be different + // byte sequences, and a reader matches a spelling byte-for-byte. Export + // is safe by construction (it writes the same string as label and as + // legend key), but the space's stored name may itself be decomposed, and + // two exports of one property must not differ by normalization form. + t.Run("NFD and NFC forms of one name yield one label", func(t *testing.T) { + nfc := "Ünïcødé" + nfd := norm.NFD.String(nfc) + assert.NotEqual(t, nfc, nfd, "the fixture has to actually differ in bytes") + assert.Equal(t, PropertyLabel(bson, nfc), PropertyLabel(bson, nfd)) + assert.Equal(t, nfc, PropertyLabel(bson, nfd)) + }) +} + +// labelVocab is a vocabulary that answers with a §3 LABEL — the shape +// storeresolver produces for a space-minted key: the display name, raw. +type labelVocab struct{ key, label, typeKey, typeLabel string } + +func (v labelVocab) PropertySlug(key string) string { + if key == v.key { + return v.label + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (v labelVocab) PropertyKey(slug string) (string, bool) { + if slug == v.label { + return v.key, true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (v labelVocab) TypeSlug(key string) string { + if key == v.typeKey { + return v.typeLabel + } + return BundledKeyVocabulary{}.TypeSlug(key) +} + +func (v labelVocab) TypeKey(slug string) (string, bool) { + if slug == v.typeLabel { + return v.typeKey, true + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// A label is not ASCII, and the whole codec has to survive that: the label +// rule keeps the name verbatim in any script, so `Тоггл` labels a property +// `Тоггл` and the format writes that as a JSON member name, as a legend +// spelling, and as an envelope type term. +// +// This is I1 (§11 — "Marshal never emits what its own Validate rejects") for +// the string shape the format itself now MINTS. It is not implied by the +// label rule being correct: the published schema states `propertyNames` as a +// pattern, and had it been the api key's `^[a-zA-Z0-9_]+$` — which is what +// every other slug-shaped surface in this codebase carries — every non-Latin +// label would have validated as an error against the document that had just +// been written. +func TestNonASCIILabelSurvivesTheWholeCodec(t *testing.T) { + // given + const key = "6a7663db61fab21cd4b9e101" + const typeKey = "6a7663db61fab21cd4b9e103" + vocab := labelVocab{key: key, label: "Тоггл", typeKey: typeKey, typeLabel: "日本語のプロパティ"} + snapshot := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + // an explicit id, or the byte-stability check below trips over + // the documented id exception (§11) rather than over a label + "id": pbtypes.String("o1"), + "name": pbtypes.String("A page"), + key: pbtypes.String("on"), + }}, + ObjectTypes: []string{"ot-" + typeKey}, + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snapshot, Options{Keys: vocab}) + require.NoError(t, err) + + // then: the document spells the labels, and carries the legends that + // invert them + var doc struct { + Type string `json:"type"` + Properties map[string]any `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + TypeKeys map[string]string `json:"type_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "on", doc.Properties["Тоггл"]) + assert.Equal(t, "日本語のプロパティ", doc.Type) + assert.Equal(t, key, doc.PropertyKeys["Тоггл"]) + assert.Equal(t, typeKey, doc.TypeKeys["日本語のプロパティ"]) + + // and its own validation accepts it (I1) — with NO vocabulary, which is + // the reader the schema speaks for + require.NoError(t, Validate(data)) + + // and it reads back onto the stored keys, through the legend alone + _, back, err := Unmarshal(data, Options{}) + require.NoError(t, err) + assert.Equal(t, "on", back.Details.Fields[key].GetStringValue()) + assert.Equal(t, []string{"ot-" + typeKey}, back.ObjectTypes) + + // and the round trip is byte-stable (§11) + again, err := Marshal(model.SmartBlockType_Page, back, Options{Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, string(data), string(again)) +} diff --git a/pkg/lib/anyblockjson/layout_test.go b/pkg/lib/anyblockjson/layout_test.go new file mode 100644 index 0000000000..7deed87068 --- /dev/null +++ b/pkg/lib/anyblockjson/layout_test.go @@ -0,0 +1,151 @@ +package anyblockjson + +// Layout is stored as a number but named in the format (§3). Before this, a +// document following the spec ("layout": "profile") imported the *string* +// onto a number-format property: every consumer reads it with an int64 +// getter, so the type silently fell back to basic (== 0). Since v0.32 the +// recommended layout travels as `type_settings.layout` (§2a); the same rule +// rides along. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func TestImport_LayoutNameToNumber(t *testing.T) { + for _, tc := range []struct { + name string + want model.ObjectTypeLayout + }{ + {"basic", model.ObjectType_basic}, + {"profile", model.ObjectType_profile}, + {"todo", model.ObjectType_todo}, + {"note", model.ObjectType_note}, + {"set", model.ObjectType_set}, + {"collection", model.ObjectType_collection}, + } { + t.Run(tc.name, func(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"layout": "` + tc.name + `"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + v := snap.Details.Fields["recommendedLayout"] + require.NotNil(t, v) + _, isNum := v.GetKind().(*types.Value_NumberValue) + require.True(t, isNum, "must be stored as a number, not %T", v.GetKind()) + assert.Equal(t, float64(tc.want), v.GetNumberValue()) + }) + } +} + +// legacy documents that wrote the raw enum still import unchanged +func TestImport_LayoutNumberStillAccepted(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"layout": 1}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, float64(model.ObjectType_profile), + snap.Details.Fields["recommendedLayout"].GetNumberValue()) +} + +func TestExport_LayoutNumberToName(t *testing.T) { + snapshot := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "t1", Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + }, + Details: fields(map[string]*types.Value{ + "id": str("t1"), + "recommendedLayout": num(float64(model.ObjectType_profile)), + "resolvedLayout": num(float64(model.ObjectType_todo)), + }), + Key: "k", + } + data, err := Marshal(model.SmartBlockType_STType, snapshot, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(data), `"layout": "profile"`, + "the recommended layout is the group's layout member (§2a)") + assert.NotContains(t, string(data), `"resolved_layout"`, + "a type document does not carry its own display provenance (§2a)") +} + +func TestRoundtrip_LayoutSurvives(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"layout": "profile"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + data, err := Marshal(model.SmartBlockType_STType, snap, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(data), `"layout": "profile"`) +} + +// a typo must not reach the snapshot as a bare string. +// +// The SCHEMA answers this now, not the semantic pass: `type_settings.layout` +// used to be `{"type": ["string","number"]}`, which meant a generator reading +// the published schema could emit any string it liked and only learn at the +// codec that the vocabulary is closed. The schema states the vocabulary, so +// the refusal arrives with the whole list — which the semantic message it +// replaced never carried. +func TestValidate_UnknownLayoutRejected(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"layout": "Profile"}}` + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, err) + assert.Contains(t, err.Error(), "/type_settings/layout") + assert.Contains(t, err.Error(), "'basic'", "the refusal names the vocabulary") + assert.Contains(t, err.Error(), "'profile'", "including the name that was nearly right") +} + +// Which stored keys are written by NAME is a per-key verdict (§3), and this +// pins each one together with the vocabulary it draws from — a key added to +// namedEnumProperties must state its concept here, and a key deliberately +// left as a number must stay listed as such. (This replaces the old +// isLayoutKey membership pin: the layout keys were the only named ones until +// the mechanism became a per-key table.) +func TestNamedEnumProperties_PerKeyVerdict(t *testing.T) { + want := map[string]string{ // stored key → the concept its refusals name + "recommendedLayout": "layout", + "layout": "layout", + "resolvedLayout": "layout", + // the object's own page alignment: user-settable (readonly false), + // stored as a model.BlockAlign — the enum the format already names + // twice, on a block's align and a view column's align + "layoutAlign": "align", + // the object's provenance, named on the format's own §2a precedent: + // "on ordinary objects origin is real provenance and stays" — the + // class of createdDate and creator, not of syncStatus. importType + // rides with it (objectorigin.go writes them as a pair). + "origin": "origin", + "importType": "import type", + // what an image was uploaded FOR, on file objects. Named on the + // measured standard the bare-integer keys beside it were left on: + // 4,079 occurrences against widgetLayout's 13 and + // headerRelationsLayout's 51. Its automatically_added member is in + // lockstep with is_hidden_discovery (4,053 of 4,053), which is the + // key a client actually filters on — so this one is named for the + // READER rather than for any behaviour that depends on it. + "imageKind": "image kind", + } + assert.Equal(t, len(want), len(namedEnumProperties), + "every named key owes a verdict here — a new one must say which vocabulary it draws from") + for key, what := range want { + vocab, named := namedEnumProperty(key) + require.True(t, named, "%s must be written by name", key) + assert.Equal(t, what, vocab.what, "%s draws from the wrong vocabulary", key) + } + // the layout-ish bundled keys that stay numbers, each for a stated + // reason: layoutWidth is a fraction, not an enum; widgetLayout and + // headerRelationsLayout hold enums almost nothing writes (13 and 51 + // occurrences across 28,831 real exported documents, against + // imageKind's 4,079) + for _, key := range []string{"layoutWidth", "widgetLayout", "headerRelationsLayout"} { + _, named := namedEnumProperty(key) + assert.False(t, named, "%s is deliberately not named", key) + } +} diff --git a/pkg/lib/anyblockjson/legendadmission_test.go b/pkg/lib/anyblockjson/legendadmission_test.go new file mode 100644 index 0000000000..3743a2b6c5 --- /dev/null +++ b/pkg/lib/anyblockjson/legendadmission_test.go @@ -0,0 +1,260 @@ +package anyblockjson + +// legendadmission_test.go — the two legends admit what they write. +// +// Every other key slot in this format checks a key before it writes it; +// `property_internal_keys` and `type_internal_keys` did not. The only admission on the way in +// was writableSlug/writableTypeSlug, and both return EARLY when the +// vocabulary has no slug for a key — so an unwritable stored key walked +// straight into the ledger, and Marshal emitted a legend its own Validate and +// Unmarshal reject. The whole object was unexportable and nothing said so. +// +// Reaching it needs only Options.Keys, which §3 accepts from anyone: a +// vocabulary that BINDS the spelling to some other stored key (precondition 2 +// broken, which is exactly why the second termInverts table exists) makes the +// identity entry owed, and the entry is one the schema refuses. + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// bindingVocabulary binds two spellings to other stored keys and spells +// nothing. Binding is the whole fixture: it is what makes an identity entry +// owed, in both namespaces, without moving any spelling. +type bindingVocabulary struct { + BundledKeyVocabulary + bind map[string]string +} + +func (v bindingVocabulary) PropertyKey(slug string) (string, bool) { + if key, ok := v.bind[slug]; ok { + return key, true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (v bindingVocabulary) TypeKey(slug string) (string, bool) { + if key, ok := v.bind[slug]; ok { + return key, true + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// overLongKey is past the §3 spelling bound by twelve characters. +var overLongKey = strings.Repeat("k", 140) + +// filterKeySnapshot names a stored key at a dataview FILTER slot. That slot +// matters: it used to hand whatever the view holds straight to propertySlug +// — where /properties dropped unwritable keys first — which is how an +// unwritable key reached the legend ledger at all. The slot now runs the +// same drop (slotPropertySlug), and this fixture pins that the legend stays +// clean either way. +func filterKeySnapshot(keys ...string) *model.SmartBlockSnapshotBase { + dv := &model.BlockContentDataview{} + view := &model.BlockContentDataviewView{Id: "v1", Name: "All"} + for i, key := range keys { + dv.RelationLinks = append(dv.RelationLinks, + &model.RelationLink{Key: key, Format: model.RelationFormat_longtext}) + view.Filters = append(view.Filters, &model.BlockContentDataviewFilter{ + Id: "f" + string(rune('1'+i)), + Condition: model.BlockContentDataviewFilter_Equal, + Format: model.RelationFormat_longtext, + RelationKey: key, Value: str("x"), + }) + } + dv.Views = []*model.BlockContentDataviewView{view} + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "o1", ChildrenIds: []string{"dv1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: dv}}, + }, + Details: fields(map[string]*types.Value{"id": str("o1"), "name": str("Board")}), + } +} + +// The property namespace. Each case pairs the hostile key with `shadowed`, a +// perfectly writable stored key the same vocabulary binds elsewhere: that one +// MUST still get its identity entry, so a change that simply stopped writing +// legend entries cannot pass this test. +func TestExport_PropertyLegendRefusesAnEntryItCannotHold(t *testing.T) { + for _, tc := range []struct { + name, key, wantWarn string + }{ + {"a control character in the stored key", "a\nb", `"a\nb" carries a control character`}, + {"a stored key past the spelling bound", overLongKey, "is 140 characters; the bound is 128"}, + } { + t.Run(tc.name, func(t *testing.T) { + // given + snap := filterKeySnapshot(tc.key, "shadowed") + var warnings []Issue + opts := Options{ + Keys: bindingVocabulary{bind: map[string]string{ + tc.key: "otherKey", "shadowed": "otherKey2"}}, + OnWarning: func(i Issue) { warnings = append(warnings, i) }, + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + + // then — I1: Marshal never emits what its own Validate rejects + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "emitted:\n%s", data) + + // an unwritable stored key no longer reaches the legend at all: + // the block key slots carry the schema bound /properties always + // had, so the SLOT drops it first, warned (slotPropertySlug) — + // which is what keeps this legend clean. The shadowed writable + // key still owes — and gets — its identity entry, so a change + // that simply stopped writing legend entries cannot pass. + assert.Equal(t, map[string]string{"shadowed": "shadowed"}, + decodeDoc(t, data).PropertyKeys, + "only the entry the legend can hold; the shadowed key still owes one") + require.NotEmpty(t, warnings, "a dropped slot is reported") + assert.Contains(t, warningsAt(warnings, ""), tc.wantWarn) + + // the unwritable key's filter is gone — dropped like the + // nameless one, never spelled — and the writable one survives + assert.Equal(t, "shadowed", backFilterKey(t, back, 0)) + }) + } +} + +// The value half, same rule from the other side: a DENIED stored key cannot +// be a legend value (§3 deny rule, pinned at validate.go's legend pass), and +// a vocabulary that both slugs it away and binds its verbatim spelling +// elsewhere made export write exactly that entry. +func TestExport_PropertyLegendRefusesADeniedValue(t *testing.T) { + // given + snap := blockKeySnapshot(map[string]*types.Value{"name": str("x")}, "uniqueKey") + var warnings []Issue + opts := Options{ + Keys: denyBindingVocabulary{}, + OnWarning: func(i Issue) { warnings = append(warnings, i) }, + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + _, _, err = Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "emitted:\n%s", data) + assert.Empty(t, decodeDoc(t, data).PropertyKeys, "a denied key cannot be a legend value") + assert.Contains(t, warningsAt(warnings, "/property_internal_keys"), + `"uniqueKey" is internal: export strips it`) +} + +// denyBindingVocabulary slugs an internal key away AND binds its verbatim +// spelling to another stored key — the two halves that together made the +// denied value reach the legend. +type denyBindingVocabulary struct{ BundledKeyVocabulary } + +func (denyBindingVocabulary) PropertySlug(key string) string { + if key == "uniqueKey" { + return "sneaky" + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (denyBindingVocabulary) PropertyKey(slug string) (string, bool) { + if slug == "uniqueKey" { + return "otherKey", true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +// The type namespace has the same two shapes at the envelope `type`, which +// carries no length or charset rule of its own (§3) and so hands the stored +// key to the ledger untouched. +func TestExport_TypeLegendRefusesAnEntryItCannotHold(t *testing.T) { + for _, tc := range []struct { + name, key, wantWarn string + }{ + {"a control character in the stored type key", "a\nb", `"a\nb" carries a control character`}, + {"a stored type key past the spelling bound", overLongKey, "is 140 characters; the bound is 128"}, + } { + t.Run(tc.name, func(t *testing.T) { + // given — one type on the object, plus a shadowed type key at a + // type property's object_types, which must still get its entry + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "t1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("t1"), + "recommendedRelations": {Kind: &types.Value_ListValue{ListValue: &types.ListValue{ + Values: []*types.Value{str("p1")}}}}, + }), + ObjectTypes: []string{"ot-" + tc.key}, + } + var warnings []Issue + opts := Options{ + Keys: bindingVocabulary{bind: map[string]string{ + tc.key: "otherType", "shadowedType": "otherType2"}}, + ResolveProperties: stubPropertyResolver{byId: map[string]PropertyDefinition{ + "p1": {Key: "prio", Format: model.RelationFormat_object, + ObjectTypes: []string{"shadowedType"}}, + }}, + OnWarning: func(i Issue) { warnings = append(warnings, i) }, + } + + // when + data, err := Marshal(model.SmartBlockType_STType, snap, opts) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + _, _, err = Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "emitted:\n%s", data) + + var doc struct { + Type string `json:"type"` + TypeKeys map[string]string `json:"type_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, tc.key, doc.Type, "the term is still spelled verbatim") + assert.Equal(t, map[string]string{"shadowedType": "shadowedType"}, doc.TypeKeys, + "only the entry the legend can hold") + require.NotEmpty(t, warnings) + assert.Contains(t, warningsAt(warnings, "/type_internal_keys"), tc.wantWarn) + }) + } +} + +// warningsAt joins the messages of every warning filed at one path, so a +// Contains assertion cannot pass on a warning about something else. +func warningsAt(issues []Issue, path string) string { + var msgs []string + for _, i := range issues { + if i.Path == path { + msgs = append(msgs, i.Message) + } + } + return strings.Join(msgs, "\n") +} + +// backFilterKey digs the nth filter's relation key out of an imported +// snapshot. +func backFilterKey(t *testing.T, snap *model.SmartBlockSnapshotBase, n int) string { + t.Helper() + for _, b := range snap.Blocks { + if c, ok := b.Content.(*model.BlockContentOfDataview); ok { + require.Len(t, c.Dataview.Views, 1) + require.Greater(t, len(c.Dataview.Views[0].Filters), n) + return c.Dataview.Views[0].Filters[n].RelationKey + } + } + t.Fatal("no dataview in the imported snapshot") + return "" +} diff --git a/pkg/lib/anyblockjson/legendorder_test.go b/pkg/lib/anyblockjson/legendorder_test.go new file mode 100644 index 0000000000..8c962ce3c4 --- /dev/null +++ b/pkg/lib/anyblockjson/legendorder_test.go @@ -0,0 +1,199 @@ +package anyblockjson + +// legendorder_test.go pins the ORDER of the three legends' members, in the +// bytes (§4 serialization canon: "`property_internal_keys`, `type_internal_keys` and +// `option_ids` entries sorted by key, and each `option_ids` inner map sorted +// by option name"). +// +// It has to read bytes, and that is the whole point of the file. Every other +// legend test decodes into a Go map, which throws member order away before +// the assertion sees it — so the canon was stated in SPEC.md and held by +// nothing: deleting a `sort.Strings` left the package green, and so did +// reversing one. Canonical byte form is what makes export∘import +// byte-stable (§11) and what lets a caller diff two generations of the same +// object, so an unordered legend is a real regression that no map-shaped +// assertion can see. +// +// Each fixture is built so that three orders differ from one another: the +// order the entries were recorded in, the order of the STORED keys, and the +// order of the SPELLINGS the legend is keyed by. Sorting the wrong column, or +// not sorting at all, therefore lands somewhere the assertion refuses. + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The stored keys ascend f01..f04 while their spellings descend — so a sort +// applied to the stored key rather than to the spelling it is keyed by comes +// out in the opposite order from the one the canon asks for. +var orderedLegendKeys = []struct{ stored, slug string }{ + {"6a32d4856761631534b22f01", "zulu"}, + {"6a32d4856761631534b22f02", "mike"}, + {"6a32d4856761631534b22f03", "alpha"}, + {"6a32d4856761631534b22f04", "bravo"}, +} + +// wantLegendOrder is those spellings in the canonical order. +var wantLegendOrder = []string{"alpha", "bravo", "mike", "zulu"} + +// rawMemberOrder returns the member names of the document's top-level `slot` +// object in the order the BYTES carry them — json.RawMessage keeps the +// encoded value intact, and json.Decoder walks its members in file order. +func rawMemberOrder(t *testing.T, data []byte, slot string) []string { + t.Helper() + var top map[string]json.RawMessage + require.NoError(t, json.Unmarshal(data, &top)) + raw, ok := top[slot] + require.True(t, ok, "the document carries no %q legend:\n%s", slot, data) + return memberOrder(t, raw) +} + +// memberOrder walks one encoded JSON object's member names in byte order. +func memberOrder(t *testing.T, raw json.RawMessage) []string { + t.Helper() + dec := json.NewDecoder(bytes.NewReader(raw)) + tok, err := dec.Token() + require.NoError(t, err) + require.Equal(t, json.Delim('{'), tok, "%s is not a JSON object", raw) + var out []string + for dec.More() { + key, err := dec.Token() + require.NoError(t, err) + name, ok := key.(string) + require.True(t, ok) + out = append(out, name) + var value json.RawMessage + require.NoError(t, dec.Decode(&value)) + } + return out +} + +// rawNestedMemberOrder walks the member names of ONE outer entry of a nested +// legend, again in byte order. +func rawNestedMemberOrder(t *testing.T, data []byte, slot, outer string) []string { + t.Helper() + var top map[string]json.RawMessage + require.NoError(t, json.Unmarshal(data, &top)) + var nested map[string]json.RawMessage + require.NoError(t, json.Unmarshal(top[slot], &nested)) + raw, ok := nested[outer] + require.True(t, ok, "%q carries no %q entry:\n%s", slot, outer, data) + return memberOrder(t, raw) +} + +func TestExport_LegendMembersAreSortedInTheBytes(t *testing.T) { + t.Run("property_internal_keys", func(t *testing.T) { + // given: four custom keys the bundled table cannot invert, so each + // owes a legend entry + slugOf := map[string]string{} + details := map[string]*types.Value{} + for _, k := range orderedLegendKeys { + slugOf[k.stored] = k.slug + details[k.stored] = num(1) + } + + // when + data, err := Marshal(model.SmartBlockType_Page, customKeySnapshot(details), + Options{Keys: spaceVocabulary{slugOf: slugOf}}) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data)) + assert.Equal(t, wantLegendOrder, rawMemberOrder(t, data, "property_internal_keys"), + "the legend is keyed by the spelling and sorted by it:\n%s", data) + }) + + t.Run("type_internal_keys", func(t *testing.T) { + // given: a type document whose one property targets four custom + // types, each of which owes an entry + typeSlugOf := map[string]string{} + var targets []string + for _, k := range orderedLegendKeys { + typeSlugOf[k.stored] = k.slug + targets = append(targets, k.stored) + } + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "t1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("t1"), + "recommendedRelations": strList("rel-owner"), + }), + ObjectTypes: []string{"ot-objectType"}, + Key: "k", + } + resolver := &staticPropertyResolver{def: PropertyDefinition{ + Key: "owner", Name: "Owner", Format: model.RelationFormat_object, + ObjectTypes: targets, + }} + + // when + data, err := Marshal(model.SmartBlockType_STType, snap, Options{ + Keys: typedSpaceVocabulary{typeSlugOf: typeSlugOf}, ResolveProperties: resolver}) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data)) + assert.Equal(t, wantLegendOrder, rawMemberOrder(t, data, "type_internal_keys"), + "the type legend obeys the same canon as the property one:\n%s", data) + }) + + t.Run("option_ids outer keys", func(t *testing.T) { + // given: the same four keys, now carrying select values, so each + // contributes one outer entry to the option legend + data := marshalOrderedOptionDoc(t) + + // then + assert.Equal(t, wantLegendOrder, rawMemberOrder(t, data, "option_ids"), + "an outer key is a property spelling, sorted like the other two legends:\n%s", data) + }) + + t.Run("option_ids inner keys", func(t *testing.T) { + // given: one property holding three options whose names sort the + // other way round from the order the value lists them in + data := marshalOrderedOptionDoc(t) + + // then + assert.Equal(t, []string{"Ace", "Mid", "Zed"}, rawNestedMemberOrder(t, data, "option_ids", "alpha"), + "an inner map is sorted by option name, not by the order the value spelled them:\n%s", data) + }) +} + +// marshalOrderedOptionDoc exports a document whose option legend has four +// outer entries, one of which (`alpha`) holds three options listed in +// reverse-sorted name order. +func marshalOrderedOptionDoc(t *testing.T) []byte { + t.Helper() + slugOf := map[string]string{} + space := spaceOptions{} + details := map[string]*types.Value{} + for _, k := range orderedLegendKeys { + slugOf[k.stored] = k.slug + space[domain.RelationKey(k.stored)] = []spaceOption{{id: "opt-" + k.slug, name: "Only"}} + details[k.stored] = strList("opt-" + k.slug) + } + // `alpha` holds three, named in descending order so the sorted inner map + // is not the order they were recorded in + alpha := orderedLegendKeys[2].stored + space[domain.RelationKey(alpha)] = []spaceOption{ + {id: "opt-z", name: "Zed"}, {id: "opt-m", name: "Mid"}, {id: "opt-a", name: "Ace"}} + details[alpha] = strList("opt-z", "opt-m", "opt-a") + + data, err := Marshal(model.SmartBlockType_Page, customKeySnapshot(details), Options{ + Keys: spaceVocabulary{slugOf: slugOf}, + ResolveFormat: selectFormats, + ResolveOptions: space, + }) + require.NoError(t, err) + require.NoError(t, Validate(data)) + return data +} diff --git a/pkg/lib/anyblockjson/manifest_test.go b/pkg/lib/anyblockjson/manifest_test.go new file mode 100644 index 0000000000..318e2cb7ed --- /dev/null +++ b/pkg/lib/anyblockjson/manifest_test.go @@ -0,0 +1,192 @@ +package anyblockjson + +// manifest_test.go pins the §2c index manifest: the one place a reader can +// find a type by stored key without a folder convention — which the spec has +// never defined — and without scanning. + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The manifest round-trips byte-stably through the index's own two entry +// points, keys sorted — the canonical form (§4). +// +// How this can fail: drop Manifest from the Index struct or from +// MarshalIndex (the maps vanish on the way round); stop sorting the two +// lookup tables (the second marshal reorders and the byte check goes red); +// or write an empty manifest object (the §4 omit-empty canon breaks and a +// bare `"manifest": {}` ships on every index). +func TestIndex_ManifestRoundTrip(t *testing.T) { + // given + in := &Index{ + Name: "Corpus", + Manifest: &Manifest{ + Types: map[string]string{"task": "types/bafytask.anyblock.json", "page": "types/bafypage.anyblock.json"}, + Properties: PropertiesFileName, + }, + } + + // when + data, err := MarshalIndex(in) + require.NoError(t, err) + got, err := UnmarshalIndex(data) + require.NoError(t, err) + data2, err := MarshalIndex(got) + require.NoError(t, err) + + // then + assert.Equal(t, string(data), string(data2), "Marshal ∘ Unmarshal must be byte-stable") + require.NotNil(t, got.Manifest) + assert.Equal(t, in.Manifest.Types, got.Manifest.Types) + assert.Equal(t, PropertiesFileName, got.Manifest.Properties) + + // an empty manifest is not written at all + bare, err := MarshalIndex(&Index{Name: "Corpus", Manifest: &Manifest{}}) + require.NoError(t, err) + assert.NotContains(t, string(bare), "manifest") +} + +// The manifest is closed: `additionalProperties: false` on the index root +// already made an undeclared root member invalid, and the manifest's own +// gate extends that inside — an undeclared member here would be a fourth +// place to put a lookup table, unread by every reader. +// +// How this can fail: drop additionalProperties: false from the manifest +// $defs (first case green), or loosen the value type of a lookup table +// (second case green on a path nobody can open). +func TestIndex_ManifestRefusals(t *testing.T) { + t.Run("an undeclared manifest member is refused", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version":2,"manifest":{"templates":{}}}`)) + require.Error(t, err) + }) + t.Run("a non-string path is refused", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version":2,"manifest":{"types":{"task":42}}}`)) + require.Error(t, err) + }) + t.Run("an empty path is refused", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version":2,"manifest":{"properties":""}}`)) + require.Error(t, err) + }) +} + +// The manifest located options until v0.46: 2,641 entries across a 77-space +// export, option object id → the option document's path. +// +// It went because a manifest answers a lookup a reader would otherwise scan +// for, and no reader has that lookup for an option. The dictionary states a +// property's whole vocabulary inline — every option's name, colour, position +// and, since the vocabulary learned `internal_key`, its stored key — so +// everything an option MEANS is in hand before a single document is opened. +// The entries pointed at documents nothing needed to read. +// +// `option_ids` is a different job and is untouched: it carries option OBJECT +// ids, resolved against the IMPORTING space's live store so a value survives +// a rename (§9a), never against the bundle. It never needed a path beside it. +// +// How this can fail: keep writing the member and every index carries a map +// no reader consults; refuse it instead of ignoring it and a bundle written +// last week stops importing. +func TestIndex_ManifestDoesNotLocateOptions(t *testing.T) { + t.Run("export never writes it", func(t *testing.T) { + data, err := MarshalIndex(&Index{ + Name: "Corpus", + Manifest: &Manifest{Types: map[string]string{"task": "types/t.anyblock.json"}, Properties: PropertiesFileName}, + }) + require.NoError(t, err) + assert.NotContains(t, string(data), `"options"`) + assert.Contains(t, string(data), `"types"`, "the lookup a reader DOES have stays") + }) + + // A bundle written before v0.46 is REFUSED rather than quietly ignored, + // which is the opposite of the accept-and-drop rule stale PROPERTIES get + // (§3). The manifest is closed — `additionalProperties: false` — and the + // closure is the point: a manifest names where things are, so a member + // the reader does not understand is a claim it cannot honour. Silently + // ignoring it would leave an author believing their options are located. + // Nothing has shipped on v0.45, so nothing is stranded. + t.Run("a bundle that still carries it is refused, not silently misread", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version":2,"manifest":{ + "options":{"bafyopt1":"relationsOptions/bafyopt1.anyblock.json"}}}`)) + require.Error(t, err, "the manifest is closed (additionalProperties: false)") + assert.Contains(t, err.Error(), "options") + }) +} + +// The manifest binds file blobs (v0.47): file object id → the blob's +// archive-relative path, keys VERBATIM — an id is its own spelling, so +// unlike `types` there is nothing to re-key on either side. The map is the +// `source`-clobber's replacement: the legacy exporter stuffed the blob path +// into the real, editable Source relation on the document itself, and the +// destruction round-tripped through import. Here the document carries no +// path at all and the binding lives where every by-id lookup lives. +// +// How this can fail: drop Files from the Manifest struct or MarshalIndex +// (the binding vanishes on the way round and every blob is orphaned); stop +// sorting the map (the byte check goes red); re-key the ids through the +// type-spelling chain (an id that happens to fold onto a bundled key gets +// rewritten and the entry dangles); or forget empty() (a files-only +// manifest is dropped as "empty" and ships nothing). +func TestIndex_ManifestBindsFileBlobs(t *testing.T) { + // given — two entries, deliberately out of sorted order + in := &Index{ + Name: "Corpus", + Manifest: &Manifest{ + Files: map[string]string{ + "bafyreigp3him": "files/bafyreigp3him.png", + "bafyreiaaaaaa": "files/bafyreiaaaaaa.pdf", + }, + }, + } + + // when + data, err := MarshalIndex(in) + require.NoError(t, err) + got, err := UnmarshalIndex(data) + require.NoError(t, err) + data2, err := MarshalIndex(got) + require.NoError(t, err) + + // then — byte-stable, keys verbatim, sorted; and a files-only manifest + // is NOT empty (empty() must know the third member) + assert.Equal(t, string(data), string(data2), "Marshal ∘ Unmarshal must be byte-stable") + require.NotNil(t, got.Manifest) + assert.Equal(t, in.Manifest.Files, got.Manifest.Files) + assert.Less(t, strings.Index(string(data), "bafyreiaaaaaa"), strings.Index(string(data), "bafyreigp3him"), + "the canonical form sorts the map's keys (§4)") + + t.Run("a non-string or empty blob path is refused", func(t *testing.T) { + _, err := UnmarshalIndex([]byte(`{"version":2,"manifest":{"files":{"bafyx":42}}}`)) + require.Error(t, err) + _, err = UnmarshalIndex([]byte(`{"version":2,"manifest":{"files":{"bafyx":""}}}`)) + require.Error(t, err) + }) +} + +// An empty path VALUE in a manifest table is omitted on the way out, like +// every empty member (§4): writing it produced bytes the index's own +// Unmarshal refuses (minLength on every manifest path) — I1 broken from +// the Go API, reachable by any caller that left a map value blank. +func TestIndex_ManifestOmitsEmptyPaths(t *testing.T) { + data, err := MarshalIndex(&Index{ + Name: "Corpus", + Manifest: &Manifest{ + Types: map[string]string{"task": "types/t.anyblock.json", "ghost": ""}, + Files: map[string]string{"bafyx": ""}, + }, + }) + require.NoError(t, err) + _, err = UnmarshalIndex(data) + require.NoError(t, err, "what Marshal writes, Unmarshal accepts (I1)") + assert.NotContains(t, string(data), "ghost") + assert.NotContains(t, string(data), "bafyx") + assert.Contains(t, string(data), "Task") + + // a manifest whose every entry is empty collapses to no manifest at all + bare, err := MarshalIndex(&Index{Name: "Corpus", Manifest: &Manifest{Files: map[string]string{"bafyx": ""}}}) + require.NoError(t, err) + assert.NotContains(t, string(bare), "manifest") +} diff --git a/pkg/lib/anyblockjson/markdownblocks.go b/pkg/lib/anyblockjson/markdownblocks.go new file mode 100644 index 0000000000..76d00ff3af --- /dev/null +++ b/pkg/lib/anyblockjson/markdownblocks.go @@ -0,0 +1,440 @@ +package anyblockjson + +// markdownblocks.go — the markdown→flat-blocks parser (the +// Phase-5 critical-path build item): block-level markdown slicing into the §4 +// flat run. Inline content is NOT parsed here — a flat block's `text` is +// already §8 inline markup source, and the fragment import (UnmarshalBlocks) +// parses it with the same inline grammar reads render with. So this file owns +// exactly what §3's build-item note scopes: headings, lists and their +// indentation, fences, quotes, dividers and tables; everything inline rides +// the existing codec, keeping authoring and reading on ONE dialect. +// +// Markdown always parses: there is no error path. Unrecognized block-level +// constructs degrade to paragraphs, an unterminated fence runs to the end of +// the input, a malformed table degrades to paragraph lines, and over-deep +// indentation is clamped to the previous block's level + 1 (CommonMark's "a +// level that hasn't been established cannot be opened", the same rule as +// Options.NormalizeIndent). The clamp also respects the two §12 containment +// rules the +1 rule alone would break: a §5 leaf block (divider, table) +// cannot parent, so a deeper line after one stays its sibling; and the F4 +// absolute nesting bound (indent ≤ 32) caps every level. So the produced run +// always satisfies the §4 strict indent rules, the V2 leaf-containment rule +// and the F4 depth bound — the "a run always imports" contract is tested +// through UnmarshalBlocks over every block type this parser can emit. +// +// Deliberate scope bounds (deterministic > clever, recorded for SKILL/docs): +// - ATX headings only (`#`…); `---` after a paragraph is a divider, never a +// setext underline. Levels 4–6 clamp to heading_3 (§5's own alias rule). +// - One quote level: `>` prefixes strip one level; consecutive quote lines +// join into one quote block. No lazy continuation — a plain line after a +// quote starts a new paragraph. +// - List indentation: each 2 leading spaces (or one tab) = one level, +// then clamped. A more-indented plain line under a list item becomes a +// child paragraph of that item. +// - Tables need the `|`-leading header row + `---` separator row; cells are +// inline markup source, `\|` escapes a literal pipe. Ragged rows widen +// the column set (never a validation error). +// - No images-as-file-blocks (file blocks need uploaded file object ids — +// R11), no toggle/callout syntax, no HTML passthrough: all degrade to +// paragraph text. + +import ( + "encoding/json" + "regexp" + "strings" +) + +// mdBlock is one parsed block before JSON encoding. +type mdBlock struct { + indent int + typ string + text string + hasText bool + extra map[string]any // checked, language, … +} + +var ( + mdHeadingRe = regexp.MustCompile(`^(#{1,6})\s+(.*?)\s*#*\s*$`) + mdDividerRe = regexp.MustCompile(`^\s*(?:(?:-\s*){3,}|(?:\*\s*){3,}|(?:_\s*){3,})$`) + mdFenceRe = regexp.MustCompile("^(`{3,}|~{3,})\\s*(\\S*).*$") + // mdFenceLangRe bounds the info string to a language-ish token; noise + // (backtick runs, punctuation soup) is dropped rather than stored. + mdFenceLangRe = regexp.MustCompile(`^[A-Za-z0-9_+#.-]{0,32}$`) + mdBulletRe = regexp.MustCompile(`^([-*+])\s+(.*)$`) + mdNumberRe = regexp.MustCompile(`^(\d{1,9})[.)]\s+(.*)$`) + mdCheckRe = regexp.MustCompile(`^\[( |x|X)\]\s+(.*)$`) + mdTableSepRe = regexp.MustCompile(`^\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$`) +) + +// ParseMarkdownBlocks converts markdown into a flat §4 blocks run (id-less +// JSON block objects with run-relative indents). It never fails — see the +// file comment for the degradation rules. An input of only whitespace +// produces an empty run. +func ParseMarkdownBlocks(md string) []json.RawMessage { + run, _ := ParseMarkdownBlocksLimit(md, 0) + return run +} + +// ParseMarkdownBlocksLimit parses like ParseMarkdownBlocks but stops as soon +// as the run exceeds maxBlocks (0 = unbounded), reporting the excess instead +// of parsing the rest. The markdown payload channel is byte-bounded by its +// schema but a few bytes can encode one block, so callers that cap a blocks +// array must cap the parsed run with the same number — and the early stop +// keeps a maximum-size hostile body from costing an unbounded parse. When +// exceeded is true the returned run holds maxBlocks+1 blocks (proof of the +// excess, not the full parse) and must not be imported. +func ParseMarkdownBlocksLimit(md string, maxBlocks int) (run []json.RawMessage, exceeded bool) { + p := &mdParser{} + lines := strings.Split(strings.ReplaceAll(md, "\r\n", "\n"), "\n") + for _, line := range lines { + p.feed(strings.TrimSuffix(line, "\r")) + if maxBlocks > 0 && len(p.blocks) > maxBlocks { + return encodeMdBlocks(p.blocks[:maxBlocks+1]), true + } + } + p.flush() + if p.inFence { + p.closeFence() + } + if maxBlocks > 0 && len(p.blocks) > maxBlocks { + return encodeMdBlocks(p.blocks[:maxBlocks+1]), true + } + return encodeMdBlocks(p.blocks), false +} + +type mdParser struct { + blocks []mdBlock + + // paragraph accumulator + para []string + paraIndent int + inPara bool + + // quote accumulator + quote []string + quoteIndent int + inQuote bool + + // fence accumulator + inFence bool + fenceMarker string + fenceIndent int + fenceStrip int + fenceLang string + fenceLines []string + + // table accumulator + tableLines []string + tableIndent int + inTable bool +} + +// feed processes one input line. +func (p *mdParser) feed(line string) { + if p.inFence { + // a closing marker tolerates at most 3 leading spaces (CommonMark); + // deeper-indented marker runs are fence CONTENT (an indented markdown + // example inside a fence must not terminate it) + lead := 0 + for lead < len(line) && line[lead] == ' ' { + lead++ + } + trimmed := strings.TrimRight(line[lead:], " \t") + if lead <= 3 && + strings.HasPrefix(trimmed, p.fenceMarker[:1]) && + strings.TrimRight(trimmed, string(p.fenceMarker[0])) == "" && + len(trimmed) >= len(p.fenceMarker) { + p.closeFence() + return + } + p.fenceLines = append(p.fenceLines, stripIndentPrefix(line, p.fenceStrip)) + return + } + + stripped, level, cols := splitMdIndent(line) + + if strings.TrimSpace(stripped) == "" { + p.flush() + return + } + + // fence open + if m := mdFenceRe.FindStringSubmatch(stripped); m != nil { + p.flush() + p.inFence = true + p.fenceMarker = m[1] + p.fenceIndent = level + p.fenceStrip = cols + p.fenceLang = m[2] + if !mdFenceLangRe.MatchString(p.fenceLang) { + p.fenceLang = "" + } + p.fenceLines = nil + return + } + + // heading + if m := mdHeadingRe.FindStringSubmatch(stripped); m != nil { + p.flush() + depth := len(m[1]) + if depth > 3 { + depth = 3 + } + p.emit(mdBlock{indent: level, typ: "heading_" + string(rune('0'+depth)), text: m[2], hasText: true}) + return + } + + // divider (before list: `- - -` has no item content) + if mdDividerRe.MatchString(stripped) { + p.flush() + p.emit(mdBlock{indent: level, typ: "divider"}) + return + } + + // table rows + if strings.HasPrefix(stripped, "|") { + if !p.inTable { + p.flush() + p.inTable = true + p.tableIndent = level + p.tableLines = nil + } + p.tableLines = append(p.tableLines, stripped) + return + } + if p.inTable { + p.flush() + } + + // quote + if strings.HasPrefix(stripped, ">") { + if !p.inQuote { + p.flush() + p.inQuote = true + p.quoteIndent = level + p.quote = nil + } + content := strings.TrimPrefix(stripped, ">") + content = strings.TrimPrefix(content, " ") + p.quote = append(p.quote, content) + return + } + if p.inQuote { + p.flush() + } + + // list items + if m := mdBulletRe.FindStringSubmatch(stripped); m != nil { + p.flush() + rest := m[2] + if cm := mdCheckRe.FindStringSubmatch(rest); cm != nil { + block := mdBlock{indent: level, typ: "checkbox", text: cm[2], hasText: true} + if cm[1] != " " { + block.extra = map[string]any{"checked": true} + } + p.emit(block) + return + } + p.emit(mdBlock{indent: level, typ: "bulleted_list_item", text: rest, hasText: true}) + return + } + if m := mdNumberRe.FindStringSubmatch(stripped); m != nil { + p.flush() + p.emit(mdBlock{indent: level, typ: "numbered_list_item", text: m[2], hasText: true}) + return + } + + // paragraph (join consecutive plain lines) + if p.inPara { + p.para = append(p.para, strings.TrimRight(stripped, " \t")) + return + } + p.inPara = true + p.paraIndent = level + p.para = []string{strings.TrimRight(stripped, " \t")} +} + +// flush closes every open accumulator except the fence (fences only close on +// their marker or at end of input). +func (p *mdParser) flush() { + if p.inPara { + p.emit(mdBlock{indent: p.paraIndent, typ: "paragraph", text: strings.Join(p.para, "\n"), hasText: true}) + p.inPara = false + p.para = nil + } + if p.inQuote { + p.emit(mdBlock{indent: p.quoteIndent, typ: "quote", text: strings.Join(p.quote, "\n"), hasText: true}) + p.inQuote = false + p.quote = nil + } + if p.inTable { + p.emitTable() + p.inTable = false + p.tableLines = nil + } +} + +func (p *mdParser) closeFence() { + block := mdBlock{indent: p.fenceIndent, typ: "code", text: strings.Join(p.fenceLines, "\n"), hasText: true} + if p.fenceLang != "" { + block.extra = map[string]any{"language": p.fenceLang} + } + p.emit(block) + p.inFence = false + p.fenceLines = nil +} + +// emit appends a block, clamping its indent so the run always imports: +// at most the previous block's level + 1 (§4 strict monotonicity, first +// block at 0), never deeper than a §5 leaf predecessor's own level (leaf +// blocks cannot have children — V2), and never past the F4 absolute bound. +func (p *mdParser) emit(b mdBlock) { + max := 0 + if n := len(p.blocks); n > 0 { + prev := p.blocks[n-1] + max = prev.indent + 1 + if leafBlockTypes[prev.typ] { + // a line cannot open a level under a block that cannot have + // children — it stays the leaf's sibling + max = prev.indent + } + } + if max > maxBlockIndent { + max = maxBlockIndent + } + if b.indent > max { + b.indent = max + } + p.blocks = append(p.blocks, b) +} + +// emitTable converts the accumulated `|` lines into a §6.1 table block, or +// degrades them to one paragraph when the separator row is missing. +func (p *mdParser) emitTable() { + if len(p.tableLines) < 2 || !mdTableSepRe.MatchString(p.tableLines[1]) { + p.emit(mdBlock{indent: p.tableIndent, typ: "paragraph", text: strings.Join(p.tableLines, "\n"), hasText: true}) + return + } + header := splitMdRow(p.tableLines[0]) + width := len(header) + rows := make([]map[string]any, 0, len(p.tableLines)-1) + rows = append(rows, map[string]any{"is_header": true, "cells": mdCells(header)}) + for _, line := range p.tableLines[2:] { + cells := splitMdRow(line) + if len(cells) > width { + width = len(cells) + } + rows = append(rows, map[string]any{"cells": mdCells(cells)}) + } + columns := make([]map[string]any, width) + for i := range columns { + columns[i] = map[string]any{} + } + p.emit(mdBlock{indent: p.tableIndent, typ: "table", extra: map[string]any{ + "columns": columns, + "rows": rows, + }}) +} + +// mdCells maps row cells onto the §6.1 cell forms: null for empties, the +// string shorthand otherwise (cells are inline markup source). +func mdCells(cells []string) []any { + out := make([]any, len(cells)) + for i, c := range cells { + if c == "" { + out[i] = nil + continue + } + out[i] = c + } + return out +} + +// splitMdRow splits one `| a | b |` line into trimmed cell strings, +// honouring `\|` escapes. +func splitMdRow(line string) []string { + line = strings.TrimSpace(line) + line = strings.TrimPrefix(line, "|") + line = strings.TrimSuffix(line, "|") + var cells []string + var cur strings.Builder + escaped := false + for _, r := range line { + switch { + case escaped: + if r != '|' { + cur.WriteByte('\\') + } + cur.WriteRune(r) + escaped = false + case r == '\\': + escaped = true + case r == '|': + cells = append(cells, strings.TrimSpace(cur.String())) + cur.Reset() + default: + cur.WriteRune(r) + } + } + if escaped { + cur.WriteByte('\\') + } + cells = append(cells, strings.TrimSpace(cur.String())) + return cells +} + +// splitMdIndent strips leading whitespace and reports the nesting level it +// implies (2 spaces or one tab per level) plus the number of leading +// whitespace characters consumed. +func splitMdIndent(line string) (stripped string, level, cols int) { + spaces := 0 + i := 0 + for ; i < len(line); i++ { + switch line[i] { + case ' ': + spaces++ + case '\t': + spaces += 2 + default: + return line[i:], spaces / 2, i + } + } + return "", 0, i +} + +// stripIndentPrefix removes up to n leading whitespace characters (fence +// content keeps its own deeper indentation). +func stripIndentPrefix(line string, n int) string { + for i := 0; i < n && line != ""; i++ { + if line[0] != ' ' && line[0] != '\t' { + break + } + line = line[1:] + } + return line +} + +// encodeMdBlocks renders parsed blocks as flat JSON block objects (canonical +// omissions: no indent 0, no empty text). +func encodeMdBlocks(blocks []mdBlock) []json.RawMessage { + out := make([]json.RawMessage, 0, len(blocks)) + for _, b := range blocks { + obj := map[string]any{"type": b.typ} + if b.indent > 0 { + obj["indent"] = b.indent + } + if b.hasText && b.text != "" { + obj["text"] = b.text + } + for k, v := range b.extra { + obj[k] = v + } + raw, err := json.Marshal(obj) + if err != nil { + // map[string]any of JSON-safe values cannot fail to marshal + continue + } + out = append(out, raw) + } + return out +} diff --git a/pkg/lib/anyblockjson/markdownblocks_test.go b/pkg/lib/anyblockjson/markdownblocks_test.go new file mode 100644 index 0000000000..47e7978984 --- /dev/null +++ b/pkg/lib/anyblockjson/markdownblocks_test.go @@ -0,0 +1,369 @@ +package anyblockjson + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// decodeRun parses the produced raw blocks into comparable maps. +func decodeRun(t *testing.T, run []json.RawMessage) []map[string]any { + t.Helper() + out := make([]map[string]any, len(run)) + for i, raw := range run { + require.NoError(t, json.Unmarshal(raw, &out[i])) + } + return out +} + +func TestParseMarkdownBlocks(t *testing.T) { + tests := []struct { + name string + md string + want []map[string]any + }{ + { + name: "empty input produces empty run", + md: " \n\n\t\n", + want: []map[string]any{}, + }, + { + name: "paragraphs split on blank lines and join soft wraps", + md: "first line\nsecond line\n\nnext para", + want: []map[string]any{ + {"type": "paragraph", "text": "first line\nsecond line"}, + {"type": "paragraph", "text": "next para"}, + }, + }, + { + name: "atx headings clamp to heading3", + md: "# One\n## Two\n### Three\n#### Four\n###### Six", + want: []map[string]any{ + {"type": "heading_1", "text": "One"}, + {"type": "heading_2", "text": "Two"}, + {"type": "heading_3", "text": "Three"}, + {"type": "heading_3", "text": "Four"}, + {"type": "heading_3", "text": "Six"}, + }, + }, + { + name: "closing hashes stripped", + md: "## Title ##", + want: []map[string]any{{"type": "heading_2", "text": "Title"}}, + }, + { + name: "list kinds", + md: "- bullet\n* star\n+ plus\n1. first\n2) second\n- [ ] open\n- [x] done", + want: []map[string]any{ + {"type": "bulleted_list_item", "text": "bullet"}, + {"type": "bulleted_list_item", "text": "star"}, + {"type": "bulleted_list_item", "text": "plus"}, + {"type": "numbered_list_item", "text": "first"}, + {"type": "numbered_list_item", "text": "second"}, + {"type": "checkbox", "text": "open"}, + {"type": "checkbox", "checked": true, "text": "done"}, + }, + }, + { + name: "nested lists two spaces per level", + md: "- top\n - child\n - grandchild\n- top again", + want: []map[string]any{ + {"type": "bulleted_list_item", "text": "top"}, + {"indent": float64(1), "type": "bulleted_list_item", "text": "child"}, + {"indent": float64(2), "type": "bulleted_list_item", "text": "grandchild"}, + {"type": "bulleted_list_item", "text": "top again"}, + }, + }, + { + name: "three-space numbered nesting clamps to one level", + md: "1. item\n - sub", + want: []map[string]any{ + {"type": "numbered_list_item", "text": "item"}, + {"indent": float64(1), "type": "bulleted_list_item", "text": "sub"}, + }, + }, + { + name: "over-deep jump clamps to previous plus one", + md: "- top\n - way too deep", + want: []map[string]any{ + {"type": "bulleted_list_item", "text": "top"}, + {"indent": float64(1), "type": "bulleted_list_item", "text": "way too deep"}, + }, + }, + { + name: "indented plain line becomes a child paragraph", + md: "- item\n continued note", + want: []map[string]any{ + {"type": "bulleted_list_item", "text": "item"}, + {"indent": float64(1), "type": "paragraph", "text": "continued note"}, + }, + }, + { + name: "first block indent clamps to zero", + md: " indented start", + want: []map[string]any{{"type": "paragraph", "text": "indented start"}}, + }, + { + name: "quote lines join into one quote", + md: "> quoted\n> more\n\nafter", + want: []map[string]any{ + {"type": "quote", "text": "quoted\nmore"}, + {"type": "paragraph", "text": "after"}, + }, + }, + { + name: "code fence with language keeps literal text", + md: "```go\nfunc main() {\n\t# not a heading\n}\n```\nafter", + want: []map[string]any{ + {"type": "code", "language": "go", "text": "func main() {\n\t# not a heading\n}"}, + {"type": "paragraph", "text": "after"}, + }, + }, + { + name: "unterminated fence runs to end of input", + md: "```\nline one\nline two", + want: []map[string]any{{"type": "code", "text": "line one\nline two"}}, + }, + { + name: "dividers", + md: "---\n***\n___\n- - -", + want: []map[string]any{ + {"type": "divider"}, + {"type": "divider"}, + {"type": "divider"}, + {"type": "divider"}, + }, + }, + { + name: "table with header separator", + md: "| Name | Status |\n| --- | --- |\n| Export | done |\n| Import | |", + want: []map[string]any{{ + "type": "table", + "columns": []any{map[string]any{}, map[string]any{}}, + "rows": []any{ + map[string]any{"is_header": true, "cells": []any{"Name", "Status"}}, + map[string]any{"cells": []any{"Export", "done"}}, + map[string]any{"cells": []any{"Import", nil}}, + }, + }}, + }, + { + name: "escaped pipe stays in the cell", + md: "| a\\|b |\n| --- |\n| c |", + want: []map[string]any{{ + "type": "table", + "columns": []any{map[string]any{}}, + "rows": []any{ + map[string]any{"is_header": true, "cells": []any{"a|b"}}, + map[string]any{"cells": []any{"c"}}, + }, + }}, + }, + { + name: "ragged row widens the column set", + md: "| a | b |\n| --- | --- |\n| 1 | 2 | 3 |", + want: []map[string]any{{ + "type": "table", + "columns": []any{map[string]any{}, map[string]any{}, map[string]any{}}, + "rows": []any{ + map[string]any{"is_header": true, "cells": []any{"a", "b"}}, + map[string]any{"cells": []any{"1", "2", "3"}}, + }, + }}, + }, + { + name: "pipe lines without a separator degrade to a paragraph", + md: "| not | a table |\n| just | pipes |", + want: []map[string]any{ + {"type": "paragraph", "text": "| not | a table |\n| just | pipes |"}, + }, + }, + { + name: "inline markup passes through verbatim", + md: "text with **bold**, `code` and [link](anytype://object?objectId=bafyx)", + want: []map[string]any{ + {"type": "paragraph", "text": "text with **bold**, `code` and [link](anytype://object?objectId=bafyx)"}, + }, + }, + { + name: "crlf input", + md: "# Title\r\n\r\nbody\r\n", + want: []map[string]any{ + {"type": "heading_1", "text": "Title"}, + {"type": "paragraph", "text": "body"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := decodeRun(t, ParseMarkdownBlocks(tt.md)) + assert.Equal(t, tt.want, got) + }) + } +} + +// mustImport asserts a parsed run imports through UnmarshalBlocks — the +// "a run always imports" contract. +func mustImport(t *testing.T, run []json.RawMessage) { + t.Helper() + if len(run) == 0 { + return + } + n := 0 + gen := func() string { n++; return fmt.Sprintf("mdgen%02d", n) } + blocks, topIds, err := UnmarshalBlocks(run, Options{GenerateId: gen}) + require.NoError(t, err) + assert.NotEmpty(t, blocks) + assert.NotEmpty(t, topIds) +} + +// markdownForBlockType opens one block of each type the parser can emit — +// the leaf-clamp regression below drives an indented line after each of +// them, so a new emitted type cannot silently reopen the leaf-parent hole. +var markdownForBlockType = map[string]string{ + "paragraph": "plain text", + "heading_1": "# Title", + "heading_2": "## Title", + "heading_3": "### Title", + "quote": "> quoted", + "code": "```go\nx\n```", + "divider": "---", + "bulleted_list_item": "- item", + "numbered_list_item": "1. item", + "checkbox": "- [ ] item", + "table": "| a |\n| - |\n| 1 |", +} + +// TestParseMarkdownBlocksImports proves every parser output is a valid §4 +// fragment run: it must import through UnmarshalBlocks without validation +// errors (the convergence contract — the wrapper's markdown channel rides the +// same fragment pipeline as a blocks payload). +func TestParseMarkdownBlocksImports(t *testing.T) { + samples := []string{ + "# Plan\n\nintro paragraph\n\n- [ ] task one\n - sub\n- [x] task two\n\n```sh\nmake build\n```\n\n> note\n\n---\n\n| a | b |\n| - | - |\n| 1 | 2 |", + " deep start\n- l\n - jumped", + "only text", + "| a |\n| broken", + // leaf-parent regressions: an indented line after a divider or table + // must become a SIBLING, never a child (V2 — leaf blocks cannot parent) + "---\n child line", + "para\n\n---\n\n indented after divider", + "| a |\n| - |\n| 1 |\n under table", + "---\n - x", + "0\n- 8\n - X\n```1X\n0\n```\n>1\n---\n 00000000000000000000", + } + for i, md := range samples { + t.Run(fmt.Sprintf("sample_%d", i), func(t *testing.T) { + mustImport(t, ParseMarkdownBlocks(md)) + }) + } + + t.Run("an indented line after every emitted block type imports", func(t *testing.T) { + for typ, md := range markdownForBlockType { + t.Run(typ, func(t *testing.T) { + run := ParseMarkdownBlocks(md + "\n indented aside") + mustImport(t, run) + blocks := decodeRun(t, run) + require.NotEmpty(t, blocks) + last := blocks[len(blocks)-1] + if leafBlockTypes[typ] { + assert.NotContains(t, last, "indent", + "a line after a leaf %s block must stay its sibling", typ) + } + }) + } + }) + + t.Run("a 40-deep staircase clamps to the 32-level bound and imports", func(t *testing.T) { + var b strings.Builder + for i := 0; i < 40; i++ { + b.WriteString(strings.Repeat(" ", i) + "- item\n") + } + run := ParseMarkdownBlocks(b.String()) + mustImport(t, run) + blocks := decodeRun(t, run) + require.Len(t, blocks, 40) + maxSeen := 0.0 + for _, blk := range blocks { + if v, ok := blk["indent"].(float64); ok && v > maxSeen { + maxSeen = v + } + } + assert.Equal(t, float64(32), maxSeen, "levels past 32 stay siblings at 32") + }) +} + +func TestParseMarkdownBlocksLimit(t *testing.T) { + t.Run("under the cap passes through", func(t *testing.T) { + run, exceeded := ParseMarkdownBlocksLimit("- a\n- b", 256) + assert.False(t, exceeded) + assert.Len(t, run, 2) + }) + t.Run("over the cap stops early and reports the excess", func(t *testing.T) { + md := strings.Repeat("- x\n", 300) + run, exceeded := ParseMarkdownBlocksLimit(md, 256) + assert.True(t, exceeded) + assert.Len(t, run, 257, "the run holds cap+1 blocks as proof, not the full parse") + }) + t.Run("zero means unbounded", func(t *testing.T) { + md := strings.Repeat("- x\n", 300) + run, exceeded := ParseMarkdownBlocksLimit(md, 0) + assert.False(t, exceeded) + assert.Len(t, run, 300) + }) +} + +func TestFenceEdges(t *testing.T) { + t.Run("a noise info string is dropped, a language token kept", func(t *testing.T) { + got := decodeRun(t, ParseMarkdownBlocks("``` ```\nx\n```")) + require.Len(t, got, 1) + assert.NotContains(t, got[0], "language", "a backtick run is not a language") + + got = decodeRun(t, ParseMarkdownBlocks("```c++\nx\n```")) + require.Len(t, got, 1) + assert.Equal(t, "c++", got[0]["language"]) + }) + t.Run("a closing marker indented past 3 spaces is fence content", func(t *testing.T) { + got := decodeRun(t, ParseMarkdownBlocks("```\n ```\nstill code\n```")) + require.Len(t, got, 1) + assert.Equal(t, "code", got[0]["type"]) + assert.Equal(t, " ```\nstill code", got[0]["text"]) + }) + t.Run("a closing marker with up to 3 leading spaces closes", func(t *testing.T) { + got := decodeRun(t, ParseMarkdownBlocks("```\ncode\n ```\nafter")) + require.Len(t, got, 2) + assert.Equal(t, "code", got[0]["type"]) + assert.Equal(t, "code", got[0]["text"]) + assert.Equal(t, "paragraph", got[1]["type"]) + }) +} + +// FuzzMarkdownImports enforces the "a run always imports" contract under +// fuzzing: whatever the markdown, the parsed run must pass UnmarshalBlocks. +// The seeds include the shapes that broke the pre-fix clamp (a divider/table +// followed by a deeper line, an over-deep staircase). +func FuzzMarkdownImports(f *testing.F) { + f.Add("# Plan\n\n- [ ] a\n - b\n\n```sh\nx\n```\n\n| a |\n| - |\n| 1 |") + f.Add("---\n child line") + f.Add("| a |\n| - |\n| 1 |\n under table") + f.Add("0\n- 8\n - X\n```1X\n0\n```\n>1\n---\n 00000000000000000000") + f.Add(strings.Repeat(" ", 40) + "- deep\n" + strings.Repeat("- x\n", 3)) + f.Fuzz(func(t *testing.T, md string) { + if len(md) > 1<<16 { + return + } + run, _ := ParseMarkdownBlocksLimit(md, 512) + if len(run) == 0 { + return + } + n := 0 + gen := func() string { n++; return fmt.Sprintf("fz%04d", n) } + if _, _, err := UnmarshalBlocks(run, Options{GenerateId: gen}); err != nil { + t.Fatalf("parsed run fails import for %q: %v", md, err) + } + }) +} diff --git a/pkg/lib/anyblockjson/missingref_test.go b/pkg/lib/anyblockjson/missingref_test.go new file mode 100644 index 0000000000..c7ebfb44f6 --- /dev/null +++ b/pkg/lib/anyblockjson/missingref_test.go @@ -0,0 +1,691 @@ +package anyblockjson + +// missingref_test.go — the missing-reference rule (§9): a reference to an +// object that does not exist in the SPACE is not written as if it did — a +// SINGULAR slot (block object_id, mention target) rewrites to the +// `_missing_object` sentinel, a LIST slot (objects/files property values, +// `object_types`) drops the entry. And the distinction that makes or breaks +// the rule: "missing from this EXPORT" is not "missing from the space" — +// only the store's own testimony, through the ObjectExistenceResolver +// capability, may move anything. No capability, no change, sentinel included. + +import ( + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// testCid mints a REAL content id from a seed: the shape gate +// (isObjectIdShaped) parses these, unlike the short invented ids the rest of +// the suite uses — which is itself part of the design under test: an id that +// is not CID-shaped can never be declared missing, so the suite's invented +// ids are untouchable by construction. +func testCid(seed string) string { + sum, err := mh.Sum([]byte(seed), mh.SHA2_256, -1) + if err != nil { + panic(err) + } + return cid.NewCidV1(cid.DagCBOR, sum).String() +} + +var ( + liveCid = testCid("live") // in the store, named + untitledCid = testCid("untitled") // in the store, NO name — the ObjectName trap + deadCid = testCid("dead") // in no store row: missing from the space +) + +// testObjectStore answers the two object-namespace questions from one table, +// the pair storeresolver implements: an id present in the map exists — +// possibly UNTITLED (name "") — and every other id does not. +type testObjectStore map[string]string + +func (m testObjectStore) ObjectName(id string) (string, bool) { + n, ok := m[id] + return n, ok && n != "" +} + +func (m testObjectStore) ObjectExists(id string) (exists, known bool) { + _, ok := m[id] + return ok, true +} + +// unansweringStore is a store that failed: known=false on everything. A +// failure to ask is not evidence of absence, so nothing may move. +type unansweringStore struct{} + +func (unansweringStore) ObjectName(string) (string, bool) { return "", false } +func (unansweringStore) ObjectExists(string) (exists, known bool) { return false, false } + +func missingRefStore() testObjectStore { + return testObjectStore{liveCid: "Live Page", untitledCid: ""} +} + +// compactDoc strips all whitespace, so a multi-line canonical array can be +// asserted as one literal string with its order pinned. +func compactDoc(data []byte) string { + return strings.NewReplacer("\n", "", " ", "").Replace(string(data)) +} + +// missingRefOptions wires the capability plus an object format for the +// custom key the fixtures use, collecting warnings. +func missingRefOptions(warnings *[]Issue) Options { + o := Options{ + ResolveFormat: func(key domain.RelationKey) (model.RelationFormat, bool) { + if key == "related" { + return model.RelationFormat_object, true + } + return 0, false + }, + ResolveObjectNames: missingRefStore(), + } + if warnings != nil { + o.OnWarning = func(i Issue) { *warnings = append(*warnings, i) } + } + return o +} + +func blockSnapshot(children ...*model.Block) *model.SmartBlockSnapshotBase { + root := &model.Block{Id: "obj1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + blocks := []*model.Block{root} + for _, b := range children { + root.ChildrenIds = append(root.ChildrenIds, b.Id) + blocks = append(blocks, b) + } + return &model.SmartBlockSnapshotBase{ + Blocks: blocks, + Details: fields(map[string]*types.Value{"id": str("obj1"), "name": str("Host")}), + } +} + +// Every singular block slot — link, bookmark, the file kinds, dataview — +// rewrites a missing target to the sentinel and leaves a live one alone, +// and the output stays a document this package's own Validate accepts (I1). +// +// How this can fail: unhook singularObjectRef from one slot and that slot's +// assertion finds the dead id written as if the object existed; break the +// shape gate and the live short-id slots start rewriting too. +func TestMissingReference_SingularBlockSlots(t *testing.T) { + cases := map[string]*model.Block{ + "link": {Id: "b1", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: deadCid}}}, + "bookmark": {Id: "b1", Content: &model.BlockContentOfBookmark{Bookmark: &model.BlockContentBookmark{ + Url: "https://anytype.io", TargetObjectId: deadCid}}}, + "image": {Id: "b1", Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + Type: model.BlockContentFile_Image, TargetObjectId: deadCid}}}, + "file legacy hash": {Id: "b1", Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + Type: model.BlockContentFile_File, Hash: deadCid}}}, + "dataview": {Id: "b1", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + TargetObjectId: deadCid}}}, + } + for name, block := range cases { + t.Run(name+" target missing from the space", func(t *testing.T) { + // given + var warnings []Issue + opts := missingRefOptions(&warnings) + + // when + data, err := Marshal(model.SmartBlockType_Page, blockSnapshot(block), opts) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (I1)") + assert.Contains(t, string(data), `"object_id": "_missing_object"`) + assert.NotContains(t, string(data), deadCid, "the dead id must not be written as if it existed") + require.Len(t, warnings, 1, "a rewrite destroys the stored id; the warning is its last appearance") + assert.Contains(t, warnings[0].Message, deadCid) + }) + } + + t.Run("a live target is untouched", func(t *testing.T) { + // given + var warnings []Issue + opts := missingRefOptions(&warnings) + link := &model.Block{Id: "b1", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: liveCid}}} + + // when + data, err := Marshal(model.SmartBlockType_Page, blockSnapshot(link), opts) + + // then + require.NoError(t, err) + assert.Contains(t, string(data), liveCid) + assert.NotContains(t, string(data), missingObjectId) + assert.Empty(t, warnings) + }) + + t.Run("a stored sentinel is kept as-is, silently", func(t *testing.T) { + // given — the corpus holds 52 of these in block object_id: the id is + // already gone, so there is nothing to rewrite and nothing to say + var warnings []Issue + opts := missingRefOptions(&warnings) + link := &model.Block{Id: "b1", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: missingObjectId}}} + + // when + data, err := Marshal(model.SmartBlockType_Page, blockSnapshot(link), opts) + + // then + require.NoError(t, err) + assert.Contains(t, string(data), `"object_id": "_missing_object"`) + assert.Empty(t, warnings) + }) + + t.Run("no capability wired: nothing is rewritten", func(t *testing.T) { + // given — a package-only export has no store to ask, and the absence + // of an answer is not evidence of absence; a name-only resolver + // (the pre-capability shape) must not arm the rule either + for name, o := range map[string]Options{ + "bare options": {}, + "name-only resolver": {ResolveObjectNames: testObjectNames{}}, + } { + link := &model.Block{Id: "b1", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: deadCid}}} + + // when + data, err := Marshal(model.SmartBlockType_Page, blockSnapshot(link), o) + + // then + require.NoError(t, err, name) + assert.Contains(t, string(data), deadCid, name) + assert.NotContains(t, string(data), missingObjectId, name) + } + }) + + t.Run("a store that cannot answer moves nothing", func(t *testing.T) { + // given — known=false is a failure to ask, not an answer of no + opts := Options{ResolveObjectNames: unansweringStore{}} + link := &model.Block{Id: "b1", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: deadCid}}} + + // when + data, err := Marshal(model.SmartBlockType_Page, blockSnapshot(link), opts) + + // then + require.NoError(t, err) + assert.Contains(t, string(data), deadCid) + assert.NotContains(t, string(data), missingObjectId) + }) +} + +// A mention is a singular slot inside inline markup: the target rewrites to +// the sentinel, the mention's own text stays, and the snapshot's marks — +// caller-owned state — are never mutated. +func TestMissingReference_MentionTargets(t *testing.T) { + mention := func(param string) *model.Block { + return textBlock("b1", model.BlockContentText_Paragraph, "Ping Roman", + &model.BlockContentTextMark{ + Range: &model.Range{From: 5, To: 10}, + Type: model.BlockContentTextMark_Mention, + Param: param, + }) + } + + t.Run("a missing mention target rewrites to the sentinel", func(t *testing.T) { + // given + var warnings []Issue + opts := missingRefOptions(&warnings) + snap := blockSnapshot(mention(deadCid)) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (I1)") + assert.Contains(t, string(data), `Roman`) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, deadCid) + // the snapshot is caller-owned: the rewrite must be copy-on-write + assert.Equal(t, deadCid, snap.Blocks[1].GetText().Marks.Marks[0].Param, + "exportMarks mutated the caller's snapshot") + }) + + t.Run("live, sentinel and derived targets are untouched", func(t *testing.T) { + // given — a `_date_…` mention targets a VIRTUAL object the space + // index is not the authority for, and a short id is not CID-shaped: + // neither may ever reach the existence question + for name, param := range map[string]string{ + "live": liveCid, + "stored sentinel": missingObjectId, + "date": "_date_2026-08-24", + "short id": "someShortLegacyId", + } { + var warnings []Issue + opts := missingRefOptions(&warnings) + + // when + data, err := Marshal(model.SmartBlockType_Page, blockSnapshot(mention(param)), opts) + + // then + require.NoError(t, err, name) + assert.Contains(t, string(data), ``, name) + assert.Empty(t, warnings, name) + } + }) + + t.Run("table cell shorthand applies the same rule", func(t *testing.T) { + // given — the shorthand renders without going through textToJSON, + // which is exactly how the emit-once bug class started (§11); pin + // that this path was not forgotten + var warnings []Issue + opts := missingRefOptions(&warnings) + cell := textBlock("r1-c1", model.BlockContentText_Paragraph, "Ping Roman", + &model.BlockContentTextMark{ + Range: &model.Range{From: 5, To: 10}, + Type: model.BlockContentTextMark_Mention, + Param: deadCid, + }) + row := &model.Block{Id: "r1", ChildrenIds: []string{"r1-c1"}, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}} + col := &model.Block{Id: "c1", + Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}} + cols := &model.Block{Id: "cols", ChildrenIds: []string{"c1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_TableColumns}}} + rows := &model.Block{Id: "rows", ChildrenIds: []string{"r1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_TableRows}}} + table := &model.Block{Id: "t1", ChildrenIds: []string{"cols", "rows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}} + root := &model.Block{Id: "obj1", ChildrenIds: []string{"t1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{root, table, cols, rows, row, col, cell}, + Details: fields(map[string]*types.Value{"id": str("obj1")}), + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + + // then + require.NoError(t, err) + assert.Contains(t, string(data), `Roman`) + require.Len(t, warnings, 1) + }) +} + +// An objects/files property value is a LIST slot: a missing entry drops — +// the sentinel silently, a real id with a warning that is the id's last +// appearance — and the emptied list stays `[]`, because the key's presence +// is meaningful (§3) and dropping dangling entries must not erase the fact +// that the property was set. +func TestMissingReference_PropertyValueLists(t *testing.T) { + withRelated := func(v *types.Value) *model.SmartBlockSnapshotBase { + snap := blockSnapshot() + snap.Details.Fields["related"] = v + return snap + } + + t.Run("missing entries drop, live entries close ranks", func(t *testing.T) { + // given + var warnings []Issue + opts := missingRefOptions(&warnings) + + // when + data, err := Marshal(model.SmartBlockType_Page, + withRelated(strList(liveCid, deadCid, missingObjectId)), opts) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (I1)") + assert.Contains(t, compactDoc(data), `"related":["`+liveCid+`"]`) + require.Len(t, warnings, 1, "the real id warns; the sentinel — which carries nothing — drops silently") + assert.Contains(t, warnings[0].Message, deadCid) + }) + + t.Run("a list emptied by the drop is written as [], never omitted", func(t *testing.T) { + // given + var warnings []Issue + opts := missingRefOptions(&warnings) + + // when + data, err := Marshal(model.SmartBlockType_Page, withRelated(strList(missingObjectId)), opts) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (I1)") + assert.Contains(t, compactDoc(data), `"related":[]`) + assert.Empty(t, warnings) + }) + + t.Run("an UNTITLED object is not a missing one", func(t *testing.T) { + // given — the trap this capability exists to avoid: ObjectName + // answers false for an object that exists but has no name, and an + // export that read that as nonexistence would drop live references + var warnings []Issue + opts := missingRefOptions(&warnings) + opts.RefNames = true // the name IS asked for — and answers no + + // when + data, err := Marshal(model.SmartBlockType_Page, withRelated(strList(untitledCid)), opts) + + // then + require.NoError(t, err) + assert.Contains(t, compactDoc(data), `"related":["`+untitledCid+`"]`, + "a nameless object exists; existence and namedness are different questions") + assert.Empty(t, warnings) + }) + + t.Run("no capability wired: every entry passes through, sentinel included", func(t *testing.T) { + // given + opts := missingRefOptions(nil) + opts.ResolveObjectNames = nil + + // when + data, err := Marshal(model.SmartBlockType_Page, + withRelated(strList(liveCid, deadCid, missingObjectId)), opts) + + // then + require.NoError(t, err) + assert.Contains(t, compactDoc(data), + `"related":["`+liveCid+`","`+deadCid+`","_missing_object"]`) + }) +} + +// A property document's `object_types` is the same list slot in the type +// namespace (§2d): a resolvable type id becomes its key, a bare key passes +// verbatim — vocabulary, not a reference — and only what the store disowns +// drops. +func TestMissingReference_PropertySettingsObjectTypes(t *testing.T) { + relSnap := func(targets *types.Value) *model.SmartBlockSnapshotBase { + return relationSnapshot(map[string]*types.Value{ + "relationFormat": num(float64(model.RelationFormat_object)), + "relationFormatObjectTypes": targets, + }) + } + relOpts := func(warnings *[]Issue) Options { + o := missingRefOptions(warnings) + o.ResolveProperties = newTypeIdVocabulary() + return o + } + + t.Run("dead id and sentinel drop; live id and bare key survive", func(t *testing.T) { + // given — the corpus shape: 56 properties carry an object id naming + // nothing, type ids from the account where a shipped use case was + // AUTHORED (an object id differs in every space; a type key does not) + var warnings []Issue + opts := relOpts(&warnings) + + // when + data, err := Marshal(model.SmartBlockType_STRelation, + relSnap(strList("typeid-page", deadCid, "wine", missingObjectId)), opts) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (I1)") + assert.Contains(t, compactDoc(data), `"object_types":["Page","wine"]`) + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, deadCid) + assert.Equal(t, "/property_settings/object_types", warnings[0].Path) + }) + + t.Run("a list emptied by the drop stays [], a cleared target set", func(t *testing.T) { + // given + var warnings []Issue + opts := relOpts(&warnings) + + // when + data, err := Marshal(model.SmartBlockType_STRelation, + relSnap(strList(missingObjectId)), opts) + + // then + require.NoError(t, err) + assert.Contains(t, compactDoc(data), `"object_types":[]`) + assert.Empty(t, warnings) + }) + + t.Run("no capability: the §2d verbatim pass-through is unchanged", func(t *testing.T) { + // given — the offline round trip must stay byte-exact: an id the + // store merely could not be asked about is still the stored value's + // meaning (§2d) + opts := Options{ResolveProperties: newTypeIdVocabulary()} + + // when + data, err := Marshal(model.SmartBlockType_STRelation, + relSnap(strList(deadCid, missingObjectId)), opts) + + // then + require.NoError(t, err) + assert.Contains(t, compactDoc(data), `"object_types":["`+deadCid+`","_missing_object"]`) + }) +} + +// The round trip is a fixpoint after one generation: the first export +// rewrites and drops, import stores what was written, and every export +// after that is byte-identical — §11 guarantee 3 under the new rule. +func TestMissingReference_RoundTripStable(t *testing.T) { + // given — a dead singular target AND a dead list entry in one snapshot + snap := blockSnapshot( + &model.Block{Id: "b1", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: deadCid}}}) + snap.Details.Fields["related"] = strList(liveCid, deadCid) + opts := missingRefOptions(nil) + + // when + first, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + sbType, imported, err := Unmarshal(first, opts) + require.NoError(t, err) + second, err := Marshal(sbType, imported, opts) + require.NoError(t, err) + + // then + assert.Equal(t, string(first), string(second), + "Export(Import(Export(S))) must equal Export(S): the rewrite converges in one generation") +} + +// Over the hostile corpus a resolver that declares EVERYTHING missing +// changes nothing: no hostile id is CID-shaped, so the shape gate keeps the +// existence question off every one of them, and the output is byte-identical +// to the package-only export. This is the no-capability equivalence and the +// shape gate pinned together, against the input class built to break ids. +func TestMissingReference_HostileIdsAreUntouchable(t *testing.T) { + everythingMissing := testObjectStore{} // exists=false, known=true for all + for n := 0; n < 100; n++ { + sbType, snap := hostileSnapshot(n) + plain, err1 := Marshal(sbType, snap, Options{}) + armed, err2 := Marshal(sbType, snap, Options{ResolveObjectNames: everythingMissing}) + require.Equal(t, err1 == nil, err2 == nil, "seed %d: the capability changed exportability", n) + if err1 != nil { + continue + } + require.Equal(t, string(plain), string(armed), + "seed %d: a store with no rows moved a non-CID id", n) + } +} + +// The predicate snapshotdiff consults is the export's own; pin its verdicts +// at the seam so the comparator and the codec cannot drift apart. +func TestDroppedMissingObjectRef(t *testing.T) { + armed := Options{ResolveObjectNames: missingRefStore()} + for name, tc := range map[string]struct { + opts Options + entry string + want bool + }{ + "dead cid, capability wired": {armed, deadCid, true}, + "sentinel, capability wired": {armed, missingObjectId, true}, + "live cid": {armed, liveCid, false}, + "untitled but existing": {armed, untitledCid, false}, + "bare type key is vocabulary": {armed, "page", false}, + "date id is virtual": {armed, "_date_2026-08-24", false}, + "no capability, dead cid": {Options{}, deadCid, false}, + "no capability, sentinel": {Options{}, missingObjectId, false}, + "store cannot answer": {Options{ResolveObjectNames: unansweringStore{}}, + deadCid, false}, + } { + assert.Equal(t, tc.want, DroppedMissingObjectRef(tc.opts, tc.entry), name) + } +} + +// guard against a future edit quietly widening the shape gate: the id forms +// this format treats as non-references must never parse as object ids. +func TestIsObjectIdShaped(t *testing.T) { + assert.True(t, isObjectIdShaped(liveCid)) + assert.True(t, isObjectIdShaped(deadCid)) + for _, s := range []string{ + "", missingObjectId, "_date_2026-08-24", "page", "task", + "62a3c8e1f0a9b4d5e6f70123", // a bson id — a custom type key's shape + "_participant_space_identity", + "_otpage", "_brdescription", + strings.Repeat("x", 70), + } { + assert.False(t, isObjectIdShaped(s), s) + } +} + +// A select vocabulary is a list of references too, so the sentinel half of +// the missing-reference rule applies there as well (§9). +// +// The corpus is unambiguous: `"tag": ["_missing_object"]` beside an EMPTY +// `option_ids` legend — the option is gone and not even a name survives to +// show. Before this, 700+ such entries travelled as the literal string +// `_missing_object` in a tag or status value, which reads as a tag NAMED +// "_missing_object". +// +// Only the sentinel drops here, and deliberately not a whole existence +// check: an option id lives in the option namespace, and optionName already +// resolves it or leaves it as written. +// +// How this can fail: drop it on export without teaching snapshotdiff and the +// corpus sweep reports a false failure per entry — the drift that once cost +// 1,344 of them. +func TestMissingRef_ASelectValueDropsTheSentinel(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), + "tag": strList(missingObjectId), + }), + } + + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + assert.NotContains(t, string(data), missingObjectId, + "an option that is gone leaves nothing to write") + assert.Contains(t, compactDoc(data), `"Tag":[]`, + "the key stays: presence is meaningful (§3), only the dead entry goes") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + + t.Run("a live option is untouched beside it", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), + "tag": strList("opt-live", missingObjectId), + }), + } + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(data), "opt-live") + assert.NotContains(t, string(data), missingObjectId) + }) +} + +// tombstoneCid is in the store and DELETED: the row survives stripped to its +// bookkeeping. It exists — ObjectExists says so deliberately — but the image +// behind it is gone. +var tombstoneCid = testCid("tombstone") + +// deletingStore answers the deletion question too. Everything in the map +// exists; the set names which of those rows are tombstones. +type deletingStore struct { + testObjectStore + tombstones map[string]bool +} + +func (d deletingStore) ObjectDeleted(id string) (deleted, known bool) { + if _, ok := d.testObjectStore[id]; !ok { + return false, true + } + return d.tombstones[id], true +} + +// An icon pointing at a file object the space DELETED used to travel as a +// reference that resolves to nothing: 134 bookmark documents in a 77-space +// export carried a favicon whose file object was a tombstone — every one +// confirmed deleted in its own space's store. +// +// It is DROPPED rather than rewritten to the sentinel, and that asymmetry is +// the rule: a link or a mention MUST have a target, so absence there has to +// be spelled; an icon is optional, and an object with no icon is an ordinary +// object. So the icon falls through to whatever channel is left — the same +// fall-through an image that is not an object id already takes. +// +// How this can fail: reuse ObjectExists here and a tombstone reads as live +// again (it is documented to); reach for the sentinel instead of dropping +// and every one of those bookmarks gets an icon that renders as a missing +// object; forget the fall-through and an object that also has a colour loses +// that too. +func TestMissingRef_ADeletedIconImageIsDropped(t *testing.T) { + store := deletingStore{ + testObjectStore: testObjectStore{liveCid: "Favicon", tombstoneCid: ""}, + tombstones: map[string]bool{tombstoneCid: true}, + } + opts := func() Options { return Options{ResolveObjectNames: store} } + + iconOf := func(t *testing.T, image string, extra map[string]*types.Value) string { + t.Helper() + det := map[string]*types.Value{"id": str("o1"), "iconImage": str(image)} + for k, v := range extra { + det[k] = v + } + data, err := Marshal(model.SmartBlockType_Page, + &model.SmartBlockSnapshotBase{Details: fields(det)}, opts()) + require.NoError(t, err) + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + return string(data) + } + + t.Run("a live target still travels", func(t *testing.T) { + assert.Contains(t, iconOf(t, liveCid, nil), liveCid) + }) + + t.Run("a deleted target drops the icon entirely", func(t *testing.T) { + out := iconOf(t, tombstoneCid, nil) + assert.NotContains(t, out, tombstoneCid) + assert.NotContains(t, out, `"icon"`, "and no empty icon is left behind") + assert.NotContains(t, out, missingObjectId, + "an icon is optional, so absence is silence — not the sentinel a link would get") + }) + + t.Run("the remaining channels still answer", func(t *testing.T) { + out := iconOf(t, tombstoneCid, map[string]*types.Value{ + "iconOption": num(3), + }) + assert.NotContains(t, out, tombstoneCid) + assert.Contains(t, out, `"format": "color"`, + "the icon falls through to the colour, as an unwritable image already does") + }) + + // the capability is the only thing that may remove an icon: a store that + // cannot answer, and a package-only export with no store at all, both + // keep it. + t.Run("no capability keeps every icon", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("o1"), "iconImage": str(tombstoneCid)}), + }, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), tombstoneCid) + }) + + t.Run("a store that cannot answer keeps it", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("o1"), "iconImage": str(tombstoneCid)}), + }, Options{ResolveObjectNames: unansweringStore{}}) + require.NoError(t, err) + assert.Contains(t, string(data), tombstoneCid, + "a failure to ask is not evidence of deletion") + }) +} diff --git a/pkg/lib/anyblockjson/namedenum_test.go b/pkg/lib/anyblockjson/namedenum_test.go new file mode 100644 index 0000000000..30c5317c44 --- /dev/null +++ b/pkg/lib/anyblockjson/namedenum_test.go @@ -0,0 +1,252 @@ +package anyblockjson + +// namedenum_test.go — the name-over-number properties beyond the layout keys +// (§3). Each stored key in namedEnumProperties writes its enum's NAME, reads +// the name back to the stored number, refuses an unknown name as an ERROR, +// and passes a raw number through unchanged in both directions. +// +// The error half is the point. Before these keys were named, the name was +// accepted-then-zeroed: `{"layout_align": "center"}` VALIDATED, Unmarshal +// stored the STRING on a number-format detail, and every consumer reading it +// with an int getter silently saw 0 — a warning existed but Validate +// discards warnings, so no caller ever learned. These tests replace that +// behaviour deliberately: same scenario, new rule, pre-freeze and +// corpus-checked — zero of the 26,803 real stored values across the three +// newly named keys is a string, so the promoted error refuses nothing any +// real export carries. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func TestNamedEnum_LayoutAlign(t *testing.T) { + t.Run("export writes the name", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), + "layoutAlign": num(float64(model.Block_AlignCenter)), + }), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"Layout align": "center"`, + "the same four names blocks and view columns spell — one concept, one spelling (§15 #14)") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + }) + + t.Run("import maps the name to the stored number", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"layout_align": "center"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + v := snap.Details.Fields["layoutAlign"] + require.NotNil(t, v) + _, isNum := v.GetKind().(*types.Value_NumberValue) + require.True(t, isNum, "must be stored as a number, not %T", v.GetKind()) + assert.Equal(t, float64(model.Block_AlignCenter), v.GetNumberValue()) + }) + + // This scenario used to be VALID: the string landed on the number-format + // detail and every int getter answered 0 (left). A warning existed, but + // Validate discards warnings — a consumer calling Validate saw a clean + // document and a silently mis-set object. The key is named now, so an + // unknown name is an ERROR that states the vocabulary. + t.Run("an unknown name is refused, naming the vocabulary", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"layout_align": "centre"}}` + err := Validate([]byte(doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/properties/layout_align") + assert.Contains(t, err.Error(), "unknown align") + assert.Contains(t, err.Error(), "'center'", "the refusal names the name that was nearly right") + _, _, unmErr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr, "Unmarshal must reject what Validate rejects (§11 I2)") + }) + + t.Run("a raw number still round-trips", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"layout_align": 2}}` + require.NoError(t, Validate([]byte(doc))) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, float64(model.Block_AlignRight), snap.Details.Fields["layoutAlign"].GetNumberValue()) + }) + + t.Run("a number outside the vocabulary exports as the number", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), + "layoutAlign": num(99), + }), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"Layout align": 99`, + "a stored value outside the vocabulary round-trips as its number rather than being lost") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + }) +} + +// origin and import_type — the object's provenance, named on the format's +// own §2a precedent ("on ordinary objects origin is real provenance and +// stays"). The corpus carried them as the two largest bare-integer enums: +// origin on 15,943 documents spanning all TEN enum values, import_type on +// 8,303 — a reader saw `origin: 7` beside `resolved_layout: "dashboard"` +// with no way to learn that 7 meant anything. +func TestNamedEnum_Provenance(t *testing.T) { + t.Run("export writes the names", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), + "origin": num(float64(model.ObjectOrigin_builtin)), + "importType": num(float64(model.Import_Markdown)), + }), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"Origin": "builtin"`) + assert.Contains(t, string(data), `"Import Type": "markdown"`) + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + }) + + t.Run("import maps the names to the stored numbers", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"origin": "webclipper", "import_type": "obsidian"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, float64(model.ObjectOrigin_webclipper), snap.Details.Fields["origin"].GetNumberValue()) + assert.Equal(t, float64(model.Import_Obsidian), snap.Details.Fields["importType"].GetNumberValue()) + }) + + // The Notion-zero trap, pinned. This document used to be VALID: the + // string "markdown" landed on the number-format detail, and every int + // getter answered 0 — which for this enum is not "unset" but NOTION, a + // false claim about where the object came from. The key is named now, + // so a name is meaningful and a typo is an error. + t.Run("markdown no longer reads as notion", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"import_type": "markdown"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, float64(model.Import_Markdown), snap.Details.Fields["importType"].GetNumberValue(), + "the name means what it says, not the enum's zero") + }) + + t.Run("an unknown origin is refused, naming the vocabulary", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"origin": "clipbord"}}` + err := Validate([]byte(doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/properties/origin") + assert.Contains(t, err.Error(), "unknown origin") + assert.Contains(t, err.Error(), "'clipboard'", "the refusal names the name that was nearly right") + _, _, unmErr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr, "Unmarshal must reject what Validate rejects (§11 I2)") + }) + + t.Run("an unknown import type is refused too", func(t *testing.T) { + err := Validate([]byte(`{"version": 2, "id": "o1", "properties": {"import_type": "md"}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown import type") + assert.Contains(t, err.Error(), "'markdown'") + }) + + t.Run("raw numbers still round-trip", func(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"origin": 7, "import_type": 1}}` + require.NoError(t, Validate([]byte(doc)), "every corpus document carries the pair this way") + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, float64(model.ObjectOrigin_builtin), snap.Details.Fields["origin"].GetNumberValue()) + assert.Equal(t, float64(model.Import_Markdown), snap.Details.Fields["importType"].GetNumberValue()) + }) +} + +// Both provenance vocabularies are TOTAL over their proto enums, the +// formatNames discipline: a member added to the proto without a name here +// would export as a bare integer again, which is the defect this file +// exists to keep closed. +func TestNamedEnum_VocabulariesTotalOverModelEnums(t *testing.T) { + for raw, enumName := range model.ObjectOrigin_name { + assert.NotEmpty(t, originNames.name(model.ObjectOrigin(raw)), + "origin %s (%d) has no §3 name", enumName, raw) + } + for raw, enumName := range model.ImportType_name { + assert.NotEmpty(t, importTypeNames.name(model.ImportType(raw)), + "import type %s (%d) has no §3 name", enumName, raw) + } + for raw, enumName := range model.BlockAlign_name { + assert.NotEmpty(t, alignNames.name(model.BlockAlign(raw)), + "align %s (%d) has no §3 name", enumName, raw) + } + for raw, enumName := range model.ImageKind_name { + assert.NotEmpty(t, imageKindNames.name(model.ImageKind(raw)), + "image kind %s (%d) has no §3 name", enumName, raw) + } +} + +// A file object's `image_kind` says what an image was uploaded FOR. It used +// to travel as the proto's bare integer, so a reader of an export saw `3` +// beside a named `origin` and had no way to learn it meant the image was +// added by a pipeline rather than by a person — on 4,079 documents across +// the 77-space corpus, which is the measured standard the bare-integer keys +// beside it (widgetLayout at 13, headerRelationsLayout at 51) were left on. +// +// Naming it changes nothing a client depends on: the filter that hides +// auto-added images reads `isHiddenDiscovery`, which travels on its own and +// is in lockstep with this key's automatically_added member (4,053 of +// 4,053). This is a change to the READ surface. +// +// How this can fail: name it on the way out and not back in, and every +// import of an exported file object silently loses the kind; leave the enum +// ZERO out of the vocabulary and a future writer of Basic — the app skips +// storing it today — exports a bare 0 again. +func TestNamedEnum_ImageKind(t *testing.T) { + t.Run("export writes the name", func(t *testing.T) { + for kind, want := range map[model.ImageKind]string{ + model.ImageKind_AutomaticallyAdded: "automatically_added", + model.ImageKind_Icon: "icon", + model.ImageKind_Cover: "cover", + model.ImageKind_Basic: "basic", + } { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{ + "id": str("f1"), + "imageKind": num(float64(kind)), + }), + } + data, err := Marshal(model.SmartBlockType_FileObject, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"Image kind": "`+want+`"`, + "the kind is spelled, not left as the proto integer") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + } + }) + + t.Run("import maps the name to the stored number", func(t *testing.T) { + doc := `{"version": 2, "id": "f1", "properties": {"image_kind": "automatically_added"}}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + v := snap.Details.Fields["imageKind"] + require.NotNil(t, v) + _, isNum := v.GetKind().(*types.Value_NumberValue) + require.True(t, isNum, "must be stored as a number, not %T", v.GetKind()) + assert.Equal(t, float64(model.ImageKind_AutomaticallyAdded), v.GetNumberValue()) + }) + + // closed vocabulary: a near-miss is refused by name rather than stored as + // a stray string on a number detail, the accepted-then-zeroed failure + // this whole file exists to prevent. + t.Run("an unknown name is refused", func(t *testing.T) { + doc := `{"version": 2, "id": "f1", "properties": {"image_kind": "Icon"}}` + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, err) + assert.Contains(t, err.Error(), "image_kind") + assert.Contains(t, err.Error(), "'icon'", "the refusal names the vocabulary") + }) +} diff --git a/pkg/lib/anyblockjson/nfcspelling_test.go b/pkg/lib/anyblockjson/nfcspelling_test.go new file mode 100644 index 0000000000..a9bf44c32a --- /dev/null +++ b/pkg/lib/anyblockjson/nfcspelling_test.go @@ -0,0 +1,201 @@ +package anyblockjson + +// nfcspelling_test.go — §3 says a key's spelling is "NFC-normalized, +// otherwise verbatim", and the WRITE half has honoured it all along +// (PropertyLabel/TypeLabel mint NFC labels). This pins the READ half: a +// spelling resolves under its canonical NFC form, so the precomposed and the +// decomposed bytes of one name land on ONE key instead of minting two +// visually indistinguishable properties — the twin a hostile or hand-edited +// document could plant beside a real one, splitting values between them with +// no diagnostic. + +import ( + "errors" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +const ( + // one name, two byte forms: U+00E9 vs "e" + U+0301 + cafeNFC = "Caf\u00e9" + cafeNFD = "Cafe\u0301" +) + +func TestNFCSpelling_ReadPath(t *testing.T) { + t.Run("an NFD spelling resolves to the NFC key", func(t *testing.T) { + // given — no legend, so the term resolves verbatim; verbatim under + // §3 means the canonical NFC form, not the accidental byte form + doc := `{"version":2,"id":"o1","properties":{"` + cafeNFD + `":"x"}}` + + // when + _, snap, err := Unmarshal([]byte(doc), Options{}) + + // then + require.NoError(t, err) + require.NotNil(t, snap.Details) + assert.Contains(t, snap.Details.Fields, cafeNFC) + assert.NotContains(t, snap.Details.Fields, cafeNFD) + }) + + t.Run("a pure-ASCII key is unaffected", func(t *testing.T) { + // given + doc := `{"version":2,"id":"o1","properties":{"priority":"high"}}` + + // when + _, snap, err := Unmarshal([]byte(doc), Options{}) + + // then + require.NoError(t, err) + assert.Contains(t, snap.Details.Fields, "priority") + }) + + t.Run("both forms in one document refuse with both spellings named", func(t *testing.T) { + // given — the two byte forms of one name, resolving onto one key + doc := `{"version":2,"id":"o1","properties":{"` + cafeNFC + `":"a","` + cafeNFD + `":"b"}}` + + // when + _, _, err := Unmarshal([]byte(doc), Options{}) + + // then — refused, not a coin flip over which value survives + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + assert.Contains(t, err.Error(), "normal forms", + "the refusal should say the two spellings are one name in two Unicode normal forms: %v", err) + }) + + t.Run("both forms as member names warn through the C11 channel", func(t *testing.T) { + // given — Validate has no vocabulary and must stay warning-grade + // here: export legitimately writes both forms when the space holds + // both byte forms as stored keys, each its own address + doc := `{"version":2,"id":"o1","property_internal_keys":{"` + + cafeNFC + `":"keyA","` + cafeNFD + `":"keyB"},"properties":{"` + + cafeNFC + `":"a","` + cafeNFD + `":"b"}}` + var warnings []Issue + + // when + err := ValidateWarn([]byte(doc), func(i Issue) { warnings = append(warnings, i) }) + + // then + require.NoError(t, err) + var twinWarnings int + for _, w := range warnings { + if strings.Contains(w.Message, "normal forms") { + twinWarnings++ + } + } + assert.GreaterOrEqual(t, twinWarnings, 2, + "one twin warning per map carrying both forms: %v", warnings) + }) + + t.Run("a legend maps in both directions across normal forms", func(t *testing.T) { + t.Run("NFC legend entry answers an NFD slot", func(t *testing.T) { + // given + doc := `{"version":2,"id":"o1","property_internal_keys":{"` + + cafeNFC + `":"customKey1"},"properties":{"` + cafeNFD + `":"x"}}` + + // when + _, snap, err := Unmarshal([]byte(doc), Options{}) + + // then + require.NoError(t, err) + assert.Contains(t, snap.Details.Fields, "customKey1") + }) + + t.Run("NFD legend entry answers an NFC slot", func(t *testing.T) { + // given + doc := `{"version":2,"id":"o1","property_internal_keys":{"` + + cafeNFD + `":"customKey2"},"properties":{"` + cafeNFC + `":"x"}}` + + // when + _, snap, err := Unmarshal([]byte(doc), Options{}) + + // then + require.NoError(t, err) + assert.Contains(t, snap.Details.Fields, "customKey2") + }) + }) + + t.Run("an NFD stored key still round-trips through its identity legend entry", func(t *testing.T) { + // given — a stored key whose own bytes are decomposed: export spells + // it verbatim and writes the identity legend entry, which binds the + // exact bytes and outranks normalization (legend VALUES are stored + // keys, byte-verbatim always) + snap := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "o1"}}, + cafeNFD: {Kind: &types.Value_StringValue{StringValue: "x"}}, + }}, + } + out, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + + // when + _, got, err := Unmarshal(out, Options{}) + + // then — the stored key keeps its own bytes + require.NoError(t, err) + assert.Contains(t, got.Details.Fields, cafeNFD) + assert.NotContains(t, got.Details.Fields, cafeNFC) + }) + + t.Run("a block key slot resolves an NFD spelling through an NFC legend entry", func(t *testing.T) { + // given — the same choke point serves every slot, not just /properties + doc := `{"version":2,"id":"o1","property_internal_keys":{"` + cafeNFC + `":"customKey3"}, + "blocks":[{"id":"b1","type":"dataview","views":[{"id":"v1","type":"table", + "filters":[{"property":"` + cafeNFD + `","condition":"equal","value":"x"}], + "sorts":[{"property":"` + cafeNFD + `"}]}]}]}` + + // when + _, snap, err := Unmarshal([]byte(doc), Options{}) + + // then + require.NoError(t, err) + var dv *model.BlockContentDataview + for _, b := range snap.Blocks { + if c, ok := b.Content.(*model.BlockContentOfDataview); ok { + dv = c.Dataview + } + } + require.NotNil(t, dv) + require.Len(t, dv.Views, 1) + require.Len(t, dv.Views[0].Filters, 1) + assert.Equal(t, "customKey3", dv.Views[0].Filters[0].RelationKey) + require.Len(t, dv.Views[0].Sorts, 1) + assert.Equal(t, "customKey3", dv.Views[0].Sorts[0].RelationKey) + }) + + t.Run("the type namespace normalizes the same way", func(t *testing.T) { + // given + doc := `{"version":2,"id":"o1","type":"` + cafeNFD + `","type_internal_keys":{"` + + cafeNFC + `":"customType1"}}` + + // when + _, snap, err := Unmarshal([]byte(doc), Options{}) + + // then + require.NoError(t, err) + assert.Contains(t, snap.ObjectTypes, "ot-customType1") + }) + + t.Run("the fragment door's legend normalizes the same way", func(t *testing.T) { + // given + raw := []byte(`[{"property":"` + cafeNFD + `","condition":"equal","value":"x"}]`) + opts := fragFilterOpts() + opts.Legend = Legend{PropertyKeys: map[string]string{cafeNFC: "customKey4"}} + + // when + got, err := UnmarshalFilters(raw, opts) + + // then + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "customKey4", got[0].RelationKey) + }) +} diff --git a/pkg/lib/anyblockjson/objectcensus_test.go b/pkg/lib/anyblockjson/objectcensus_test.go new file mode 100644 index 0000000000..35d35328bf --- /dev/null +++ b/pkg/lib/anyblockjson/objectcensus_test.go @@ -0,0 +1,197 @@ +package anyblockjson + +// objectcensus_test.go pins buildLabelPlan's OBJECT census (export.go, §9a), +// position by position. +// +// Only one population of ids relabels — doc-local block/row/column/view ids — +// but the plan walks a second one to build the `fullIds` avoid-set: every +// OBJECT id the document references, each of which is now spelled verbatim +// somewhere in the output. mintedSuffixLabels' own census counts local ids +// only, so that avoid-set is the sole thing standing between a minted block +// and a label that already names something else in the same document +// (TestExport_CompactLabelCannotTakeAServedId states the rule; this file +// states that every arm of the walk is live). +// +// It is a coverage file, and it exists because of a specific accident that +// nearly happened: 42396b448 deleted a `compactObjectId` call from all +// thirteen of these positions at once, and afterwards the whole census was +// held by ONE of them (a text mark). A surgery that dropped an arm would have +// been silently green. Each subtest below plants the SAME 5-hex-character +// object id at exactly one position, next to a minted block whose 5-character +// suffix is that string, and asserts the block keeps its full id — which it +// can only do while that position feeds the avoid-set. + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +const ( + // censusObject is compactIdMinLen wide and hex-lower, which is exactly + // the shape of a minted label: an object id that a block could steal. + censusObject = "abcde" + // censusBlock is a 24-hex minted block id (isMintedLocalId) whose + // last-5 suffix is censusObject, so it is the relabel candidate that + // wants that very label. + censusBlock = "0000000000000000000abcde" +) + +// censusSnapshot wires one block under a root, with an explicit envelope id +// so the object id under test is the only interesting string in the document. +func censusSnapshot(block *model.Block, details map[string]*types.Value) *model.SmartBlockSnapshotBase { + all := map[string]*types.Value{"id": str("obj1"), "name": str("Ticket")} + for k, v := range details { + all[k] = v + } + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{block.Id}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + block, + }, + Details: fields(all), + } +} + +// censusBlockId exports under CompactBlockLabels and returns the id written +// for the document's single block. It also insists the object id is really +// spelled in the output: a fixture whose object id never reaches the document +// would pass the label assertion for the wrong reason (nothing to collide +// with), which is precisely the trap this file is built to avoid. +func censusBlockId(t *testing.T, snap *model.SmartBlockSnapshotBase) string { + t.Helper() + data, err := Marshal(model.SmartBlockType_Page, snap, Options{CompactBlockLabels: true}) + require.NoError(t, err) + require.NoError(t, Validate(data), "%s", data) + assert.Contains(t, string(data), censusObject, + "the fixture only bites while the object id is spelled in the document:\n%s", data) + + var doc struct { + Blocks []struct { + Id string `json:"id"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + require.Len(t, doc.Blocks, 1, "%s", data) + return doc.Blocks[0].Id +} + +// censusTextBlock is a text block carrying one mark of the given type. +func censusTextBlock(markType model.BlockContentTextMarkType, param string) *model.Block { + return &model.Block{Id: censusBlock, Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "x", Marks: &model.BlockContentTextMarks{ + Marks: []*model.BlockContentTextMark{{ + Range: &model.Range{From: 0, To: 1}, + Type: markType, Param: param}}}}}} +} + +// censusDataviewBlock wraps a dataview whose one property is object-formatted, +// so filter values and custom orders resolve down the object branch. +func censusDataviewBlock(dv *model.BlockContentDataview) *model.Block { + dv.RelationLinks = []*model.RelationLink{{Key: "assignee", Format: model.RelationFormat_object}} + return &model.Block{Id: censusBlock, Content: &model.BlockContentOfDataview{Dataview: dv}} +} + +// TestExport_TheObjectCensusCoversEveryPosition walks every arm of +// buildLabelPlan's object walk. Neutering any single arm — dropping its +// addObject call — makes exactly its subtest fail, because the id it no +// longer reserves is then free for the minted block to take as a label. +func TestExport_TheObjectCensusCoversEveryPosition(t *testing.T) { + cases := map[string]*model.SmartBlockSnapshotBase{ + // --- text marks --- + "a mention mark's target": censusSnapshot( + censusTextBlock(model.BlockContentTextMark_Mention, censusObject), nil), + "an object mark's target": censusSnapshot( + censusTextBlock(model.BlockContentTextMark_Object, censusObject), nil), + "an object URL behind a link mark": censusSnapshot( + censusTextBlock(model.BlockContentTextMark_Link, objectLinkDest(censusObject)), nil), + "a callout's icon image": censusSnapshot(&model.Block{Id: censusBlock, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Text: "x", Style: model.BlockContentText_Callout, IconImage: censusObject}}}, nil), + + // --- file, bookmark, link blocks --- + "a file block's target": censusSnapshot(&model.Block{Id: censusBlock, + Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + TargetObjectId: censusObject, Type: model.BlockContentFile_Image}}}, nil), + "a file block's legacy hash": censusSnapshot(&model.Block{Id: censusBlock, + Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + Hash: censusObject, Type: model.BlockContentFile_Image}}}, nil), + "a bookmark's target": censusSnapshot(&model.Block{Id: censusBlock, + Content: &model.BlockContentOfBookmark{Bookmark: &model.BlockContentBookmark{ + Url: "https://anytype.io", TargetObjectId: censusObject}}}, nil), + "a link block's target": censusSnapshot(&model.Block{Id: censusBlock, + Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: censusObject}}}, nil), + + // --- dataview --- + "a dataview's target": censusSnapshot(censusDataviewBlock( + &model.BlockContentDataview{TargetObjectId: censusObject}), nil), + "a view's default template": censusSnapshot(censusDataviewBlock( + &model.BlockContentDataview{Views: []*model.BlockContentDataviewView{ + {Id: "v1", Name: "All", DefaultTemplateId: censusObject}}}), nil), + "a view's default type": censusSnapshot(censusDataviewBlock( + &model.BlockContentDataview{Views: []*model.BlockContentDataviewView{ + {Id: "v1", Name: "All", DefaultObjectTypeId: censusObject}}}), nil), + "an object-valued filter": censusSnapshot(censusDataviewBlock( + &model.BlockContentDataview{Views: []*model.BlockContentDataviewView{ + {Id: "v1", Name: "All", Filters: []*model.BlockContentDataviewFilter{{ + Id: "f1", RelationKey: "assignee", Format: model.RelationFormat_object, + Condition: model.BlockContentDataviewFilter_In, + Value: strList(censusObject)}}}}}), nil), + "an object-valued sort's custom order": censusSnapshot(censusDataviewBlock( + &model.BlockContentDataview{Views: []*model.BlockContentDataviewView{ + {Id: "v1", Name: "All", Sorts: []*model.BlockContentDataviewSort{{ + Id: "s1", RelationKey: "assignee", Format: model.RelationFormat_object, + Type: model.BlockContentDataviewSort_Custom, + CustomOrder: []*types.Value{str(censusObject)}}}}}}), nil), + "a view's object order": censusSnapshot(censusDataviewBlock( + &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{{Id: "v1", Name: "All"}}, + ObjectOrders: []*model.BlockContentDataviewObjectOrder{{ + ViewId: "v1", GroupId: "g1", ObjectIds: []string{censusObject}}}}), nil), + + // --- the envelope --- + "an object-valued property": censusSnapshot( + &model.Block{Id: censusBlock, Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "x"}}}, + map[string]*types.Value{"assignee": strList(censusObject)}), + + // The two typed envelope fields (§2b). The icon's file used to reach + // this set through the property walk, because iconImage is a `file` + // relation — the lift takes it out of that walk, so it needs an arm of + // its own. The cover's file NEVER reached it: coverId is declared + // `longtext`, so the property walk skipped it and a compact label + // equal to a file-backed cover id was always possible. + "the icon's file": censusSnapshot( + &model.Block{Id: censusBlock, Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "x"}}}, + map[string]*types.Value{"iconImage": strList(censusObject)}), + "the cover's file": censusSnapshot( + &model.Block{Id: censusBlock, Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "x"}}}, + map[string]*types.Value{"coverId": str(censusObject), "coverType": num(1)}), + } + for name, snap := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, censusBlock, censusBlockId(t, snap), + "the object id at this position must be reserved, so the minted block cannot label itself with it") + }) + } + + // The collection census is the one position that is not a block or a + // property: the items list lives on Collections, and export lifts it into + // the envelope's `items`. + t.Run("a collection item", func(t *testing.T) { + snap := censusSnapshot(&model.Block{Id: censusBlock, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "x"}}}, nil) + snap.Collections = fields(map[string]*types.Value{storeKeyItems: strList(censusObject)}) + assert.Equal(t, censusBlock, censusBlockId(t, snap), + "a collection item is an object reference like any other") + }) +} diff --git a/pkg/lib/anyblockjson/omittedrelation.go b/pkg/lib/anyblockjson/omittedrelation.go new file mode 100644 index 0000000000..fad73c74b5 --- /dev/null +++ b/pkg/lib/anyblockjson/omittedrelation.go @@ -0,0 +1,405 @@ +package anyblockjson + +// omittedrelation.go — the §2f omission rule: a bundle does not carry a +// relation document whose definition restates the bundled table. +// +// Measured over the 38,061-document corpus: 9,675 of 10,617 relation +// documents are installed copies of the 194 bundled relations, and ~98% of +// them are field-identical to bundle/relations.json — each a ~967-byte +// restatement of `{key, name, format}` every reader already ships. The +// dictionary's `installed` list stands for them (§2f); the composition omits +// the documents; and a reader reconstructs each one from its own table, +// which is exactly what a restore does anyway. +// +// The predicate is FAIL-CLOSED in every direction: a detail key it cannot +// classify keeps the document, a stored value of an alien kind keeps the +// document, a block the format preserves keeps the document. Omission is an +// optimization; keeping a document is never wrong, and a predicate that +// omits one carrying real data would delete that data silently — the +// disqualifying failure for a backup format. + +import ( + "math" + + "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// relationDefinitionKeys are the stored keys that ARE the property's +// definition — what the bundled table states and what an omitted document +// must match, member for member. Everything a relation document carries is +// one of three things: a definition key (compared against the table), an +// install artifact (relationInstallArtifactKeys, any value), or an internal +// key the format never writes (strippedDetailKeys); a key that is none of +// them is real data and keeps the document. +var relationDefinitionKeys = map[string]bool{ + "name": true, + "description": true, + "isHidden": true, + detailKeyRelationFormat: true, + "relationMaxCount": true, + "relationReadonlyValue": true, + "relationDefaultValue": true, + detailKeyRelationFormatIncludeTime: true, + detailKeyRelationFormatObjectTypes: true, + // relationKey is definition-adjacent: it IS the identity the predicate + // matched the table on, so it can never diverge on an omitted document + // and the reconstruction re-states it exactly + "relationKey": true, +} + +// relationInstallArtifactKeys are the stored details of an installed copy +// that describe the INSTALL rather than the property — re-stamped by the +// next install, so omitting the document loses nothing a reader could act +// on. Every entry passed the §15 #12 admission test individually, against +// the 9,675 bundled-key relation documents in the corpus; the map value +// records the verdict. Keys that were candidates and FAILED the test — they +// keep the document, because they carry something a person did: +// +// - `isUninstalled` (32 docs, all true): the user REMOVED this property +// from the space; listing its key as installed would undo that. +// - `isFavorite` / `isArchived`: user intent on the relation's own page, +// same verdict §2a reached for a type's isHidden. +// - `includeTime` — the BARE spelling, 7 docs: an orphan detail beside +// relationFormatIncludeTime that no admission evidence explains; a key +// the test cannot explain keeps the document, by the fail-closed rule. +var relationInstallArtifactKeys = map[string]string{ + // 10,617 of 10,617 docs, ONE distinct value each ("relation"): derivable + // from the kind — the §2a layout verdict, on the other kind + "layout": "one distinct value, derivable from the kind", + "resolvedLayout": "one distinct value, derivable from the kind", + // how the INSTALL happened (builtin/usecase/api), not what the property + // is — §2a's origin verdict + "origin": "install provenance, not the property's definition", + // an install timestamp at best — §2a's addedDate verdict + "addedDate": "install timestamp", + // the bundled url this copy was installed from — derivable from the key + // (`_br`) + "sourceObject": "install artifact, derivable from the property key", + // the bundled-table revision at install time; absent, the system re-runs + // the bundled migrations and restamps it — §2a's revision verdict + "revision": "bundled migration marker, restamped on install", + // the moment the installed COPY was created — an install artifact, not + // user data: nobody authored a bundled relation into the space (§2f) + "createdDate": "the install moment of the copy, not user data", + // restamped whenever the install machinery touches the copy; follows + // createdDate + "lastModifiedDate": "restamped by the install machinery", + // derived from the bundled definition at install: measured, 154 bundled + // keys carry one across 9,675 copies and NOT ONE key has a second + // distinct value — a per-space fact would + "apiObjectKey": "derived from the bundled definition: 0 of 154 keys carry a second value", + // what the relation OBJECT's page features — an app-version stamp, not + // the definition: 90 of 134 keys carry two different stamps for the SAME + // key across spaces + "featuredRelations": "the copy's page stamp: 90 of 134 keys carry two versions of it", + // the deprecated pre-object-relations scope enum, written by legacy + // installs; nothing reads it (330 docs) + "scope": "deprecated legacy relation scope, unread", + // which importer produced this copy — provenance of the machinery, the + // same family as origin (32 docs) + "importType": "import-machinery provenance", + // a type-schema stamp on an object that defines no type: recommended + // lists are read off TYPE objects only, and a relation is not one + // (141 docs, all three lists together) + "recommendedFeaturedRelations": "a type-schema stamp on an object that defines no type", + "recommendedRelations": "a type-schema stamp on an object that defines no type", + "recommendedHiddenRelations": "a type-schema stamp on an object that defines no type", +} + +// RelationInstallArtifactKey reports a stored detail that describes the +// install of a bundled relation copy rather than the property it defines — +// the keys an omitted document (§2f) loses and the next install re-stamps. +// Exported for the round-trip comparator, which must skip exactly these on +// the way back and nothing else: the predicate is the format's own, not a +// copy, so the comparator and the composition cannot disagree (the miss +// that produced 1,344 false failures in one sweep). +func RelationInstallArtifactKey(key string) bool { + _, ok := relationInstallArtifactKeys[key] + return ok +} + +// InstallStampedDefault reports a definition key carrying its empty default +// — what a reinstall stamps for a member the original copy never stored +// (`isHidden: false`, `object_types: []`). The comparator consults it for +// the added-details direction of an omitted-document round trip: absent and +// empty say the same thing for a definition member with a defined default, +// the same reading that lets the §2a settings follow the omit-empty canon. +// Scoped to definition keys and empty values only, so a reconstruction that +// invents a NON-empty member, or a key outside the definition, still +// reports. +func InstallStampedDefault(key string, v *types.Value) bool { + return relationDefinitionKeys[key] && isEmptySystemValue(v) +} + +// OmittedBundledRelation reports whether a relation snapshot is an installed +// copy whose definition is field-identical to the bundled table — the §2f +// omission rule: the bundle composition writes no document for it, lists its +// key in the dictionary's `installed`, and a reader reconstructs it from the +// table. The returned key is the bundled key the `installed` list carries. +// +// opts matters for one member: relationFormatObjectTypes stores type OBJECT +// ids (objectcreator rewrites bundled urls to derived ids at creation), and +// only the TypeResolver capability can turn them back into the keys the +// table speaks. Without one the comparison runs verbatim, which fails on +// every derived id — fewer omissions, never a wrong one, the same +// degradation every resolver-less path in this format takes. +func OmittedBundledRelation(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase, opts Options) (string, bool) { + if !isPropertySmartBlock(sbType) || base == nil { + return "", false + } + det := base.GetDetails().GetFields() + key := stringDetail(det, "relationKey") + if key == "" { + return "", false + } + rel, err := bundle.GetRelation(domain.RelationKey(key)) + if err != nil { + return "", false + } + if !relationBlocksCarryNothing(base) { + // 19 corpus relation documents carry a dataview or free text on + // their page; a document is the only place that survives + return "", false + } + internal := strippedDetailKeys() + for k := range det { + switch { + case isAttributionProperty(k): + // `creator` and `lastModifiedBy` are in strippedDetailKeys, but + // unlike the rest of that set they are NOT absent from a + // document: export writes the §3 attribution spelling + // `#` for both, so a KEPT copy of this relation would + // have carried them and an omitted one does not. Every one of + // the 10,617 corpus relation documents holds a `creator`. + // + // They are omitted anyway, on their own verdict rather than on + // the internal set's: attribution on an installed copy of a + // bundled relation records WHO RAN THE INSTALL, not who authored + // the property — the bundled original is authored by nobody in + // this space. Same class as createdDate, which + // RelationInstallArtifactKey already covers. + case internal[k]: + // the raw stored value never travels in any document; nothing to + // lose. (Attribution is the exception, handled above.) + case RelationInstallArtifactKey(k): + // re-stamped by the next install, any value + case relationDefinitionKeys[k]: + // compared below, table-side, so an ABSENT stored member is + // judged too + default: + // unclassified is real data — fail closed + return "", false + } + } + if !bundledIdenticalDefinition(det, rel, opts) { + return "", false + } + return key, true +} + +// bundledIdenticalDefinition compares the stored definition members against +// the bundled table, absent-reads-as-zero on both sides — the same reading +// every consumer of these details applies. A stored value of an alien kind +// (a string where a bool belongs, a NULL that presence-mirroring §2d would +// carry) fails the comparison rather than coercing: the reconstruction +// writes the natural kind, so anything else is a difference by definition. +func bundledIdenticalDefinition(det map[string]*types.Value, rel *model.Relation, opts Options) bool { + name, ok := stringDetailOK(det, "name") + if !ok || name != rel.Name { + return false + } + desc, ok := stringDetailOK(det, "description") + if !ok || desc != rel.Description { + return false + } + format, ok := numberDetailOK(det, detailKeyRelationFormat) + if !ok || math.IsNaN(format) || math.IsInf(format, 0) || + format < 0 || format > math.MaxInt32 || model.RelationFormat(int32(format)) != rel.Format { + return false + } + maxCount, ok := numberDetailOK(det, "relationMaxCount") + if !ok || int32(maxCount) != rel.MaxCount || float64(int32(maxCount)) != maxCount { + return false + } + for detailKey, table := range map[string]bool{ + "isHidden": rel.Hidden, + "relationReadonlyValue": rel.ReadOnly, + detailKeyRelationFormatIncludeTime: rel.IncludeTime, + } { + b, ok := boolDetailOK(det, detailKey) + if !ok || b != table { + return false + } + } + if v := det["relationDefaultValue"]; v != nil { + if _, isNull := v.GetKind().(*types.Value_NullValue); !isNull { + if rel.DefaultValue == nil || !proto.Equal(v, rel.DefaultValue) { + return false + } + } + // a stored null is the absence of a default — trimmedWhenEmpty's + // own verdict for this key + } else if rel.DefaultValue != nil { + return false + } + stored := installedTargetKeys(valueStringList(det[detailKeyRelationFormatObjectTypes]), opts) + table := make([]string, 0, len(rel.ObjectTypes)) + for _, u := range rel.ObjectTypes { + if k, err := bundle.TypeKeyFromUrl(u); err == nil { + table = append(table, string(k)) + } else { + table = append(table, u) + } + } + if len(stored) != len(table) { + return false + } + for i := range stored { + if stored[i] != table[i] { + return false + } + } + return true +} + +// installedTargetKeys translates a stored relationFormatObjectTypes list to +// type KEYS: bundled urls directly, derived ids through the TypeResolver +// capability, anything else verbatim — relationTargetKeys' chain (§2d), +// restated here because this path has no exporter to memoize on. +func installedTargetKeys(entries []string, opts Options) []string { + tr, _ := opts.ResolveProperties.(TypeResolver) + out := make([]string, 0, len(entries)) + for _, entry := range entries { + if k, err := bundle.TypeKeyFromUrl(entry); err == nil { + out = append(out, string(k)) + continue + } + if tr != nil { + if k, ok := tr.TypeKeyById(entry); ok && k != "" { + out = append(out, k) + continue + } + } + out = append(out, entry) + } + return out +} + +// relationBlocksCarryNothing reports whether the snapshot's blocks are the +// standard relation-page scaffolding — root, layout, featured-relations, +// title/description text — which the editor regenerates and the format +// already drops as structural (§7). Anything else (a dataview, free text) is +// content only a document can carry. +func relationBlocksCarryNothing(base *model.SmartBlockSnapshotBase) bool { + for _, b := range base.Blocks { + if b == nil { + return false + } + switch c := b.Content.(type) { + case *model.BlockContentOfSmartblock, *model.BlockContentOfLayout, *model.BlockContentOfFeaturedRelations: + case *model.BlockContentOfText: + if c.Text.GetStyle() != model.BlockContentText_Title && + c.Text.GetStyle() != model.BlockContentText_Description { + return false + } + default: + return false + } + } + return true +} + +// InstalledRelationDetails is the import half of the `installed` list: the +// stored details a reader reconstructs for a bundled key, the shape a fresh +// install writes (relationutils.Relation.ToDetails, minus the ids and +// provenance the installer stamps itself). Definition members are written +// even when empty — an install states the whole definition — which is why +// the comparator's added-details direction reads InstallStampedDefault. The +// TypeResolver capability translates the table's bundled type urls into this +// space's derived ids, exactly as objectcreator does on a real install; a +// reader without one keeps the urls, each its own address (§3). +func InstalledRelationDetails(key string, opts Options) (*types.Struct, bool) { + rel, err := bundle.GetRelation(domain.RelationKey(key)) + if err != nil { + return nil, false + } + tr, _ := opts.ResolveProperties.(TypeResolver) + targets := make([]*types.Value, 0, len(rel.ObjectTypes)) + for _, u := range rel.ObjectTypes { + id := u + if k, err := bundle.TypeKeyFromUrl(u); err == nil && tr != nil { + if resolved, ok := tr.TypeIdByKey(string(k)); ok && resolved != "" { + id = resolved + } + } + targets = append(targets, &types.Value{Kind: &types.Value_StringValue{StringValue: id}}) + } + fields := map[string]*types.Value{ + "name": {Kind: &types.Value_StringValue{StringValue: rel.Name}}, + "relationKey": {Kind: &types.Value_StringValue{StringValue: rel.Key}}, + "description": {Kind: &types.Value_StringValue{StringValue: rel.Description}}, + detailKeyRelationFormat: {Kind: &types.Value_NumberValue{ + NumberValue: float64(rel.Format)}}, + "isHidden": {Kind: &types.Value_BoolValue{BoolValue: rel.Hidden}}, + "relationReadonlyValue": {Kind: &types.Value_BoolValue{BoolValue: rel.ReadOnly}}, + "relationMaxCount": {Kind: &types.Value_NumberValue{NumberValue: float64(rel.MaxCount)}}, + detailKeyRelationFormatIncludeTime: {Kind: &types.Value_BoolValue{ + BoolValue: rel.IncludeTime}}, + detailKeyRelationFormatObjectTypes: {Kind: &types.Value_ListValue{ + ListValue: &types.ListValue{Values: targets}}}, + } + if rel.DefaultValue != nil { + fields["relationDefaultValue"] = rel.DefaultValue + } + return &types.Struct{Fields: fields}, true +} + +// typed detail readers: value-or-zero with a kind verdict, so an alien kind +// fails the identity comparison instead of coercing to a zero that happens +// to match the table. + +func stringDetail(det map[string]*types.Value, key string) string { + s, _ := stringDetailOK(det, key) + return s +} + +func stringDetailOK(det map[string]*types.Value, key string) (string, bool) { + v := det[key] + if v == nil { + return "", true + } + k, isString := v.GetKind().(*types.Value_StringValue) + if !isString { + return "", false + } + return k.StringValue, true +} + +func numberDetailOK(det map[string]*types.Value, key string) (float64, bool) { + v := det[key] + if v == nil { + return 0, true + } + k, isNumber := v.GetKind().(*types.Value_NumberValue) + if !isNumber { + return 0, false + } + return k.NumberValue, true +} + +func boolDetailOK(det map[string]*types.Value, key string) (bool, bool) { + v := det[key] + if v == nil { + return false, true + } + k, isBool := v.GetKind().(*types.Value_BoolValue) + if !isBool { + return false, false + } + return k.BoolValue, true +} diff --git a/pkg/lib/anyblockjson/omittedrelation_test.go b/pkg/lib/anyblockjson/omittedrelation_test.go new file mode 100644 index 0000000000..41f89f9e9d --- /dev/null +++ b/pkg/lib/anyblockjson/omittedrelation_test.go @@ -0,0 +1,180 @@ +package anyblockjson + +// omittedrelation_test.go pins the §2f omission rule: which relation +// documents a bundle composition may leave out, and the fail-closed +// discipline that keeps every other one. A predicate that omits a document +// carrying real data deletes that data silently — the disqualifying failure +// for a backup format — so every widening here has to be red first. + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/gogo/protobuf/types" +) + +// installedCopySnapshot builds the snapshot of a field-identical installed +// copy of a bundled relation: the reconstruction's own details plus the +// install provenance a real copy carries. +func installedCopySnapshot(t *testing.T, key string, opts Options) *model.SmartBlockSnapshotBase { + t.Helper() + det, ok := InstalledRelationDetails(key, opts) + require.True(t, ok) + det.Fields["createdDate"] = &types.Value{Kind: &types.Value_NumberValue{NumberValue: 1700000000}} + det.Fields["origin"] = &types.Value{Kind: &types.Value_NumberValue{NumberValue: 2}} + det.Fields["sourceObject"] = &types.Value{Kind: &types.Value_StringValue{StringValue: "_br" + key}} + det.Fields["layout"] = &types.Value{Kind: &types.Value_NumberValue{NumberValue: float64(model.ObjectType_relation)}} + return &model.SmartBlockSnapshotBase{Details: det} +} + +// A field-identical installed copy is omitted, install provenance +// notwithstanding: the artifact keys may hold ANY value, because the next +// install re-stamps them (§2f). And the reconstruction the reader builds +// from the `installed` key states the table's own facts. +// +// How this can fail: drop an artifact key (createdDate, origin, …) from +// relationInstallArtifactKeys — the copy stops being omittable and the +// first assertion goes red; or make InstalledRelationDetails restate the +// key instead of the table's name, and the anchor assertion catches the +// reconstruction drifting from the table. +func TestOmittedBundledRelation_IdenticalCopyOmits(t *testing.T) { + // given + base := installedCopySnapshot(t, "dueDate", Options{}) + + // when + key, omitted := OmittedBundledRelation(model.SmartBlockType_STRelation, base, Options{}) + + // then + require.True(t, omitted) + assert.Equal(t, "dueDate", key) + // the reconstruction anchors to the TABLE, not to the copy + det, ok := InstalledRelationDetails("dueDate", Options{}) + require.True(t, ok) + assert.Equal(t, "Due date", det.Fields["name"].GetStringValue()) + assert.Equal(t, float64(model.RelationFormat_date), det.Fields["relationFormat"].GetNumberValue()) + assert.Equal(t, float64(1), det.Fields["relationMaxCount"].GetNumberValue()) +} + +// Everything that must KEEP the document, case by case — the fail-closed +// half of the rule. Each case is one way real data could hide in a relation +// document, and each mutation that would lose it is named. +// +// How this can fail: add the unclassified key to the artifact map (its case +// goes red — that is the admission test running in reverse); compare a +// definition field against the copy instead of the table (the divergent +// cases go red); read an alien-kinded value through a coercing getter (the +// alien-kind case); or stop looking at blocks (the dataview case). +func TestOmittedBundledRelation_FailClosed(t *testing.T) { + strVal := func(s string) *types.Value { + return &types.Value{Kind: &types.Value_StringValue{StringValue: s}} + } + for name, mutate := range map[string]func(base *model.SmartBlockSnapshotBase){ + "a divergent name is the §2f rename case": func(base *model.SmartBlockSnapshotBase) { + base.Details.Fields["name"] = strVal("End Date") + }, + "a divergent isHidden is the 132-document case": func(base *model.SmartBlockSnapshotBase) { + base.Details.Fields["isHidden"] = &types.Value{Kind: &types.Value_BoolValue{BoolValue: true}} + }, + "an unclassified key is real data": func(base *model.SmartBlockSnapshotBase) { + base.Details.Fields["somethingNobodyVetted"] = strVal("x") + }, + "isUninstalled is user intent, not an artifact": func(base *model.SmartBlockSnapshotBase) { + base.Details.Fields["isUninstalled"] = &types.Value{Kind: &types.Value_BoolValue{BoolValue: true}} + }, + "an alien-kinded value never coerces to a match": func(base *model.SmartBlockSnapshotBase) { + // GetBoolValue would read this as false == the table's false + base.Details.Fields["isHidden"] = strVal("false") + }, + "a stored null include_time is presence §2d carries": func(base *model.SmartBlockSnapshotBase) { + base.Details.Fields["relationFormatIncludeTime"] = &types.Value{Kind: &types.Value_NullValue{}} + }, + "a dataview block is content only a document carries": func(base *model.SmartBlockSnapshotBase) { + base.Blocks = []*model.Block{{Id: "dv", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{}}}} + }, + "free text on the page is content too": func(base *model.SmartBlockSnapshotBase) { + base.Blocks = []*model.Block{{Id: "t", Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "notes", Style: model.BlockContentText_Paragraph}}}} + }, + "no relationKey, no identity to match": func(base *model.SmartBlockSnapshotBase) { + delete(base.Details.Fields, "relationKey") + }, + } { + t.Run(name, func(t *testing.T) { + base := installedCopySnapshot(t, "dueDate", Options{}) + mutate(base) + _, omitted := OmittedBundledRelation(model.SmartBlockType_STRelation, base, Options{}) + assert.False(t, omitted, "the document must be kept") + }) + } + t.Run("a non-relation kind is never omitted", func(t *testing.T) { + base := installedCopySnapshot(t, "dueDate", Options{}) + _, omitted := OmittedBundledRelation(model.SmartBlockType_Page, base, Options{}) + assert.False(t, omitted) + }) + t.Run("title and description scaffolding does not keep the document", func(t *testing.T) { + base := installedCopySnapshot(t, "dueDate", Options{}) + base.Blocks = []*model.Block{ + {Id: "r", Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "t", Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Text: "Due date", Style: model.BlockContentText_Title}}}, + } + _, omitted := OmittedBundledRelation(model.SmartBlockType_STRelation, base, Options{}) + assert.True(t, omitted, "the editor regenerates the scaffolding; the format drops it as structural (§7)") + }) +} + +// omittedTypeResolver is the TypeResolver capability over one derived id. +type omittedTypeResolver struct { + capturingPropertyResolver + idToKey map[string]string +} + +func (r *omittedTypeResolver) TypeKeyById(id string) (string, bool) { + k, ok := r.idToKey[id] + return k, ok +} + +func (r *omittedTypeResolver) TypeIdByKey(key string) (string, bool) { + for id, k := range r.idToKey { + if k == key { + return id, true + } + } + return "", false +} + +// The store keeps target types as derived OBJECT ids (objectcreator rewrites +// bundled urls at creation), and only the TypeResolver capability can turn +// them back into the keys the bundled table speaks. With it, a copy whose +// targets are derived ids still matches; without it, the comparison runs +// verbatim and the copy is KEPT — fewer omissions, never a wrong one. +// +// How this can fail: drop the TypeResolver arm from installedTargetKeys +// (the with-resolver case stops matching), or "fix" the degradation by +// treating an untranslatable id as its key (the without-resolver case +// starts omitting on a match nobody proved). +func TestOmittedBundledRelation_TargetTypesTranslate(t *testing.T) { + // given: `tasks` targets the task type; the copy stores a derived id + rel, err := bundle.GetRelation(domain.RelationKey("tasks")) + require.NoError(t, err) + require.NotEmpty(t, rel.ObjectTypes) + tr := &omittedTypeResolver{idToKey: map[string]string{"bafyderivedtask": "task"}} + withResolver := Options{ResolveProperties: tr} + + base := installedCopySnapshot(t, "tasks", withResolver) + require.Equal(t, "bafyderivedtask", + base.Details.Fields["relationFormatObjectTypes"].GetListValue().Values[0].GetStringValue(), + "the fixture stores the derived id, as a real space does") + + // when / then + _, omitted := OmittedBundledRelation(model.SmartBlockType_STRelation, base, withResolver) + assert.True(t, omitted, "the resolver inverts the id to the table's key") + + _, omitted = OmittedBundledRelation(model.SmartBlockType_STRelation, base, Options{}) + assert.False(t, omitted, "without the capability the id stays opaque and the document is kept") +} diff --git a/pkg/lib/anyblockjson/optionids_slug_test.go b/pkg/lib/anyblockjson/optionids_slug_test.go new file mode 100644 index 0000000000..6601e2872a --- /dev/null +++ b/pkg/lib/anyblockjson/optionids_slug_test.go @@ -0,0 +1,88 @@ +package anyblockjson + +import ( + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +type slugGuardOptions struct{} + +func (slugGuardOptions) OptionName(key domain.RelationKey, id string) (string, bool) { + if id == "optX" { + return "High", true + } + return "", false +} +func (slugGuardOptions) OptionId(key domain.RelationKey, name string) (string, bool) { + if name == "High" { + return "optX", true + } + return "", false +} + +func dataviewOnKey(key string) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "o1", ChildrenIds: []string{"dv"}, Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dv", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{{Id: "v1", Name: "V", Filters: []*model.BlockContentDataviewFilter{ + {RelationKey: key, Condition: model.BlockContentDataviewFilter_Equal, + Value: &types.Value{Kind: &types.Value_StringValue{StringValue: "optX"}}}, + }}}, + RelationLinks: []*model.RelationLink{{Key: key, Format: model.RelationFormat_status}}, + }}}, + }, + Details: fields(map[string]*types.Value{"id": str("o1")}), + ObjectTypes: []string{"ot-page"}, + } +} + +// propertySlug returns the STORED key verbatim when the vocabulary has no +// spelling for it, and a stored key need not be writable — §3's key rule is a +// deny rule precisely because real stores hold things like +// `completion_status_Not Started`. /properties filters unwritable keys before +// slugging; a dataview FILTER or SORT slot does not, so the option legend +// could take an outer key `propertyNameIssues` refuses, and Marshal emitted a +// document its own Validate and Unmarshal rejected — losing the whole object. +// +// This can only fail if export writes such a key into the legend again: the +// assertions are on Validate and Unmarshal accepting Marshal's own bytes (I1), +// not on the legend's contents, so a legend that silently stops being written +// at all would still have to keep I1 to pass. +func TestOptionIds_AnUnwritableSpellingNeverReachesTheLegend(t *testing.T) { + for name, key := range map[string]string{ + "a control character in the stored key": "a\nb", + "a stored key past the spelling bound": strings.Repeat("y", 140), + } { + t.Run(name, func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, dataviewOnKey(key), Options{ResolveOptions: slugGuardOptions{}}) + require.NoError(t, err) + + require.NoError(t, Validate(data), + "Marshal must not emit a document its own Validate rejects (§11 I1): %s", string(data)) + _, back, err := Unmarshal(data, Options{}) + require.NoError(t, err, "…nor one its own Unmarshal rejects (§11 I1)") + + // the object itself survives — the point of the invariant + require.NotNil(t, back) + assert.NotEmpty(t, back.GetBlocks(), "the whole object was lost, not merely the legend") + }) + } +} + +// the positive control: a WRITABLE spelling in the same slot still earns its +// legend entry, so the guard above cannot pass by suppressing everything. +func TestOptionIds_AWritableSpellingInAFilterStillEarnsItsEntry(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, dataviewOnKey("severity"), Options{ResolveOptions: slugGuardOptions{}}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.Contains(t, string(data), `"option_ids"`, "a filter-only property must still pin its option identity") + assert.Contains(t, string(data), `"severity"`) +} diff --git a/pkg/lib/anyblockjson/optionrefs.go b/pkg/lib/anyblockjson/optionrefs.go new file mode 100644 index 0000000000..18e407381b --- /dev/null +++ b/pkg/lib/anyblockjson/optionrefs.go @@ -0,0 +1,371 @@ +package anyblockjson + +// optionrefs.go — the `option_ids` legend: the id of the option each select +// value NAMES (§3, §9a) — and, with it, the whole of option resolution. +// Export records an entry at the one site that substitutes a name for an id +// (recordOptionRef), and import resolves every select value through the one +// function below (resolveOption), so both halves of "which option does this +// name mean?" are answered in this file and nowhere else. +// +// Select and multi_select values are spelled by name (§3) because a bundle +// carries no option objects — unlike a linked object, which the bundle +// carries and the importer relinks, an option id from another space would +// dangle. Names cost identity in two ways a live account shows, and both were +// measured on a 34 339-object sweep: +// +// 1. Duplicate names. A space may hold two distinct options with one name +// under one relation; name resolution returns the FIRST +// (storeresolver.OptionId scans a list), so 7 objects came back pointing +// at an option they were never on. +// 2. Rename. Export writes the name; if the option is renamed before the +// document is read back, nothing resolves, and the import wiring mints a +// NEW option carrying the stale name — resurrecting the duplicate and +// orphaning the object from the renamed option. +// +// The entry is a HINT, not an address. Import uses the id only when it is a +// live option OF THAT RELATION in the target space, and falls back to name +// resolution otherwise, so a bundle carried to a space that never saw those +// ids keeps working exactly as it does without the legend. That is the +// deliberate difference from `property_internal_keys`/`type_internal_keys`, whose values are +// taken at face value: a stored key IS the address, while an option id is a +// shortcut past a name that is already one (§3). +// +// NESTED, not joined by a separator. The legend is +// {property spelling: {option name: option id}} because a name in this format +// is arbitrary user text and no character can be reserved to join it to its +// scope. The flat spelling this replaced keyed entries `#`, +// and `strcase.ToSnake("C#")` is `c#` — a legal api slug — so an option of a +// property named `C#` had no representable entry at all: the escape hatch was +// unreachable exactly where it was needed. Nesting removes the separator and +// with it the split rule, the two-charset admission rule, and the joined +// key's length bound. The inner key carries no charset rule at all, on +// purpose: it is the same string the value slot already holds, and a legend +// that cannot name a value its own document carries is that same hole one +// level down. + +import ( + "github.com/anyproto/anytype-heart/core/domain" +) + +// +// ---- export ---- +// + +// optionRefPair is one entry before its property has a spelling: export +// records the STORED key, because the term ledger has not necessarily +// finished claiming spellings when a value is written, and renders the +// spelling at emission time (buildOptionIds). +type optionRefPair struct { + key string // stored property key + name string // the option name written into the document +} + +// recordOptionRef notes that a name written into the document stands for a +// particular option id. Called from exactly one place — optionName, the one +// site where export substitutes a name for an id — so the legend covers +// exactly the values that need it and nothing else: there is no pruning pass +// because there is nothing unused to prune (§9a). +// +// FIRST WRITING WINS. Two distinct options of one property sharing one name +// produce one key, and a JSON list of two identical strings has no way to say +// which entry means which option — the collapse §11 already documents for +// name resolution. Keeping the first makes the collapse deterministic, which +// is what export∘import byte-stability needs; dropping the entry instead +// would hand the choice back to the resolver's list order and make a second +// generation differ from the first. +func (e *exporter) recordOptionRef(key, name, id string) { + if key == "" || name == "" || id == "" || name == id { + return + } + if e.optionRefs == nil { + e.optionRefs = map[optionRefPair]string{} + } + pair := optionRefPair{key: key, name: name} + if _, seen := e.optionRefs[pair]; seen { + return + } + e.optionRefs[pair] = id +} + +// buildOptionIds groups the recorded pairs under the document's own property +// spellings. It runs at envelope-assembly time, when every key slot has +// already claimed its term, so the spelling here is the spelling the values +// were written under. +// +// The spelling grouped on is the spelling the slot itself wrote — the +// ledger's answer for a key is memoized — so every outer key this emits is in +// the document's own property census by construction (below), which +// TestInvariant_MarshalOutputValidates checks over the hostile corpus. +// +// Nothing is skipped. Under the flat spelling this replaced, a property whose +// slug carried the separator and an option name past the joined key's bound +// both lost their entry silently; neither residue survives nesting (§11). +// The one thing that can still turn an entry away is a property that claimed +// no spelling at all, which no value in the document can be written under +// either. +// +// The intermediate map is what makes duplicate outer keys structurally +// impossible: the envelope's omap appends blindly, so two stored keys landing +// on one spelling would otherwise write the same JSON key twice. +func (e *exporter) buildOptionIds() map[string]map[string]string { + if len(e.optionRefs) == 0 { + return nil + } + out := map[string]map[string]string{} + for pair, id := range e.optionRefs { + slug := e.propertySlug(pair.key) + // propertySlug hands back the STORED key verbatim when the vocabulary + // has no spelling for it, and a stored key need not be a writable one + // (§3 admits `a\nb`, a 140-character key, whatever the store holds). + // /properties filters those before slugging; a dataview filter or sort + // slot does not, so without this guard the legend takes an outer key + // propertyNameIssues refuses and Marshal emits a document its own + // Validate and Unmarshal reject — losing the whole object, not just + // the legend. The `#` grammar this replaced bounded BOTH halves; only + // dropping the name half was intended. + if slug == "" || !isWritablePropertyKey(slug) { + continue + } + if out[slug] == nil { + out[slug] = map[string]string{} + } + out[slug][pair.name] = id + } + if len(out) == 0 { + return nil + } + return out +} + +// optionIdsFor is the value-level slice of buildOptionIds: the {name: id} +// map recorded for ONE stored key, ungrouped and unslugged, because a +// value-level caller holds the key rather than a document spelling. +func (e *exporter) optionIdsFor(key string) map[string]string { + if key == "" || len(e.optionRefs) == 0 { + return nil + } + out := map[string]string{} + for pair, id := range e.optionRefs { + if pair.key == key { + out[pair.name] = id + } + } + if len(out) == 0 { + return nil + } + return out +} + +// +// ---- import ---- +// + +// resolveOption resolves ONE select value — the whole of §3's three-step +// chain, and the only place any of it lives. Every option slot in the format +// arrives here: property values (import.go) and dataview filter values and +// sort custom orders (dataview.go) alike, which is what makes "how is an +// option value resolved?" a question this file answers by itself. +// +// First answer wins: +// +// 1. the document's own `option_ids` entry, honored only for an id the +// target space still serves as an option of that relation +// (optionIdFromLegend below); +// 2. name resolution through the wired resolver, which is what a bundle +// carried to a space that never saw those ids falls back on; +// 3. the value unchanged, because creating a missing option is the wiring's +// job (§3). +// +// `key` is the stored key the value lands on and `slug` the spelling the slot +// wrote: the resolver is asked with the former and the legend keyed by the +// latter, because the reader that resolves the legend is reading the +// document, not the store. +func (imp *importer) resolveOption(key, slug, name string) string { + if id, ok := imp.optionIdFromLegend(key, slug, name); ok { + return id + } + if imp.opts.ResolveOptions != nil { + if id, ok := imp.opts.ResolveOptions.OptionId(domain.RelationKey(key), name); ok { + return id + } + } + return name +} + +// optionIdFromLegend is step 1 of §3's option resolution: the `option_ids` +// entry, honored only when the id it carries is a live option OF THAT +// RELATION in the target space. The liveness question is the resolver's +// OptionName — it answers for an id exactly when that id is an option of that +// key — which is why a reader with no resolver ignores these entries +// altogether: it has no space to ask, and an id it cannot check is not an +// answer it can give. +// +// There is no reachability precondition left to state. The lookup is indexed +// by the spelling the slot in hand just wrote, so an entry filed under any +// other spelling is never consulted — where the flat spelling needed a census +// to make the key's right half MEAN a property rather than be a string, the +// nesting makes that structural. Validate still takes the census, to warn +// about an entry that can never be consulted (§12); import does not need it. +func (imp *importer) optionIdFromLegend(key, slug, name string) (string, bool) { + if slug == "" || name == "" || imp.opts.ResolveOptions == nil { + return "", false + } + // the slot's exact spelling first, then its §3 canonical NFC form — the + // same two-step every key slot resolves by (propertyKeyIn); option NAMES + // (the inner level) stay byte-exact, they are the value strings + // themselves + id := imp.optionLegend()[slug][name] + if id == "" { + if n := nfcTerm(slug); n != slug { + id = imp.optionLegend()[n][name] + } + } + if id == "" { + return "", false + } + if _, live := imp.opts.ResolveOptions.OptionName(domain.RelationKey(key), id); !live { + return "", false + } + return id, true +} + +// +// ---- the property vocabulary ---- +// + +// An `option_ids` outer key is a PROPERTY SPELLING, and a spelling this +// document never uses qualifies nothing: import indexes the legend by the +// spelling the slot it is resolving wrote, so such an entry is unreachable +// and the value it was written for resolves by name as if the legend were +// absent. Validate reports that (§12), and this is the census of where a +// document can spell a property: +// +// - `properties` — member names (§3) +// - `property_internal_keys` — member names, the spelling→stored-key legend (§3) +// - `type_settings.property_definitions[].property` — §2a +// - a `property` block's `key` (§5) +// - a `link` block's `properties[]`, the shown-property list (§5) +// - a `dataview` block's `properties[].key` (§6.2) +// - a view's `group_by`, `cover_property`, `end_property` (§6.2) +// - a view's `columns[].property` (§6.2) +// - a view's `sorts[].property` (§6.2) +// - a view's `filters[].property`, through nested `filters[]` (§6.2) +// - every block position again inside a table cell, which holds any block +// but a table (§6.1, schema `cellBlock`) +// +// The three that can reach an option value are `properties`, a sort's +// `property` and a filter's `property` (§3 says option values are names +// "everywhere": property values, filter values, custom orders). The rest are +// in the census because the vocabulary is a statement about the DOCUMENT, not +// about one slot — a property a document only groups by is still a property +// it uses — and because a census that tracked the reaching slots alone would +// silently narrow the moment a new slot started resolving options. +// +// A filter's `nested_property` is deliberately not in it: it names a property +// of the object the filter walks TO, not a key slot of this document — the +// importer passes it through without translating it (dataview.go) — and no +// option value is ever resolved under it. +// +// ONE reader, not two. The census used to exist in a decoded twin as well, +// because import took it too, and an agreement test stood between them. Import +// no longer takes a census at all (optionIdFromLegend), so the twin lost its +// only caller — and a function kept alive so a test can check it agrees with +// the one that is actually used proves nothing about behaviour. What the +// agreement test really guarded is that the census covers every position a +// property can be spelled in, and that is pinned directly, position by +// position, in TestOptionRefs_ThePropertyCensusCoversEveryPosition. + +// rawPropertySpellings is the same census over an undecoded document +// (Validate's side). Every read is shape-tolerant: this runs after the schema +// has passed, but a census is not the place to have an opinion about a shape +// somebody else refuses. +func rawPropertySpellings(doc map[string]any) map[string]bool { + out := map[string]bool{} + add := func(spelling string) { + if spelling != "" { + out[spelling] = true + } + } + addString := func(v any) { + s, _ := v.(string) + add(s) + } + addMembers := func(v any) { + m, _ := v.(map[string]any) + for term := range m { + add(term) + } + } + addMembers(doc["properties"]) + addMembers(doc[memberPropertyInternalKeys]) + if list, _ := typePropertyDefinitionsOf(doc); list != nil { + for _, raw := range list { + tp, _ := raw.(map[string]any) + addString(tp[memberProperty]) + } + } + var walkFilters func(v any) + walkFilters = func(v any) { + nodes, _ := v.([]any) + for _, raw := range nodes { + node, _ := raw.(map[string]any) + addString(node[memberProperty]) + walkFilters(node["filters"]) + } + } + var walk func(v any) + walk = func(v any) { + blocks, _ := v.([]any) + for _, raw := range blocks { + block, _ := raw.(map[string]any) + switch typ, _ := block["type"].(string); typ { + case "property": + addString(block[memberProperty]) + case "link": + items, _ := block["properties"].([]any) + for _, item := range items { + addString(item) + } + case "dataview": + items, _ := block["properties"].([]any) + for _, item := range items { + p, _ := item.(map[string]any) + addString(p[memberProperty]) + } + views, _ := block["views"].([]any) + for _, rawView := range views { + view, _ := rawView.(map[string]any) + addString(view["group_by"]) + addString(view["cover_property"]) + addString(view["end_property"]) + columns, _ := view["columns"].([]any) + for _, rawColumn := range columns { + column, _ := rawColumn.(map[string]any) + addString(column[memberProperty]) + } + sorts, _ := view["sorts"].([]any) + for _, rawSort := range sorts { + sortNode, _ := rawSort.(map[string]any) + addString(sortNode[memberProperty]) + } + walkFilters(view["filters"]) + } + } + rows, _ := block["rows"].([]any) + for _, rawRow := range rows { + row, _ := rawRow.(map[string]any) + cells, _ := row["cells"].([]any) + for _, cell := range cells { + switch c := cell.(type) { + case map[string]any: + walk([]any{c}) + case []any: + walk(c) + } + } + } + } + } + walk(doc["blocks"]) + return out +} diff --git a/pkg/lib/anyblockjson/optionrefs_test.go b/pkg/lib/anyblockjson/optionrefs_test.go new file mode 100644 index 0000000000..d8e424a255 --- /dev/null +++ b/pkg/lib/anyblockjson/optionrefs_test.go @@ -0,0 +1,902 @@ +package anyblockjson + +// optionrefs_test.go — the `option_ids` legend (optionrefs.go, §3, §9a). +// +// Every test here is written against a resolver that behaves the way the real +// one does: `spaceOptions` scans a per-relation list and answers with the +// FIRST match by name, which is exactly `storeresolver.OptionId`. That is not +// incidental — the two defects this legend closes are both consequences of +// that scan, so a resolver that answered by a map would make the tests pass +// for the wrong reason. + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// spaceOption is one relation option as a space holds it: an id and a name, +// with nothing enforcing that names are unique. +type spaceOption struct { + id, name string +} + +// spaceOptions is a stand-in for a space's option pool, scanned exactly as +// storeresolver.Resolvers does — first match wins, both directions, and an id +// that is not in the pool has no name (which is how import asks whether an id +// is live here). +type spaceOptions map[domain.RelationKey][]spaceOption + +func (s spaceOptions) OptionName(key domain.RelationKey, id string) (string, bool) { + for _, o := range s[key] { + if o.id == id { + return o.name, true + } + } + return "", false +} + +func (s spaceOptions) OptionId(key domain.RelationKey, name string) (string, bool) { + for _, o := range s[key] { + if o.name == name { + return o.id, true + } + } + return "", false +} + +// selectFormats answers `tag` for every key it is given, so a test can use a +// custom property key and still exercise the select path (§3 format +// resolution). Bundled keys never reach it — the bundle answers first. +func selectFormats(domain.RelationKey) (model.RelationFormat, bool) { + return model.RelationFormat_tag, true +} + +// optionSnapshot is a one-object snapshot carrying select values by id. +func optionSnapshot(props map[string]*types.Value) *model.SmartBlockSnapshotBase { + // an explicit envelope id, so a second generation is comparable byte for + // byte: import mints one for a snapshot that has none (§9, §11.2) + details := map[string]*types.Value{"id": str("bafyreiticket"), "name": str("Ticket")} + for k, v := range props { + details[k] = v + } + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "obj1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(details), + } +} + +// docOptionIds reads the `option_ids` legend out of an exported document. +func docOptionIds(t *testing.T, data []byte) map[string]map[string]string { + t.Helper() + var got struct { + OptionIds map[string]map[string]string `json:"option_ids"` + } + require.NoError(t, json.Unmarshal(data, &got)) + return got.OptionIds +} + +// legend is the nested literal these tests assert against, spelled once. +func legend(slug string, entries map[string]string) map[string]map[string]string { + return map[string]map[string]string{slug: entries} +} + +// docProperty reads one property value out of an exported document. +func docProperty(t *testing.T, data []byte, slug string) any { + t.Helper() + var got struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &got)) + return got.Properties[slug] +} + +// storedList reads a property back off an imported snapshot as a string list. +func storedList(t *testing.T, snap *model.SmartBlockSnapshotBase, key string) []string { + t.Helper() + require.NotNil(t, snap.Details) + return valueStringList(snap.Details.Fields[key]) +} + +// The motivating shape, and the reason the key is qualified rather than the +// bare name: one name, two properties, two different options. A legend keyed +// by the name alone could carry only one of them, and whichever lost would +// have its value silently re-pointed at the other property's option. +func TestOptionRefs_TwoPropertiesShareAnOptionName(t *testing.T) { + // given + space := spaceOptions{ + "status": {{id: "bafyopt1", name: "High"}}, + "tag": {{id: "bafyopt2", name: "High"}}, + } + snap := optionSnapshot(map[string]*types.Value{ + "status": strList("bafyopt1"), + "tag": strList("bafyopt2"), + }) + want := map[string]map[string]string{ + "Status": {"High": "bafyopt1"}, + "Tag": {"High": "bafyopt2"}, + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{ResolveOptions: space}) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data)) + assert.Equal(t, want, docOptionIds(t, data)) + assert.Equal(t, []any{"High"}, docProperty(t, data, "Status")) + assert.Equal(t, []any{"High"}, docProperty(t, data, "Tag")) + + _, back, err := Unmarshal(data, Options{ResolveOptions: space}) + require.NoError(t, err) + assert.Equal(t, []string{"bafyopt1"}, storedList(t, back, "status")) + assert.Equal(t, []string{"bafyopt2"}, storedList(t, back, "tag")) +} + +// Defect 1 of 2, measured: a space holding two options with one name under +// one relation. Name resolution returns the first, so an object sitting on +// the SECOND came back pointing at the first — 7 objects on a 34 339-object +// account. The legend names the option the document was exported from. +func TestOptionRefs_DuplicateNameKeepsTheOptionTheObjectWasOn(t *testing.T) { + // given — the pool lists "books" twice; the object is on the second one + space := spaceOptions{"tag": { + {id: "bafyfirst", name: "books"}, + {id: "bafysecond", name: "books"}, + }} + snap := optionSnapshot(map[string]*types.Value{"tag": strList("bafysecond")}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{ResolveOptions: space}) + require.NoError(t, err) + _, back, err := Unmarshal(data, Options{ResolveOptions: space}) + require.NoError(t, err) + + // then + assert.Equal(t, legend("Tag", map[string]string{"books": "bafysecond"}), docOptionIds(t, data)) + assert.Equal(t, []string{"bafysecond"}, storedList(t, back, "tag")) + // and the name resolution the legend overrides really does answer the + // other option, so this test cannot pass by the fallback agreeing + id, ok := space.OptionId("tag", "books") + require.True(t, ok) + assert.Equal(t, "bafyfirst", id, "the fixture must reproduce the first-match scan") +} + +// The one case the legend cannot rescue, stated so it is not discovered +// later: ONE object holding BOTH same-named options. The document spells +// ["books", "books"] and a JSON list of two identical strings has no way to +// say which entry means which option, so the legend holds the first and both +// values land on it — the collapse §11 already documents for name resolution, +// no worse than today and now deterministic, which is what keeps a second +// export byte-identical to the first. +func TestOptionRefs_SameNameTwiceInOneValueCollapses(t *testing.T) { + // given + space := spaceOptions{"tag": { + {id: "bafyfirst", name: "books"}, + {id: "bafysecond", name: "books"}, + }} + snap := optionSnapshot(map[string]*types.Value{"tag": strList("bafysecond", "bafyfirst")}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{ResolveOptions: space}) + require.NoError(t, err) + _, back, err := Unmarshal(data, Options{ResolveOptions: space}) + require.NoError(t, err) + + // then — the value keeps its arity, the identities collapse onto the + // first one written, and the legend says so out loud + assert.Equal(t, []any{"books", "books"}, docProperty(t, data, "Tag")) + assert.Equal(t, legend("Tag", map[string]string{"books": "bafysecond"}), docOptionIds(t, data)) + assert.Equal(t, []string{"bafysecond", "bafysecond"}, storedList(t, back, "tag")) + + // and the collapse is a FIXPOINT: exporting what came back reproduces the + // document (§11.3). Dropping the entry instead would hand the choice to + // the resolver's list order and make this second generation differ. + again, err := Marshal(model.SmartBlockType_Page, back, Options{ResolveOptions: space}) + require.NoError(t, err) + assert.Equal(t, string(data), string(again)) +} + +// Defect 2 of 2: the option is renamed in the target space before the +// document is read back. Name resolution finds nothing — or, worse, finds a +// DIFFERENT option that has since taken the old name — and the wiring mints a +// third option carrying the stale name. The id wins. +func TestOptionRefs_RenamedOptionResolvesById(t *testing.T) { + // given + source := spaceOptions{"tag": {{id: "bafyorig", name: "High"}}} + target := spaceOptions{"tag": { + {id: "bafyorig", name: "Urgent"}, // renamed since the export + {id: "bafyother", name: "High"}, // and a different option took the name + }} + snap := optionSnapshot(map[string]*types.Value{"tag": strList("bafyorig")}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{ResolveOptions: source}) + require.NoError(t, err) + _, back, err := Unmarshal(data, Options{ResolveOptions: target}) + require.NoError(t, err) + + // then + assert.Equal(t, legend("Tag", map[string]string{"High": "bafyorig"}), docOptionIds(t, data)) + assert.Equal(t, []string{"bafyorig"}, storedList(t, back, "tag"), + "the id names the option the document came from; the name now names another") + id, ok := target.OptionId("tag", "High") + require.True(t, ok) + assert.Equal(t, "bafyother", id, "the fixture must make name resolution answer differently") +} + +// The fallback chain is the point of the whole design: a bundle carried to a +// space that never saw those option ids has to keep working exactly as it +// does without the legend. The id is checked for liveness against the target +// relation, and an id that is not an option there is simply not an answer. +func TestOptionRefs_UnknownIdFallsBackToTheName(t *testing.T) { + // given + source := spaceOptions{"tag": {{id: "bafysource", name: "High"}}} + target := spaceOptions{"tag": {{id: "bafytarget", name: "High"}}} + snap := optionSnapshot(map[string]*types.Value{"tag": strList("bafysource")}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{ResolveOptions: source}) + require.NoError(t, err) + _, back, err := Unmarshal(data, Options{ResolveOptions: target}) + require.NoError(t, err) + + // then + assert.Equal(t, legend("Tag", map[string]string{"High": "bafysource"}), docOptionIds(t, data)) + assert.Equal(t, []string{"bafytarget"}, storedList(t, back, "tag")) +} + +// An id that is live under some OTHER relation is not an answer either: the +// legend qualifies the name with a property, and the liveness question is +// asked of that property's pool. +func TestOptionRefs_IdLiveUnderAnotherRelationIsNotAnAnswer(t *testing.T) { + // given — the target holds bafysource, but as an option of `status` + source := spaceOptions{"tag": {{id: "bafysource", name: "High"}}} + target := spaceOptions{ + "status": {{id: "bafysource", name: "High"}}, + "tag": {{id: "bafytarget", name: "High"}}, + } + snap := optionSnapshot(map[string]*types.Value{"tag": strList("bafysource")}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{ResolveOptions: source}) + require.NoError(t, err) + _, back, err := Unmarshal(data, Options{ResolveOptions: target}) + require.NoError(t, err) + + // then + assert.Equal(t, []string{"bafytarget"}, storedList(t, back, "tag")) +} + +// A name carrying what used to be the separator — `C#`, `#1 priority` — is +// nothing special now: there is no separator to collide with, so the name is +// the inner key character for character. The case is kept because it is the +// one the flat spelling had to reason about (split at the LAST `#`), and the +// hazard is worth a standing regression rather than an argument. +func TestOptionRefs_NameCarryingTheOldSeparator(t *testing.T) { + for _, name := range []string{"C#", "#1 priority", "a#b#c", "#"} { + t.Run(name, func(t *testing.T) { + // given + space := spaceOptions{"language": {{id: "bafyopt", name: name}}} + snap := optionSnapshot(map[string]*types.Value{"language": strList("bafyopt")}) + opts := Options{ResolveOptions: space, ResolveFormat: selectFormats} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + _, back, err := Unmarshal(data, opts) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data)) + assert.Equal(t, legend("language", map[string]string{name: "bafyopt"}), docOptionIds(t, data)) + assert.Equal(t, []string{"bafyopt"}, storedList(t, back, "language")) + }) + } +} + +// An ordinary tag name with a space in it — `import issue` is a real one from +// the account this was measured on — which the deleted plain-label charset +// ([A-Za-z0-9_-]) rejected outright. The inner key carries no charset rule at +// all, which is what makes the legend usable on real option names. +func TestOptionRefs_NameCarryingASpace(t *testing.T) { + // given + space := spaceOptions{"tag": {{id: "bafyopt", name: "import issue"}}} + snap := optionSnapshot(map[string]*types.Value{"tag": strList("bafyopt")}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{ResolveOptions: space}) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data), "a name with a space must be a legal legend key:\n%s", data) + assert.Equal(t, legend("Tag", map[string]string{"import issue": "bafyopt"}), docOptionIds(t, data)) + + _, back, err := Unmarshal(data, Options{ResolveOptions: space}) + require.NoError(t, err) + assert.Equal(t, []string{"bafyopt"}, storedList(t, back, "tag")) +} + +// A value that is neither a known name nor a legend key travels unchanged, as +// it does today: creating a missing option is the wiring's job (§3). +func TestOptionRefs_UnknownValuePassesThrough(t *testing.T) { + // given + space := spaceOptions{"tag": {{id: "bafyopt", name: "High"}}} + doc := `{"version": 2, "id": "obj1", "properties": {"tag": ["Brand new"]}, + "option_ids": {"tag": {"High": "bafyopt"}}}` + + // when + _, back, err := Unmarshal([]byte(doc), Options{ResolveOptions: space, GenerateId: seqIds("g")}) + + // then + require.NoError(t, err) + assert.Equal(t, []string{"Brand new"}, storedList(t, back, "tag")) +} + +// The legend is identity, not compaction, so it is not behind the compaction +// flag — and it is not pruned, because there is nothing unused to prune: the +// only place an entry is recorded is the substitution itself. +// +// OmitIds is the ONE shape that drops it, and that is not an exception to the +// rule but the same rule read the other way: the legend is nothing but ids, so +// a shape that declares itself id-less and then ships a map of them is not one +// (§9). This used to write them, which is what made `rich_omit_ids.json` an +// id-less golden carrying two option ids. +func TestOptionRefs_WrittenWithoutCompactionAndOnlyForWhatIsWritten(t *testing.T) { + // given — one option the resolver knows and one raw id it does not + space := spaceOptions{"tag": {{id: "bafyknown", name: "High"}}} + snap := optionSnapshot(map[string]*types.Value{"tag": strList("bafyknown", "bafyunknown")}) + + for name, tc := range map[string]struct { + opts Options + wantLegend bool + }{ + "plain": {Options{ResolveOptions: space}, true}, + "compact": {Options{ResolveOptions: space, CompactBlockLabels: true}, true}, + "omitIds": {Options{ResolveOptions: space, OmitIds: true}, false}, + } { + t.Run(name, func(t *testing.T) { + // when + data, err := Marshal(model.SmartBlockType_Page, snap, tc.opts) + require.NoError(t, err) + + // then + if tc.wantLegend { + assert.Equal(t, legend("Tag", map[string]string{"High": "bafyknown"}), docOptionIds(t, data), + "the unresolved id is written verbatim and owes no entry") + } else { + assert.Nil(t, docOptionIds(t, data), + "an id-less shape ships no legend of ids (§9)") + assert.NotContains(t, string(data), "bafyknown", + "and the id it would have carried appears nowhere else either") + } + assert.Equal(t, []any{"High", "bafyunknown"}, docProperty(t, data, "Tag")) + }) + } +} + +// Filter values and sort custom orders are option slots too (§3, §6.2), and +// they take the same legend — the rule §3 states is "everywhere", and a slot +// that wrote the name without recording the entry would be a slot whose +// options silently keep the old behavior. +func TestOptionRefs_FilterValuesAndCustomOrders(t *testing.T) { + // given + space := spaceOptions{"tag": { + {id: "bafyfilter", name: "Filtered"}, + {id: "bafyorder", name: "Ordered"}, + }} + dv := &model.BlockContentDataview{ + RelationLinks: []*model.RelationLink{{Key: "tag", Format: model.RelationFormat_tag}}, + Views: []*model.BlockContentDataviewView{{Id: "v1", Name: "All", + Filters: []*model.BlockContentDataviewFilter{{ + Id: "f1", RelationKey: "tag", Format: model.RelationFormat_tag, + Condition: model.BlockContentDataviewFilter_In, + Value: strList("bafyfilter"), + }}, + Sorts: []*model.BlockContentDataviewSort{{ + Id: "s1", RelationKey: "tag", Format: model.RelationFormat_tag, + Type: model.BlockContentDataviewSort_Custom, + CustomOrder: []*types.Value{str("bafyorder")}, + }}, + }}, + } + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"dv1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: dv}}, + }, + Details: fields(map[string]*types.Value{"name": str("Board")}), + } + opts := Options{ResolveOptions: space} + want := legend("Tag", map[string]string{"Filtered": "bafyfilter", "Ordered": "bafyorder"}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + _, back, err := Unmarshal(data, opts) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data)) + assert.Equal(t, want, docOptionIds(t, data)) + view := backView(t, back) + assert.Equal(t, []string{"bafyfilter"}, valueStringList(view.Filters[0].Value)) + require.Len(t, view.Sorts[0].CustomOrder, 1) + assert.Equal(t, "bafyorder", view.Sorts[0].CustomOrder[0].GetStringValue()) +} + +// backView digs the single dataview view out of an imported snapshot. +func backView(t *testing.T, snap *model.SmartBlockSnapshotBase) *model.BlockContentDataviewView { + t.Helper() + for _, b := range snap.Blocks { + if c, ok := b.Content.(*model.BlockContentOfDataview); ok { + require.Len(t, c.Dataview.Views, 1) + return c.Dataview.Views[0] + } + } + t.Fatal("no dataview in the imported snapshot") + return nil +} + +// The legend's OUTER key carries the SPELLING the document writes, not the +// stored key — the reader that resolves it is reading the document and has no +// store to translate with. +func TestOptionRefs_KeyIsTheSpellingNotTheStoredKey(t *testing.T) { + // given — a same-named twin ahead of it in the pool, so name resolution + // answers a DIFFERENT option and only the legend can be right. Without + // it the fallback rescues a key built from the wrong half and the test + // passes while looking up nothing. + space := spaceOptions{"6a32d4856761631534b22f85": { + {id: "bafydecoy", name: "High"}, + {id: "bafyopt", name: "High"}, + }} + snap := optionSnapshot(map[string]*types.Value{"6a32d4856761631534b22f85": strList("bafyopt")}) + opts := Options{ResolveOptions: space, ResolveFormat: selectFormats, Keys: slugVocab{ + slugs: map[string]string{"6a32d4856761631534b22f85": "priority"}, + }} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + _, back, err := Unmarshal(data, opts) + require.NoError(t, err) + + // then + assert.Equal(t, legend("priority", map[string]string{"High": "bafyopt"}), docOptionIds(t, data)) + assert.Equal(t, []string{"bafyopt"}, storedList(t, back, "6a32d4856761631534b22f85")) + id, ok := space.OptionId("6a32d4856761631534b22f85", "High") + require.True(t, ok) + assert.Equal(t, "bafydecoy", id, "the fixture must make name resolution answer differently") +} + +// THE CAPABILITY HOLE, CLOSED. A property whose SPELLING carries a `#` used +// to get no entry at all: `strcase.ToSnake("C#")` is `c#`, a legal api slug, +// and a flat key `#` with a separator on both sides of the split +// was not invertible — so the escape hatch was unreachable exactly where a +// user's own naming needed it, and the value fell back to name resolution in +// silence. Nesting has no separator, so the entry is simply written. +// +// The pool lists a same-named decoy FIRST, so name resolution answers a +// different id: without it the fallback would rescue the value and this test +// would pass whether or not the entry was written. +func TestOptionRefs_SeparatorInThePropertySpellingStillGetsAnEntry(t *testing.T) { + // given + space := spaceOptions{"csharpTag": { + {id: "bafydecoy", name: "High"}, + {id: "bafyopt", name: "High"}, + }} + snap := optionSnapshot(map[string]*types.Value{"csharpTag": strList("bafyopt")}) + opts := Options{ResolveOptions: space, ResolveFormat: selectFormats, Keys: slugVocab{ + slugs: map[string]string{"csharpTag": "c#_lang"}, + }} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + _, back, err := Unmarshal(data, opts) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data)) + assert.Equal(t, legend("c#_lang", map[string]string{"High": "bafyopt"}), docOptionIds(t, data)) + assert.Equal(t, []any{"High"}, docProperty(t, data, "c#_lang")) + assert.Equal(t, []string{"bafyopt"}, storedList(t, back, "csharpTag"), + "the legend, not name resolution, is what carries the identity here") + id, ok := space.OptionId("csharpTag", "High") + require.True(t, ok) + assert.Equal(t, "bafydecoy", id, "the fixture must make name resolution answer differently") +} + +// The other residue nesting removes: an option name past the bound the joined +// key carried. There is no joined key, and the inner key is bounded only by +// being non-empty, so a name of any length gets its entry — as it must, since +// the same string is already sitting in the value slot beside it. +func TestOptionRefs_OverLongNameStillGetsAnEntry(t *testing.T) { + for _, tc := range []struct { + name string + length int + }{ + {"at the old bound", maxPropertyKeyLen}, + {"past the old bound", maxPropertyKeyLen + 1}, + {"far past it", maxPropertyKeyLen * 8}, + } { + t.Run(tc.name, func(t *testing.T) { + // given — a same-named decoy again, so the entry is load-bearing + optName := strings.Repeat("n", tc.length) + space := spaceOptions{"tag": { + {id: "bafydecoy", name: optName}, + {id: "bafyopt", name: optName}, + }} + snap := optionSnapshot(map[string]*types.Value{"tag": strList("bafyopt")}) + opts := Options{ResolveOptions: space} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + _, back, err := Unmarshal(data, opts) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data), "%s", data) + assert.Equal(t, legend("Tag", map[string]string{optName: "bafyopt"}), docOptionIds(t, data)) + assert.Equal(t, []string{"bafyopt"}, storedList(t, back, "tag")) + }) + } +} + +// A reader with no option resolver has no space to ask whether an id is live +// there, and an id it cannot check is not an answer it can give — so the +// entries are ignored and the value passes through as the name, exactly as it +// does today. This is what keeps a package-only read unchanged by the legend. +func TestOptionRefs_ReaderWithoutAResolverIgnoresTheLegend(t *testing.T) { + // given + doc := `{"version": 2, "id": "obj1", "properties": {"tag": ["High"]}, + "option_ids": {"tag": {"High": "bafyopt"}}}` + + // when + _, back, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + + // then + require.NoError(t, err) + assert.Equal(t, []string{"High"}, storedList(t, back, "tag")) +} + +// The published schema and the Go restatement have to admit the same keys, or +// an external validator (§12) and this package disagree about what a document +// is. `option_ids` carries a rule at BOTH levels and the table runs both. +// +// The outer rule is the writable-key rule every property spelling carries. +// The inner rule is only "non-empty", deliberately: an option name is the +// same string the value slot already holds, so anything stricter would refuse +// a legend entry for a value the document itself carries — the `C#` hole one +// level down. +func TestOptionRefs_LegendKeyRulesAreTheSameInBothValidators(t *testing.T) { + for _, tc := range []struct { + name string + slug string + optId string + valid bool + }{ + {"a plain spelling", "tag", "High", true}, + {"a spelling carrying the old separator", "c#_lang", "High", true}, + {"an option name with a space", "tag", "import issue", true}, + {"an option name carrying the old separator", "tag", "C#", true}, + {"an option name that is only the old separator", "tag", "#", true}, + {"an option name past the old joined bound", "tag", strings.Repeat("n", 400), true}, + {"a spelling at the bound", strings.Repeat("p", 128), "High", true}, + {"a spelling past the bound", strings.Repeat("p", 129), "High", false}, + {"an empty spelling", "", "High", false}, + {"a control character in the spelling", "ta\ng", "High", false}, + {"an empty option name", "tag", "", false}, + // the bound counts CHARACTERS in both validators — a byte count + // would put a 65-character Cyrillic spelling past 128 and refuse a + // document the package writes + {"a Cyrillic spelling at the bound", strings.Repeat("\u044f", 128), "High", true}, + {"a Cyrillic spelling past it", strings.Repeat("\u044f", 129), "High", false}, + } { + t.Run(tc.name, func(t *testing.T) { + // given + raw, err := json.Marshal(map[string]any{ + "version": 2, + "option_ids": map[string]any{tc.slug: map[string]string{tc.optId: "bafyreiopt"}}, + }) + require.NoError(t, err) + + // when — ValidateWarn, because an unreachable entry is a WARNING + // and every document here has one: the point is the key rule, not + // the census + schemaErr := ValidateWarn(raw, func(Issue) {}) + + // then + if tc.valid { + assert.NoError(t, schemaErr, "%s", raw) + } else { + assert.Error(t, schemaErr, "%s", raw) + } + }) + } +} + +// The three legends have a canonical order (§2, §4), and `option_ids` is last +// of them because its OUTER keys are property spellings: the legend that +// inverts a spelling has to precede the legend keyed by one, so a reader +// working through the document linearly meets `property_internal_keys` first. +// +// No golden pins this — all four carry `option_ids` and none carries +// `property_internal_keys`, so the two never appear together in a frozen document. +func TestOptionRefs_TheLegendFollowsPropertyKeys(t *testing.T) { + // given — a stored key the bundled table cannot invert, so the document + // owes a property_internal_keys entry, carrying a select value so it owes an + // option_ids entry too + space := spaceOptions{"6a32d4856761631534b22f85": {{id: "bafyopt", name: "High"}}} + snap := optionSnapshot(map[string]*types.Value{"6a32d4856761631534b22f85": strList("bafyopt")}) + opts := Options{ResolveOptions: space, ResolveFormat: selectFormats, Keys: slugVocab{ + slugs: map[string]string{"6a32d4856761631534b22f85": "priority"}, + }} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + + // then + require.NoError(t, Validate(data)) + s := string(data) + properties := strings.Index(s, `"properties"`) + propertyKeys := strings.Index(s, `"property_internal_keys"`) + optionIds := strings.Index(s, `"option_ids"`) + require.NotEqual(t, -1, propertyKeys, "the fixture must produce a property_internal_keys legend:\n%s", s) + require.NotEqual(t, -1, optionIds, "and an option_ids legend:\n%s", s) + assert.Less(t, properties, propertyKeys, "properties precede the legends:\n%s", s) + assert.Less(t, propertyKeys, optionIds, + "option_ids is keyed by spellings property_internal_keys inverts, so it comes after:\n%s", s) + // and the outer key is the SPELLING, which is what makes the order matter + assert.Equal(t, legend("priority", map[string]string{"High": "bafyopt"}), docOptionIds(t, data)) +} + +// slugVocab spells the keys it is given and inverts them, and nothing else — +// a conforming vocabulary just wide enough to move a spelling. +type slugVocab struct { + BundledKeyVocabulary + slugs map[string]string +} + +func (v slugVocab) PropertySlug(key string) string { + if slug, ok := v.slugs[key]; ok { + return slug + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (v slugVocab) PropertyKey(slug string) (string, bool) { + for key, s := range v.slugs { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +// +// ---- the property vocabulary (optionrefs.go) ---- +// + +// optionIdsWarnings keeps the warnings addressed at the legend and drops the +// rest, so a corpus that legitimately warns about something else (an unguarded +// date filter, a tag-shaped literal) cannot make one of these tests pass or +// fail for a reason it is not about. +func optionIdsWarnings(issues []Issue) []Issue { + var out []Issue + for _, i := range issues { + if strings.HasPrefix(i.Path, "/option_ids/") { + out = append(out, i) + } + } + return out +} + +// An outer key naming no property the document uses qualifies nothing: import +// indexes the legend by the spelling the slot it is resolving wrote, so an +// entry filed under `priorty` is never asked for and the value falls back to +// name resolution — the silent degradation the warning exists to say out loud. +// It stays a WARNING: a legend is allowed to carry more than one document +// needs, and hard-rejecting one would contradict that. +// +// The pool lists a same-named DECOY first, so name resolution and the legend +// answer different ids: without it both branches would land on one id and the +// test would pass while asking nothing. +func TestOptionRefs_ALegendEntryForAPropertyTheDocumentDoesNotUse(t *testing.T) { + space := spaceOptions{"tag": { + {id: "bafyname", name: "High"}, // what the NAME resolves to + {id: "bafylegend", name: "High"}, // what the LEGEND names + }} + for _, tc := range []struct { + name string + slug string + wantWarn bool + wantId string + }{ + {"a typo in the property spelling", "priorty", true, "bafyname"}, + {"the spelling the document uses", "tag", false, "bafylegend"}, + } { + t.Run(tc.name, func(t *testing.T) { + // given + doc := fmt.Sprintf(`{"version": 2, "id": "obj1", "properties": {"tag": ["High"]}, + "option_ids": {%q: {"High": "bafylegend"}}}`, tc.slug) + + // when + var warned []Issue + validateErr := ValidateWarn([]byte(doc), func(i Issue) { warned = append(warned, i) }) + _, back, err := Unmarshal([]byte(doc), Options{ResolveOptions: space, GenerateId: seqIds("g")}) + + // then + require.NoError(t, validateErr, "an unreachable entry is a warning, not an error (§9a)") + require.NoError(t, err) + got := optionIdsWarnings(warned) + if tc.wantWarn { + require.Len(t, got, 1, "warnings: %v", warned) + assert.Equal(t, "/option_ids/"+tc.slug, got[0].Path) + assert.Contains(t, got[0].Message, `spells "priorty"`) + } else { + assert.Empty(t, got, "the document spells this property; nothing to report") + } + assert.Equal(t, []string{tc.wantId}, storedList(t, back, "tag")) + }) + } +} + +// The census is a statement about the DOCUMENT, not about one slot: a +// property a document uses only in a dataview filter — or only in a sort's +// custom order — is a property it knows, and the legend under it has to be +// honored. A census that read the `properties` map alone would turn these +// entries away, and it would do it in silence, because the value still +// resolves by name afterwards. So the pool lists a same-named decoy FIRST: +// name resolution answers `bafydecoy`, and only the legend can answer +// `bafylegend`. +// +// Written as documents rather than round trips on purpose. An exported +// dataview carries a `properties` list naming its own properties, which would +// put `tag` in the census by a second route and leave the filter and sort +// positions untested. +func TestOptionRefs_PropertySpelledOnlyInsideADataview(t *testing.T) { + space := spaceOptions{"tag": { + {id: "bafydecoy", name: "High"}, + {id: "bafylegend", name: "High"}, + }} + for _, tc := range []struct { + name string + view string + }{ + {"only in a filter", `{"id": "v1", "filters": + [{"property": "tag", "condition": "in", "value": ["High"]}]}`}, + {"only in a nested filter", `{"id": "v1", "filters": [{"operator": "or", "filters": + [{"property": "tag", "condition": "in", "value": ["High"]}]}]}`}, + {"only in a sort's custom order", `{"id": "v1", "sorts": + [{"property": "tag", "direction": "custom", "custom_order": ["High"]}]}`}, + } { + t.Run(tc.name, func(t *testing.T) { + // given + doc := fmt.Sprintf(`{"version": 2, "id": "obj1", + "option_ids": {"tag": {"High": "bafylegend"}}, + "blocks": [{"id": "dv1", "type": "dataview", "views": [%s]}]}`, tc.view) + + // when + var warned []Issue + validateErr := ValidateWarn([]byte(doc), func(i Issue) { warned = append(warned, i) }) + _, back, err := Unmarshal([]byte(doc), Options{ResolveOptions: space, GenerateId: seqIds("g")}) + + // then + require.NoError(t, validateErr, doc) + require.NoError(t, err) + assert.Empty(t, optionIdsWarnings(warned), "the document spells `tag` in this dataview") + assert.Equal(t, []string{"bafylegend"}, resolvedOptionIds(t, back), + "the legend under a filter-only property must still be honored") + // and the fallback the legend overrides really does answer the + // other option, so this cannot pass by the two agreeing + id, ok := space.OptionId("tag", "High") + require.True(t, ok) + assert.Equal(t, "bafydecoy", id, "the fixture must reproduce the first-match scan") + }) + } +} + +// resolvedOptionIds reads back whatever select values an imported dataview +// carries — the filter value or the sort's custom order, whichever the +// document had. +func resolvedOptionIds(t *testing.T, snap *model.SmartBlockSnapshotBase) []string { + t.Helper() + view := backView(t, snap) + var out []string + var walk func(fs []*model.BlockContentDataviewFilter) + walk = func(fs []*model.BlockContentDataviewFilter) { + for _, f := range fs { + out = append(out, valueStringList(f.Value)...) + walk(f.NestedFilters) + } + } + walk(view.Filters) + for _, s := range view.Sorts { + for _, v := range s.CustomOrder { + out = append(out, v.GetStringValue()) + } + } + return out +} + +// The census itself, position by position (optionrefs.go). Each document +// spells one probe in exactly one place and the census has to find it. +// +// This IS the guard now. The census used to exist in two implementations — +// import's, over the decoded document, and Validate's, over the undecoded +// one — with an agreement test between them; import no longer takes a census +// at all, so the twin is gone and so is that test. What the agreement really +// stood for is this table: a census that stopped covering a position would +// make Validate warn about entries import honours, and it is this list, not +// a second implementation, that says it does not. +// +// The last two entries are the boundary: a filter's `nested_property` names a +// property of the object the filter walks TO, and an `option_ids` key is not +// a use of the property it names, or every typo would vouch for itself. +func TestOptionRefs_ThePropertyCensusCoversEveryPosition(t *testing.T) { + const probe = "probe_property" + for _, tc := range []struct { + name string + doc string + counted bool + }{ + {"a properties member", `{"properties": {"probe_property": ["High"]}}`, true}, + {"a property_internal_keys spelling", `{"property_internal_keys": {"probe_property": "storedKey"}}`, true}, + {"a type_properties key", `{"kind": "object_type", + "type_settings": {"property_definitions": [{"property": "probe_property", "format": "select"}]}}`, true}, + {"a property block's key", `{"blocks": + [{"type": "property", "property": "probe_property"}]}`, true}, + {"a link block's shown properties", `{"blocks": + [{"type": "link", "object_id": "bafyreitarget", "properties": ["probe_property"]}]}`, true}, + {"a dataview's properties list", `{"blocks": [{"type": "dataview", + "properties": [{"property": "probe_property", "format": "select"}]}]}`, true}, + {"a view's group_by", `{"blocks": [{"type": "dataview", + "views": [{"id": "v1", "group_by": "probe_property"}]}]}`, true}, + {"a view's cover_property", `{"blocks": [{"type": "dataview", + "views": [{"id": "v1", "cover_property": "probe_property"}]}]}`, true}, + {"a view's end_property", `{"blocks": [{"type": "dataview", + "views": [{"id": "v1", "end_property": "probe_property"}]}]}`, true}, + {"a view column", `{"blocks": [{"type": "dataview", "views": + [{"id": "v1", "columns": [{"property": "probe_property"}]}]}]}`, true}, + {"a sort", `{"blocks": [{"type": "dataview", "views": + [{"id": "v1", "sorts": [{"property": "probe_property"}]}]}]}`, true}, + {"a filter", `{"blocks": [{"type": "dataview", "views": + [{"id": "v1", "filters": [{"property": "probe_property", "condition": "in"}]}]}]}`, true}, + {"a nested filter", `{"blocks": [{"type": "dataview", "views": [{"id": "v1", "filters": + [{"operator": "and", "filters": [{"property": "probe_property"}]}]}]}]}`, true}, + {"a property block inside a table cell", `{"blocks": [{"type": "table", + "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": + [{"type": "property", "property": "probe_property"}]}]}]}`, true}, + {"a dataview inside a table cell", `{"blocks": [{"type": "table", + "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [[{"type": "dataview", + "views": [{"id": "v1", "filters": [{"property": "probe_property"}]}]}]]}]}]}`, true}, + {"a filter's nested_property", `{"blocks": [{"type": "dataview", "views": [{"id": "v1", + "filters": [{"property": "assignee", "nested_property": "probe_property"}]}]}]}`, false}, + {"the option_ids key itself", `{"option_ids": + {"probe_property": {"High": "bafyreiopt"}}}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + // given + data := []byte(`{"version": 2, "id": "obj1",` + strings.TrimPrefix(tc.doc, "{")) + + // when + var raw map[string]any + require.NoError(t, json.Unmarshal(data, &raw)) + + // then + assert.Equal(t, tc.counted, rawPropertySpellings(raw)[probe]) + }) + } +} diff --git a/pkg/lib/anyblockjson/options_test.go b/pkg/lib/anyblockjson/options_test.go new file mode 100644 index 0000000000..9633618c16 --- /dev/null +++ b/pkg/lib/anyblockjson/options_test.go @@ -0,0 +1,229 @@ +package anyblockjson + +// A select's vocabulary is otherwise discovered only from values that happen +// to be used, so a schema value no sample record carries never exists (its +// kanban column is simply missing), and minted options get no orderId and +// sort alphabetically. typeProperties[].options declares both (§2a). + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/constant" +) + +type recordingPropertyResolver struct{ defs []PropertyDefinition } + +func (r *recordingPropertyResolver) PropertyById(string) (PropertyDefinition, bool) { + return PropertyDefinition{}, false +} +func (r *recordingPropertyResolver) PropertyId(def PropertyDefinition) (string, bool) { + r.defs = append(r.defs, def) + return string(def.Key), true +} + +func TestImport_OptionsReachTheWiring(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [ + {"property": "stage", "name": "Stage", "format": "select", + "options": ["Backlog", "In progress", "Done"]}, + {"property": "note", "name": "Note", "format": "text"}]}}` + r := &recordingPropertyResolver{} + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), ResolveProperties: r}) + require.NoError(t, err) + + require.Len(t, r.defs, 2) + assert.Equal(t, domain.RelationKey("stage"), r.defs[0].Key) + assert.Equal(t, + []OptionDefinition{{Name: "Backlog"}, {Name: "In progress"}, {Name: "Done"}}, + r.defs[0].Options, "declaration order is the display order") + assert.Empty(t, r.defs[1].Options) +} + +// A color is declared on the option it belongs to rather than in a parallel +// array, so inserting or reordering an option cannot shift it (§2a). +func TestImport_OptionColorsReachTheWiring(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "stage", "name": "Stage", "format": "select", + "options": ["Backlog", {"name": "In progress", "color": "blue"}, + {"name": "Done", "color": "lime"}]}]}}` + want := []OptionDefinition{ + {Name: "Backlog"}, + {Name: "In progress", Color: "blue"}, + {Name: "Done", Color: "lime"}, + } + r := &recordingPropertyResolver{} + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), ResolveProperties: r}) + require.NoError(t, err) + + require.Len(t, r.defs, 1) + assert.Equal(t, want, r.defs[0].Options, + "a bare string is an option that declares no color") +} + +func TestExport_OptionsRoundTrip(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "stage", "name": "Stage", "format": "select", + "options": ["Backlog", "In progress", "Done"]}]}}` + _, snap, err := Unmarshal([]byte(doc), Options{ + GenerateId: seqIds("g"), ResolveProperties: &recordingPropertyResolver{}}) + require.NoError(t, err) + + data, err := Marshal(model.SmartBlockType_STType, snap, Options{ResolveProperties: &staticPropertyResolver{ + def: PropertyDefinition{Key: "stage", Name: "Stage", Format: 3, + Options: []OptionDefinition{{Name: "Backlog"}, {Name: "In progress"}, {Name: "Done"}}}}}) + require.NoError(t, err) + assert.Contains(t, string(data), `"options"`) + assert.Contains(t, string(data), `"Backlog"`) + assert.NoError(t, Validate(data)) +} + +// The bare string is canonical whenever the option carries no color, the +// object form otherwise — the rule §6.1 already gives table cells. +func TestExport_ColorlessOptionStaysABareString(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "stage", "name": "Stage", "format": "select", + "options": ["Backlog", {"name": "Done", "color": "lime"}]}]}}` + _, snap, err := Unmarshal([]byte(doc), Options{ + GenerateId: seqIds("g"), ResolveProperties: &recordingPropertyResolver{}}) + require.NoError(t, err) + + data, err := Marshal(model.SmartBlockType_STType, snap, Options{ResolveProperties: &staticPropertyResolver{ + def: PropertyDefinition{Key: "stage", Name: "Stage", Format: 3, + Options: []OptionDefinition{{Name: "Backlog"}, {Name: "Done", Color: "lime"}}}}}) + require.NoError(t, err) + + var out struct { + TypeSettings struct { + PropertyDefinitions []struct { + Options []any `json:"options"` + } `json:"property_definitions"` + } `json:"type_settings"` + } + require.NoError(t, json.Unmarshal(data, &out)) + require.Len(t, out.TypeSettings.PropertyDefinitions, 1) + assert.Equal(t, + []any{"Backlog", map[string]any{"name": "Done", "color": "lime"}}, + out.TypeSettings.PropertyDefinitions[0].Options) + assert.NoError(t, Validate(data)) + + // canonical key order inside the object form is name then color + rendered := string(data)[strings.Index(string(data), `"options"`):] + assert.Less(t, strings.Index(rendered, `"name"`), strings.Index(rendered, `"color"`)) +} + +func TestValidate_OptionColorRules(t *testing.T) { + t.Run("unknown color rejected", func(t *testing.T) { + err := Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "stage", "format": "select", + "options": [{"name": "a", "color": "chartreuse"}]}]}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "options/0/color") + }) + t.Run("colored options are not duplicates of each other", func(t *testing.T) { + assert.NoError(t, Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "stage", "format": "select", + "options": ["a", {"name": "b", "color": "blue"}, {"name": "c", "color": "lime"}]}]}}`))) + }) + t.Run("duplicate across the two forms rejected", func(t *testing.T) { + err := Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "stage", "format": "select", + "options": ["a", {"name": "a", "color": "blue"}]}]}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate option") + }) + t.Run("object form on a non-select rejected", func(t *testing.T) { + err := Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "note", "format": "text", "options": [{"name": "a"}]}]}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "only meaningful on select") + }) +} + +// The palette lives in util/constant and is restated as an enum in the +// published schema, which is hand-maintained; this keeps the two in step. +func TestSchema_OptionColorEnumMatchesPalette(t *testing.T) { + var schema struct { + Defs struct { + OptionColor struct { + Enum []string `json:"enum"` + } `json:"optionColor"` + } `json:"$defs"` + } + require.NoError(t, json.Unmarshal(schemaJSON, &schema)) + + want := make([]string, 0, len(constant.OptionColors())) + for _, c := range constant.OptionColors() { + want = append(want, c.String()) + } + assert.Equal(t, want, schema.Defs.OptionColor.Enum) +} + +type staticPropertyResolver struct{ def PropertyDefinition } + +func (r *staticPropertyResolver) PropertyById(string) (PropertyDefinition, bool) { + return r.def, true +} +func (r *staticPropertyResolver) PropertyId(def PropertyDefinition) (string, bool) { + return string(def.Key), true +} + +func TestValidate_OptionsRules(t *testing.T) { + t.Run("rejected on a non-select format", func(t *testing.T) { + err := Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "note", "format": "text", "options": ["a"]}]}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "only meaningful on select") + }) + t.Run("duplicates rejected", func(t *testing.T) { + err := Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "stage", "format": "select", "options": ["a", "b", "a"]}]}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate option") + }) + t.Run("accepted on select and multi_select", func(t *testing.T) { + for _, f := range []string{"select", "multi_select"} { + assert.NoError(t, Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "stage", "format": "`+f+`", "options": ["a", "b"]}]}}`)), f) + } + }) +} + +func TestImport_ObjectTypesReachTheWiring(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [ + {"property": "owner", "name": "Owner", "format": "objects", + "object_types": ["wikiPerson", "participant"]}, + {"property": "anything", "name": "Anything", "format": "objects"}]}}` + r := &recordingPropertyResolver{} + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), ResolveProperties: r}) + require.NoError(t, err) + + require.Len(t, r.defs, 2) + assert.Equal(t, []string{"wikiPerson", "participant"}, r.defs[0].ObjectTypes, + "priority order is preserved") + assert.Empty(t, r.defs[1].ObjectTypes, "untargeted accepts any object") +} + +func TestValidate_ObjectTypesRules(t *testing.T) { + t.Run("rejected on a non-object format", func(t *testing.T) { + for _, f := range []string{"select", "date", "text"} { + err := Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "p", "format": "` + f + `", "object_types": ["participant"]}]}}`)) + require.Error(t, err, f) + assert.Contains(t, err.Error(), "only meaningful on objects/files") + } + }) + t.Run("accepted on objects and files", func(t *testing.T) { + for _, f := range []string{"objects", "files"} { + assert.NoError(t, Validate([]byte(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "p", "format": "`+f+`", "object_types": ["participant"]}]}}`)), f) + } + }) +} diff --git a/pkg/lib/anyblockjson/panel_review_test.go b/pkg/lib/anyblockjson/panel_review_test.go new file mode 100644 index 0000000000..4d82c67169 --- /dev/null +++ b/pkg/lib/anyblockjson/panel_review_test.go @@ -0,0 +1,294 @@ +package anyblockjson + +// Regression tests for the 4-lens review panel findings: adversarial +// round-trip defects, robustness/DoS quadratics, and the §8 resource bounds +// that close them. + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// Lens 2, finding A: a Link mark whose target is an object deep-link renders +// identically to an Object mark; it must normalize to one, or the parse-back +// type flip changes same-type overlap resolution. +func TestInline_ObjectDeepLinkNormalizes(t *testing.T) { + marks := []*model.BlockContentTextMark{ + mark(mObject, 0, 2, "outerid"), + mark(mLink, 1, 2, objectLinkDest("innerlink")), + } + md1 := renderInline("ab", marks) + text1, marks1, err := parseInline(md1) + require.NoError(t, err) + md2 := renderInline(text1, marks1) + require.Equal(t, md1, md2, "must be byte-stable") + + // standalone equivalence: the deep-link Link renders as an Object link + asLink := renderInline("x", []*model.BlockContentTextMark{mark(mLink, 0, 1, objectLinkDest("id9"))}) + asObject := renderInline("x", []*model.BlockContentTextMark{mark(mObject, 0, 1, "id9")}) + assert.Equal(t, asObject, asLink) +} + +// Lens 2, finding B: a ']' (or backtick) inside a link destination must be +// escaped, or it derails the enclosing label scan when links nest. +func TestInline_BracketInDestInsideLabel(t *testing.T) { + for _, dest := range []string{"a]b", "a[b", "a`b`c"} { + marks := []*model.BlockContentTextMark{ + mark(mObject, 0, 1, "z"), + mark(mLink, 0, 1, dest), + } + md1 := renderInline("k", marks) + text1, marks1, err := parseInline(md1) + require.NoError(t, err, "dest %q", dest) + assert.Equal(t, "k", text1) + md2 := renderInline(text1, marks1) + require.Equal(t, md1, md2, "dest %q must be byte-stable", dest) + } +} + +// Lens 2, finding D: cyclic ChildrenIds must not recurse into a fatal stack +// overflow; the cycle is cut and the output still validates. +func TestExport_CyclicChildren(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"a"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "a", ChildrenIds: []string{"b", "a"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "a"}}}, + {Id: "b", ChildrenIds: []string{"a"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "b"}}}, + }, + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data)) +} + +// Lens 2, finding E: a block shared by two parents is emitted once; duplicate +// table column ids are dropped — the canonical output must pass validation. +func TestExport_SharedAndDuplicateIds(t *testing.T) { + shared := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"p1", "p2"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "p1", ChildrenIds: []string{"kid"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "1", Style: model.BlockContentText_Toggle}}}, + {Id: "p2", ChildrenIds: []string{"kid"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "2", Style: model.BlockContentText_Toggle}}}, + {Id: "kid", Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "kid"}}}, + }, + } + data, err := Marshal(model.SmartBlockType_Page, shared, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data), "shared child must be emitted once") + + dupCols := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"table1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "table1", ChildrenIds: []string{"tcols", "trows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "tcols", ChildrenIds: []string{"dup", "dup"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: "trows", ChildrenIds: []string{"r1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + {Id: "dup", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + {Id: "r1", ChildrenIds: []string{"r1-dup"}, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + {Id: "r1-dup", Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "x"}}}, + }, + } + data, err = Marshal(model.SmartBlockType_Page, dupCols, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data), "duplicate column ids must be dropped") +} + +// Lens 2, finding C: the CompactIds path guards nil inner content. Finding F +// (a refs key outside the schema charset) is closed by deletion — no object +// id is labelled at all now — so what stands in its place is the statement +// that an id no charset would have admitted travels verbatim. +func TestExport_CompactIdsHardening(t *testing.T) { + contents := []model.IsBlockContent{ + &model.BlockContentOfText{}, + &model.BlockContentOfFile{}, + &model.BlockContentOfBookmark{}, + &model.BlockContentOfLink{}, + &model.BlockContentOfDataview{}, + } + for _, c := range contents { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"b1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "b1", Content: c}, + }, + } + require.NotPanics(t, func() { + _, _ = Marshal(model.SmartBlockType_Page, snap, Options{CompactIds: true}) + }, "content %T with CompactIds", c) + } + + // a mention target whose characters no label charset would admit, beside + // one that the deleted labeller would happily have shortened: both are + // written out, in the mark, with nothing in the envelope + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"b1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + textBlock("b1", model.BlockContentText_Paragraph, "hi", + mark(mMention, 0, 1, "a`b"), mark(mMention, 1, 2, "bafyreiregularlylongobjectid")), + }, + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{CompactIds: true}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + // the odd id stays full (entity-encoded in the attribute per §8.1) + assert.Contains(t, string(data), "a`b") + assert.Contains(t, string(data), `object_id=\"bafyreiregularlylongobjectid\"`, + "a compactable id is written in full too (§9a)") + assert.NotContains(t, string(data), `"ectid"`, "and no label is minted for it") +} + +// Lens 3: the parse boundary must stay effectively linear. Every input here +// took quadratic time before the rework (16s+ at 400KB); the generous bounds +// only trip on a reintroduced O(n²). +func TestInline_ParseIsLinearish(t *testing.T) { + if testing.Short() { + t.Skip("timing test") + } + cases := []struct { + name string + md string + }{ + {"plain 1MB", strings.Repeat("lorem ipsum dolor sit amet ", 40000)}, + {"unmatched brackets", strings.Repeat("[", 200000)}, + {"unmatched emphasis", strings.Repeat("*a ", 70000)}, + {"backtick staircase", func() string { + var b strings.Builder + for i := 1; b.Len() < 200000; i++ { + b.WriteString(strings.Repeat("`", i%40+1)) + b.WriteString("x") + } + return b.String() + }()}, + {"nested links", strings.Repeat("[a](u)", 30000)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + start := time.Now() + _, _, err := parseInline(tc.md) + require.NoError(t, err) + require.Less(t, time.Since(start), 10*time.Second, "quadratic behavior reintroduced") + }) + } +} + +// Lens 3: the §8 resource bounds are deterministic local rules. +func TestInline_ResourceBounds(t *testing.T) { + // an over-long link destination is not recognized; the mark drops on + // export and the text stays intact + longDest := "https://x.io/" + strings.Repeat("a", maxLinkDestLen) + md := renderInline("ab", []*model.BlockContentTextMark{mark(mLink, 0, 2, longDest)}) + assert.Equal(t, "ab", md, "over-long link param is dropped") + + doc := fmt.Sprintf("[a](%s)", longDest) + text, marks, err := parseInline(doc) + require.NoError(t, err) + assert.Empty(t, marks) + assert.Equal(t, doc, text, "over-long dest stays literal") + + // an over-long emoji param is invalid and dropped + md = renderInline("ab", []*model.BlockContentTextMark{ + mark(mEmoji, 0, 1, strings.Repeat("😀", maxEmojiParamLen)), + }) + assert.Equal(t, "ab", md) + + // links nested beyond the cap stay literal but still parse cleanly + deep := strings.Repeat("[", 40) + "x" + strings.Repeat("](u)", 40) + text, _, err = parseInline(deep) + require.NoError(t, err) + assert.Contains(t, text, "x") + canonical := renderInline(text, nil) + text2, marks2, err := parseInline(canonical) + require.NoError(t, err) + assert.Equal(t, text, text2) + assert.Empty(t, marks2) +} + +// Lens 4: the wiring dispatches on the version/$schema markers (§13). +func TestDetectFormat(t *testing.T) { + v, schema, ok := DetectFormat([]byte(`{"$schema": "` + SchemaURL + `", "version": 2}`)) + require.True(t, ok) + assert.Equal(t, 2, v) + assert.Equal(t, SchemaURL, schema) + + v, _, ok = DetectFormat([]byte(`{"version": 3}`)) + require.True(t, ok) + assert.Equal(t, 3, v) + + _, _, ok = DetectFormat([]byte(`{"blocks": []}`)) + assert.False(t, ok) + _, _, ok = DetectFormat([]byte(`not json`)) + assert.False(t, ok) +} + +// Lens 4: the default id generator (no GenerateId option) mints +// editor-shaped 24-hex ids. +func TestImport_DefaultIdGenerator(t *testing.T) { + _, snap, err := Unmarshal([]byte(`{"version": 2, "blocks": [{"type": "paragraph", "text": "x"}]}`), Options{}) + require.NoError(t, err) + require.Len(t, snap.Blocks, 2) + for _, b := range snap.Blocks { + assert.Regexp(t, "^[0-9a-f]{24}$", b.Id) + } +} + +// Suffix labels are a fixed 5 characters; ids whose suffixes collide stay +// uncompacted (full-id fallback) rather than resolving ambiguously. Carried +// over from the deleted object half onto the half that survives: two minted +// BLOCK ids sharing a tail must both keep their full spelling, or one label +// would name two blocks. +func TestExport_SuffixCollisionFallsBackToFullId(t *testing.T) { + const ( + mintedA = "aaaaaaaaaaaaaaaaaaa11111" + mintedB = "bbbbbbbbbbbbbbbbbbb11111" + ) + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{mintedA, mintedB}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + textBlock(mintedA, model.BlockContentText_Paragraph, "a"), + textBlock(mintedB, model.BlockContentText_Paragraph, "b"), + }, + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{CompactIds: true}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + s := string(data) + // both ids share the suffix "11111": neither may claim it + assert.NotContains(t, s, `"11111"`) + assert.Contains(t, s, `"id": "`+mintedA+`"`) + assert.Contains(t, s, `"id": "`+mintedB+`"`) + + // and the same fixture with only one of them present proves it is the + // collision refusing the label, not the shape + snap.Blocks = snap.Blocks[:2] + snap.Blocks[0].ChildrenIds = []string{mintedA} + solo, err := Marshal(model.SmartBlockType_Page, snap, Options{CompactIds: true}) + require.NoError(t, err) + assert.Contains(t, string(solo), `"id": "11111"`) +} diff --git a/pkg/lib/anyblockjson/participantprovenance.go b/pkg/lib/anyblockjson/participantprovenance.go new file mode 100644 index 0000000000..70dc4fab35 --- /dev/null +++ b/pkg/lib/anyblockjson/participantprovenance.go @@ -0,0 +1,60 @@ +package anyblockjson + +// participantprovenance.go — the stored details a PARTICIPANT document does +// not carry, because on that kind the value is not a fact about the member +// (§3; the transient-key policy, scoped by kind — typeProvenanceKeys' +// pattern, on the other machine-derived kind). +// +// A participant document is derived from the ACL: it has no creation change +// of its own, and an object whose root change carries no creation date gets +// `createdDate` stamped with time.Now() at load +// (core/block/editor/smartblock, detailsinject) — so the stored value is +// the moment the object was last COLD-BUILT wearing the name of a fact. +// Measured, which is what admitted the drop: two exports of the same 7 +// spaces, 1,164 documents compared field-by-field — the only drifting kind +// is participant (22 of 22), and the only drifting field is `created_date` +// (22 of 22); every other kind is byte-stable across exports. On a full +// 155-space run the drift was 2,322 documents against 2,492 participants. +// A value that changes whenever the cache evicts between two reads of an +// unchanged object describes the reader, not the object. +// +// The other two provenance fields a participant carries — `creator` and +// `last_modified_by` — STAY, by decision: both are `_anytype_profile` on +// 2,492 of 2,492 corpus participants, which is upstream's bug to fix (a +// participant's creator should be the real identity), not this format's to +// paper over by omission. + +import ( + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// participantProvenanceKeys are the stored details export omits on a +// participant document and import drops there (stale, not wrong). Each +// entry needs the measured proof that the value describes the reading +// session rather than the member — the §15 #12 discipline. +var participantProvenanceKeys = map[string]string{ + // stamped time.Now() on every cold build (no creation change to derive + // it from); the ONLY field that drifted across a 1,164-document + // double-export comparison, on 22 of 22 participants + "createdDate": "a load timestamp wearing the name of a fact: re-stamped on every cold build", +} + +// isParticipantSmartBlock is the snapshot-side statement of which kind the +// rule scopes to, isTypeSmartBlock's shape. +func isParticipantSmartBlock(sbType model.SmartBlockType) bool { + return sbType == model.SmartBlockType_Participant +} + +// DroppedParticipantProvenanceKey reports a stored detail that export omits +// on a PARTICIPANT document because its value describes the reading session +// rather than the member (§3). It is the exported half of the rule, for the +// round-trip comparator — the predicate is the format's own, not a copy, so +// the comparator and the exporter cannot disagree (the drift class that +// once produced 1,344 false failures in one sweep). +func DroppedParticipantProvenanceKey(sbType model.SmartBlockType, key string) bool { + if !isParticipantSmartBlock(sbType) { + return false + } + _, dropped := participantProvenanceKeys[key] + return dropped +} diff --git a/pkg/lib/anyblockjson/participantprovenance_test.go b/pkg/lib/anyblockjson/participantprovenance_test.go new file mode 100644 index 0000000000..7fe7419111 --- /dev/null +++ b/pkg/lib/anyblockjson/participantprovenance_test.go @@ -0,0 +1,103 @@ +package anyblockjson + +// participantprovenance_test.go pins the participant-scoped provenance drop +// (§3, participantprovenance.go): a participant document does not carry +// `created_date`, because the object is derived from the ACL, has no +// creation change, and the stored value is time.Now() stamped on every cold +// build. The measurement that admitted the drop: two exports of the same 7 +// spaces, 1,164 documents compared field-by-field — the only drifting kind +// was participant (22 of 22) and the only drifting field `created_date` +// (22 of 22); on a full 155-space run, 2,322 drifts against 2,492 +// participants, every other kind byte-stable. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func participantSnapshot() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{ + "id": str("AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA"), + "name": str("Roman"), + "createdDate": num(1756180000), // the load timestamp wearing a fact's name + "lastModifiedDate": num(1700000000), + }), + } +} + +// Export's half: the key is omitted on a participant, and ONLY there — on a +// page createdDate is real provenance and stays. The kind scoping is the +// whole rule: widen it and every object loses its creation date; narrow it +// away and every eviction-separated pair of exports disagrees on 2,492 +// documents again. +// +// How this can fail: gate on the key without the kind (the page case goes +// red); drop the export site but keep the predicate (the participant case +// finds the key in the document); or stamp the drop into strippedDetailKeys +// (kind-blind, same page regression). +func TestParticipantProvenance_CreatedDateNeverExported(t *testing.T) { + // given / when + doc, err := Marshal(model.SmartBlockType_Participant, participantSnapshot(), Options{}) + require.NoError(t, err) + + // then + assert.NotContains(t, string(doc), "Creation date", + "a participant's created_date is a load timestamp, not a fact") + assert.Contains(t, string(doc), "Last modified date", + "only the measured drifting key is dropped") + + t.Run("on a page the same key stays", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{ + "id": str("bafypage"), "name": str("A page"), "createdDate": num(1756180000), + }), + } + doc, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(doc), "Creation date", + "on every other kind createdDate is real provenance") + }) +} + +// Import's half: dropped, not refused — a document written before the rule +// (every pre-v0.47 dump carries the key on its participants) is stale +// rather than wrong, the transientProperties policy scoped by kind. +// +// How this can fail: refuse instead of drop (every existing dump's +// participants stop importing); or let the value through (the destination's +// derived detail is shadowed by the source's load timestamp). +func TestParticipantProvenance_DroppedNotRefusedOnImport(t *testing.T) { + doc := `{"version": 2, "kind": "participant", + "id": "AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA", + "properties": {"name": "Roman", "created_date": "2026-08-26T21:42:26Z"}}` + + require.NoError(t, Validate([]byte(doc)), "stale, not wrong") + sbType, snap, err := Unmarshal([]byte(doc), Options{}) + require.NoError(t, err, "Validate and Unmarshal agree (§11 I2)") + require.Equal(t, model.SmartBlockType_Participant, sbType) + assert.NotContains(t, snap.GetDetails().GetFields(), "createdDate", + "the load timestamp must not reach the snapshot") + assert.Equal(t, "Roman", snap.GetDetails().GetFields()["name"].GetStringValue()) + + t.Run("on a page the same property still lands", func(t *testing.T) { + doc := `{"version": 2, "properties": {"name": "A page", "created_date": "2026-08-26T21:42:26Z"}}` + _, snap, err := Unmarshal([]byte(doc), Options{}) + require.NoError(t, err) + assert.Contains(t, snap.GetDetails().GetFields(), "createdDate") + }) +} + +// The predicate the comparator consults, pinned at both edges of its scope. +func TestDroppedParticipantProvenanceKey_Scope(t *testing.T) { + assert.True(t, DroppedParticipantProvenanceKey(model.SmartBlockType_Participant, "createdDate")) + assert.False(t, DroppedParticipantProvenanceKey(model.SmartBlockType_Page, "createdDate"), + "kind-scoped: a page's creation date is real provenance") + assert.False(t, DroppedParticipantProvenanceKey(model.SmartBlockType_Participant, "lastModifiedDate"), + "key-scoped: only the measured drifting key") +} diff --git a/pkg/lib/anyblockjson/prefreeze_review_test.go b/pkg/lib/anyblockjson/prefreeze_review_test.go new file mode 100644 index 0000000000..9cad5976e3 --- /dev/null +++ b/pkg/lib/anyblockjson/prefreeze_review_test.go @@ -0,0 +1,743 @@ +package anyblockjson + +// Regression tests for the confirmed findings of the pre-freeze review +// (PREFREEZE_REVIEW.md, Tier 1). The two property tests the same review asks +// for live in flat_invariants_test.go — these are the individual instances, +// hand-written so the fixture can express the failure. + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// dateOptions resolves customDate as a date property, which is what makes the +// date export path run at all — a fixture without it never reaches the code +// under test. +func dateOptions(onWarning func(Issue)) Options { + return Options{ + ResolveFormat: testFormatResolver, + GenerateId: seqIds("g"), + OnWarning: onWarning, + } +} + +func dateSnapshot(sec float64) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "obj1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("obj1"), + "customDate": {Kind: &types.Value_NumberValue{NumberValue: sec}}, + }), + } +} + +// Tier 1 #4. `customDate = 1751791445000` is milliseconds stored where seconds +// belong — a real corruption class, and one this format inherits rather than +// creates. It formatted as "57482-01-22T22:43:20Z", which parseDate cannot +// read back, so re-import stored a *string* on a date-format property: the +// value stopped being a date, permanently and quietly, and byte-stably +// thereafter, so nothing ever corrects it. +func TestExport_DateOutsideRFC3339RangeKeepsTheNumber(t *testing.T) { + for _, sec := range []float64{1751791445000, -62167219201, 253402300800} { + t.Run(fmt.Sprintf("%.0f", sec), func(t *testing.T) { + var warnings []Issue + data, err := Marshal(model.SmartBlockType_Page, dateSnapshot(sec), + dateOptions(func(i Issue) { warnings = append(warnings, i) })) + require.NoError(t, err) + require.NoError(t, Validate(data)) + + var doc struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.IsType(t, float64(0), doc.Properties["customDate"], + "an unrepresentable date stays a number rather than becoming an unparseable string") + + _, back, err := Unmarshal(data, dateOptions(nil)) + require.NoError(t, err) + got := back.Details.Fields["customDate"] + require.NotNil(t, got) + _, isNumber := got.GetKind().(*types.Value_NumberValue) + require.True(t, isNumber, "the value must survive as a number, got %v", got) + assert.Equal(t, sec, got.GetNumberValue()) + assert.NotEmpty(t, warnings, "a date this far out is worth saying out loud") + }) + } +} + +// The in-range behaviour is unchanged: a date property is an RFC 3339 string. +func TestExport_DateInsideRangeIsStillAString(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, dateSnapshot(1751791445), dateOptions(nil)) + require.NoError(t, err) + assert.Contains(t, string(data), `"customDate": "2025-07-06T08:44:05Z"`) + + _, back, err := Unmarshal(data, dateOptions(nil)) + require.NoError(t, err) + assert.Equal(t, float64(1751791445), back.Details.Fields["customDate"].GetNumberValue()) +} + +// The representable range is defined by what parses back, not by taste: any +// other definition would let export write a string parseDate cannot read. +func TestFormatDate_RangeIsExactlyWhatParsesBack(t *testing.T) { + for _, sec := range []int64{minDateSec, maxDateSec, 0, 1751791445, -1} { + s, ok := formatDate(sec) + require.True(t, ok, "%d must be representable", sec) + back, parsed := parseDate(s) + require.True(t, parsed, "%d rendered as %q, which does not parse", sec, s) + assert.Equal(t, sec, back, "%d rendered as %q, which parses as %d", sec, s, back) + } + for _, sec := range []int64{minDateSec - 1, maxDateSec + 1, 1751791445000} { + s, ok := formatDate(sec) + assert.False(t, ok, "%d must be out of range, got %q", sec, s) + } +} + +// A file block's addedAt is a schema string with no number form to fall back +// to, so an unrepresentable timestamp is dropped with a warning rather than +// written as a string no reader can parse back. +func TestExport_FileAddedAtOutsideRangeIsOmitted(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"f1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "f1", Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + TargetObjectId: "file1", Name: "doc.pdf", AddedAt: 1751791445000, + }}}, + }, + Details: fields(map[string]*types.Value{"id": str("obj1")}), + } + var warnings []Issue + data, err := Marshal(model.SmartBlockType_Page, snap, Options{ + OnWarning: func(i Issue) { warnings = append(warnings, i) }}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.NotContains(t, string(data), "added_at") + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "added_at") +} + +// ---- Tier 1 #1: one id domain ---- +// +// The rule was known and written down (table.go: "Emitting one verbatim would +// make Marshal write a document its own Validate rejects, so normalize it once +// here") and then applied to table inner ids only. Every other id surface +// skipped it, in six confirmed ways. Two of them lose data. + +// assertUniqueBlockIds is the snapshot-side half of the id invariant: whatever +// import mints, no two blocks may end up with the same id. +func assertUniqueBlockIds(t *testing.T, snap *model.SmartBlockSnapshotBase) { + t.Helper() + seen := map[string]bool{} + for _, b := range snap.Blocks { + require.False(t, seen[b.Id], "duplicate block id %q in the rebuilt snapshot", b.Id) + seen[b.Id] = true + } +} + +// (a) a stored block id outside the schema's charset was written verbatim. +func TestExport_BlockIdOutsideCharsetIsSanitized(t *testing.T) { + for _, stored := range []string{"a.b", "dir/file", "блок", strings.Repeat("x", 65)} { + t.Run(stored, func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{stored}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: stored, Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "hi"}}}, + }, + Details: fields(map[string]*types.Value{"id": str("obj1")}), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal must not emit what Validate rejects:\n%s", data) + }) + } +} + +// (c) a sanitized column id could land on a sibling paragraph's id, because +// the used-id set covered table inner ids only. +func TestExport_SanitizedColumnIdCannotTakeASiblingsId(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"c_1", "t1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "c_1", Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "a paragraph that got there first"}}}, + {Id: "t1", ChildrenIds: []string{"cols", "rows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "cols", ChildrenIds: []string{"c-1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_TableColumns}}}, + {Id: "c-1", Content: &model.BlockContentOfTableColumn{ + TableColumn: &model.BlockContentTableColumn{}}}, + {Id: "rows", ChildrenIds: []string{"r1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_TableRows}}}, + {Id: "r1", Content: &model.BlockContentOfTableRow{ + TableRow: &model.BlockContentTableRow{}}}, + }, + Details: fields(map[string]*types.Value{"id": str("obj1")}), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data), "duplicate id across two surfaces:\n%s", data) + assert.Contains(t, string(data), "a paragraph that got there first", + "the paragraph keeps its own id and its content") +} + +// (d) under CompactBlockLabels a suffix label must not take an id the document +// already serves verbatim, or two things answer to one name. +// +// The finding was raised against the refs labeller, which is deleted (§9a). +// What survives it is the BLOCK half, and there the rule is live for exactly +// one reason: `mintedSuffixLabels`' census counts LOCAL ids only. A block id +// cannot alias another block id whether or not the avoid-set is consulted — +// the second subtest states that, since it is what is actually in force — but +// an OBJECT id is invisible to that census, and every object id is now spelled +// verbatim in the document, so the `fullIds` avoid-set is the only thing +// standing between a minted block and a label that already names something +// else. Nothing downstream reports the result: the document is valid, it just +// has two meanings for one string. +func TestExport_CompactLabelCannotTakeAServedId(t *testing.T) { + const ( + shortObject = "abcde" // compactIdMinLen wide: spelled verbatim + mintedBlock = "0000000000000000000abcde" // whose suffix is that same string + ) + + t.Run("a block label cannot take a short OBJECT id", func(t *testing.T) { + // the block half against the OTHER id population, which is the half + // its own census cannot see: mintedSuffixLabels counts local ids only, + // so a 5-char object id spelled verbatim in the document is invisible + // to it and only the fullIds avoid-set stands between the minted block + // and a label that already names something else in the same document. + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{mintedBlock}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + mentionBlock(mintedBlock, shortObject), + }, + Details: fields(map[string]*types.Value{"id": str("obj1")}), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{CompactBlockLabels: true}) + require.NoError(t, err) + require.NoError(t, Validate(data), "%s", data) + + var doc struct { + Blocks []struct { + Id string `json:"id"` + Text string `json:"text"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + require.Len(t, doc.Blocks, 1) + assert.NotEqual(t, shortObject, doc.Blocks[0].Id, + "the block must not label itself with an object id the document serves verbatim:\n%s", data) + assert.Equal(t, mintedBlock, doc.Blocks[0].Id, + "and with that label refused it stays full") + assert.Contains(t, doc.Blocks[0].Text, shortObject, + "the fixture only bites while the object id is spelled in the document") + }) + + t.Run("a block label cannot take a short block id", func(t *testing.T) { + // the block half of the same rule under the shape rule that now + // governs it. Unlike the refs half above, removing the avoid-set does + // NOT break this: the census already covers every local id, and the + // avoid-set is defence in depth over ids from the other population. + assert.Empty(t, mintedSuffixLabels([]string{mintedBlock, shortObject}, compactIdMinLen, nil), + "the census over every local id refuses the suffix") + assert.NotEmpty(t, mintedSuffixLabels([]string{mintedBlock}, compactIdMinLen, nil), + "and it is the collision that refuses it, not the shape") + }) +} + +func mentionBlock(id, target string) *model.Block { + return &model.Block{Id: id, Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "x", Marks: &model.BlockContentTextMarks{ + Marks: []*model.BlockContentTextMark{{ + Range: &model.Range{From: 0, To: 1}, + Type: model.BlockContentTextMark_Mention, Param: target}}}}}} +} + +// (b) a derived cell id belongs to the table whether or not the cell is +// written: the editor materializes missing cells on open, and the id it uses +// is rowId-colId. So a block claiming one is a duplicate, and validation is +// the only place that can say so. +func TestValidate_DerivedCellIdIsClaimedEvenWhenTheCellIsAbsent(t *testing.T) { + // the trailing cell is absent from the array, so nothing in the document + // mentions r1-c2 — an explicit null would already have been claimed + doc := `{"version": 2, "blocks": [ + {"type": "paragraph", "id": "r1-c2", "text": "x"}, + {"type": "table", + "columns": [{"id": "c1"}, {"id": "c2"}], + "rows": [{"id": "r1", "cells": ["first"]}]}]}` + err := Validate([]byte(doc)) + require.Error(t, err, "r1-c2 is the id the table will use when that cell is filled") + assert.Contains(t, err.Error(), "duplicate id") + + // and the claim is not over-eager: no table, no derived ids + require.NoError(t, Validate([]byte(`{"version": 2, "blocks": [ + {"type": "paragraph", "id": "r1-c2", "text": "x"}]}`))) +} + +// (e) pinPrimaryDataview scanned top-level block ids only, so an authored +// table row named "dataview" was invisible to it — and it minted the same id +// for the dataview block *after* validation had passed. Re-export then lost +// the whole table body. +func TestImport_PrimaryDataviewDoesNotCollideWithATableRowId(t *testing.T) { + doc := `{"version": 2, "id": "obj1", "blocks": [ + {"type": "table", + "columns": [{"id": "c1"}], + "rows": [{"id": "dataview", "cells": ["cell text"]}]}, + {"type": "dataview", "views": [{"id": "v1", "name": "All"}]}]}` + require.NoError(t, Validate([]byte(doc))) + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assertUniqueBlockIds(t, snap) + + out, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(out)) + assert.Contains(t, string(out), "cell text", "the table body must survive the round trip") +} + +// (f) Options.GenerateId is the caller's, and the convert wiring derives ids +// from file paths — both halves author-controlled. genId never checked the ids +// the document itself already used. +func TestImport_GeneratedIdCannotTakeAnAuthoredId(t *testing.T) { + doc := `{"version": 2, "blocks": [ + {"type": "paragraph", "id": "g1", "text": "authored"}, + {"type": "paragraph", "text": "needs an id"}, + {"type": "table", "columns": [{"id": "g2"}], "rows": [{"cells": ["c"]}]}]}` + require.NoError(t, Validate([]byte(doc))) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assertUniqueBlockIds(t, snap) +} + +// ---- Tier 1 #5: property-key admission control ---- +// +// §4a claims the import surface "treats supplied values as authoritative only +// where semantically safe". Nothing implemented that clause: import copied +// every supplied key onto details, skipping only id/type, so the input surface +// was strictly *wider* than the output surface — export strips +// bundle.LocalAndDerivedRelationKeys, import took them. Among them are the keys +// that decide which existing object a snapshot merges into. + +// The rule, stated once: import refuses exactly what export strips, except for +// the keys it DROPS. Deriving all of it from strippedDetailKeys is what keeps +// the two surfaces from drifting apart again — a second hand-written list +// would. +// +// The exception is narrow and deliberate (§3), and covers two families. A +// TRANSIENT key describes the moment an object was written rather than the +// object; an ATTRIBUTION key names the member who wrote it, and is recovered +// from the tree root's signature on every rebuild whatever a document says. +// Either way a document carrying one is stale, not hostile, and refusing it +// would make an old export unimportable to no purpose. Everything else on the +// stripped list is either derived state or a merge-resolution vector, and +// those stay errors. The two exempt halves are asserted in +// TestTransientProperties_DroppedNotRefused and +// TestAttributionProperties_DroppedNotRefused. +func TestValidate_ImportRefusesWhatExportStrips(t *testing.T) { + refused := 0 + for key := range strippedDetailKeys() { + if isDroppedOnImport(key) { + continue + } + refused++ + doc := fmt.Sprintf(`{"version": 2, "id": "obj1", "properties": {%q: "x"}}`, key) + err := Validate([]byte(doc)) + require.Error(t, err, "%s is stripped on export, so it must be refused on import", key) + assert.Contains(t, err.Error(), "/properties/"+key) + } + require.NotZero(t, refused, "every stripped key became droppable — the deny rule went dead") +} + +// The named resolution vectors matter most: existingobject.go resolves which +// object in the victim's space a snapshot merges into from oldAnytypeID, +// uniqueKey and sourceFilePath. All three ARE bundled relations — what makes +// two of them need naming by hand is that bundle.LocalAndDerivedRelationKeys, +// the list the deny-rule derives from, does not carry them. +func TestValidate_ResolutionVectorPropertiesRefused(t *testing.T) { + for _, key := range []string{"oldAnytypeID", "uniqueKey", "sourceFilePath"} { + doc := fmt.Sprintf(`{"version": 2, "id": "obj1", "properties": {%q: "x"}}`, key) + err := Validate([]byte(doc)) + require.Error(t, err, key) + assert.Contains(t, err.Error(), "/properties/"+key) + } + + // the comment above, pinned — the claim that these are not bundled + // relations survived in three places, and neverWritableProperties exists + // only because of the second half of it + onList := map[string]bool{} + for _, k := range bundle.LocalAndDerivedRelationKeys { + onList[string(k)] = true + } + for key, wantOnList := range map[string]bool{ + "oldAnytypeID": false, "sourceFilePath": false, "uniqueKey": true, + } { + assert.True(t, bundle.HasRelation(domain.RelationKey(key)), + "%s is a bundled relation", key) + assert.Equal(t, wantOnList, onList[key], + "%s in bundle.LocalAndDerivedRelationKeys", key) + } +} + +// The six §3 exemptions are the whole point of the exemption list: they are +// internal keys the importer meaningfully preserves, so they stay writable. +func TestValidate_ExemptedInternalPropertiesStillAccepted(t *testing.T) { + doc := `{"version": 2, "id": "obj1", "properties": { + "createdDate": "2026-07-06T08:44:05Z", "lastModifiedDate": "2026-07-06T08:44:05Z", + "creator": "bafyparticipant", "isFavorite": true, "isArchived": false, + "resolvedLayout": "basic", "name": "N"}}` + require.NoError(t, Validate([]byte(doc))) +} + +// A property key that is not a key at all — empty, or carrying a control +// character — landed on details verbatim and was written back out. +func TestValidate_PropertyKeyShape(t *testing.T) { + t.Run("refused", func(t *testing.T) { + for _, doc := range []string{ + `{"version": 2, "properties": {"": "empty"}}`, + `{"version": 2, "properties": {"a\nb": "newline"}}`, + "{\"version\": 2, \"properties\": {\"a\\u0000b\": \"nul\"}}", + "{\"version\": 2, \"properties\": {\"a\\u007fb\": \"del\"}}", + } { + assert.Error(t, Validate([]byte(doc)), doc) + } + }) + t.Run("accepted", func(t *testing.T) { + // the shapes real keys have: bundled lowerCamel, a bson-hex custom key, + // and the bare names old accounts carry (ANOMALIES §7). The pattern is + // deliberately a deny rule rather than an allowlist — an allowlist + // would have to be verified against every key in every account before + // export could depend on it. + doc := `{"version": 2, "properties": { + "dueDate": null, "68f0d9c3b3c8a94e0d0b0a12": "x", "artist": "y"}}` + require.NoError(t, Validate([]byte(doc))) + }) +} + +// A value whose shape contradicts its property's format is not stored as +// written: it reads as the format's zero forever. "next Friday" on a date is +// the case the review names. The fixtures spell the CANONICAL snake_case slug +// (§3) — this test used to pass only because it spelled the stored key, which +// made its subject dead code for every document the format itself produces. +func TestValidate_PropertyValueShapeWarns(t *testing.T) { + warningsFor := func(t *testing.T, doc string) []Issue { + var got []Issue + require.NoError(t, ValidateWarn([]byte(doc), func(i Issue) { got = append(got, i) }), doc) + return got + } + + t.Run("a date that is not a date", func(t *testing.T) { + got := warningsFor(t, `{"version": 2, "id": "o1", "properties": {"due_date": "next Friday"}}`) + require.Len(t, got, 1) + assert.Equal(t, "/properties/due_date", got[0].Path) + assert.Contains(t, got[0].Message, "date") + }) + + t.Run("the stored spelling is an address too", func(t *testing.T) { + // §3 chain step 1: a spelling the table does not know binds verbatim, + // so the stored key keeps warning alongside the canonical slug + got := warningsFor(t, `{"version": 2, "id": "o1", "properties": {"dueDate": "next Friday"}}`) + require.Len(t, got, 1) + assert.Equal(t, "/properties/dueDate", got[0].Path) + assert.Contains(t, got[0].Message, "date") + }) + + t.Run("a spelling the legend binds is checked as what it resolves to", func(t *testing.T) { + got := warningsFor(t, `{"version": 2, "id": "o1", + "property_internal_keys": {"prio": "dueDate"}, "properties": {"prio": "next Friday"}}`) + require.Len(t, got, 1) + assert.Equal(t, "/properties/prio", got[0].Path) + assert.Contains(t, got[0].Message, "date") + }) + + t.Run("a checkbox that is not a boolean", func(t *testing.T) { + got := warningsFor(t, `{"version": 2, "id": "o1", "properties": {"done": "yes"}}`) + require.Len(t, got, 1) + assert.Equal(t, "/properties/done", got[0].Path) + }) + + t.Run("shapes the format does hold are quiet", func(t *testing.T) { + // including the raw number a date out of RFC 3339 range exports as + assert.Empty(t, warningsFor(t, `{"version": 2, "id": "o1", "properties": { + "due_date": "2026-07-06T08:44:05Z", "created_date": 1751791445000, + "done": true, "name": "N", "plural_name": "Ns", "tag": ["a", "b"]}}`)) + }) + + t.Run("null is always a value", func(t *testing.T) { + // §3: an explicit null records that the property was set and cleared + assert.Empty(t, warningsFor(t, `{"version": 2, "id": "o1", "properties": { + "due_date": null, "done": null}}`)) + }) +} + +// The mention attribute is snake_case like every other identifier the format +// defines, which the tag grammar had to learn: its attribute-name scanner read +// ASCII letters only, so `object_id` parsed as the attribute `object` and then +// failed on the underscore (§8.1). +func TestInline_MentionAttributeIsSnakeCase(t *testing.T) { + md := `ping Roman` + text, marks, err := parseInline(md) + require.NoError(t, err) + assert.Equal(t, "ping Roman", text) + require.Len(t, marks, 1) + assert.Equal(t, "bafyid", marks[0].Param) + assert.Equal(t, md, renderInline(text, marks), "canonical form is byte-stable") + + // the previous draft's spelling is an error that names the attribute, + // rather than a silently dropped mention + _, _, err = parseInline(`ping Roman`) + require.Error(t, err) + assert.Contains(t, err.Error(), `unknown attribute "objectId"`) +} + +// The envelope `key` is the STORED identity key, written verbatim (§2) — so +// the charset it can carry is whatever the store already holds, and a closed +// allowlist over it was falsified by the first sweep that ran with one: 59 +// objects in a 36 808-object account failed their own export, every one of +// them a relation option whose stored key is built from the option's *name*. +// These are the real keys, from that sweep's reports. +func TestValidate_EnvelopeKeyAcceptsRealStoredKeys(t *testing.T) { + for _, key := range []string{ + "completion_status_Not Started", + "challenge_resolution_status_In Progress", + "69bbfc78877a91b1d12d1a7c_C/C++", + "69bbfc78877a91b1d12d1a7c_C#", + "69bbfc78877a91b1d12d1a84_$addToSet", + "69bbfc78877a91b1d12d1a7c_.NET", + "69aab06861fab2bc0d9afbe2_Roma Khafizianov", + "69a56205ccba0a47d8d8eb71_тогглы", + "69bbfc78877a91b1d12d1a7c_JavaScript/TypeScript", + } { + doc := fmt.Sprintf(`{"version": 2, "kind": "property_option", "id": "o1", "internal_key": %q}`, key) + assert.NoError(t, Validate([]byte(doc)), "stored key %q must round-trip", key) + } +} + +// What a key may never be is unreadable: empty, or carrying a control +// character. That is the same deny rule property keys get (§3), for the same +// reason — an allowlist can only be trusted after auditing every key in every +// account, and this one was not. +func TestValidate_EnvelopeKeyRejectsUnreadable(t *testing.T) { + for _, doc := range []string{ + `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": ""}`, + "{\"version\": 2, \"kind\": \"object_type\", \"id\": \"t1\", \"key\": \"a\\u0000b\"}", + "{\"version\": 2, \"kind\": \"object_type\", \"id\": \"t1\", \"key\": \"a\\nb\"}", + } { + assert.Error(t, Validate([]byte(doc)), doc) + } +} + +// ---- post-freeze review: admission keyed off the raw document spelling ---- +// +// The three property checks above (deny rule, layout names, format shapes) +// were written before §3's key vocabulary arrived, and keyed off the RAW +// document spelling. The format's canonical spelling is the snake_case api +// slug, so writing "unique_key" instead of "uniqueKey" walked past all three +// — including the deny rule that stops a document from aiming itself at an +// existing object (existingobject.go resolves merge targets from oldAnytypeID, +// uniqueKey and sourceFilePath). The fix: every check runs on the STORED key a +// spelling resolves to, through the same chain import uses — the document's +// own legend, then the bundled table, then the spelling verbatim (§3). + +// Every stripped key is a bundled relation with an api slug, so the canonical +// document spells the slug — and the deny rule has to hold for that spelling, +// derived from the same set as the stored-spelling test above. +func TestValidate_DeniedKeysRefusedInCanonicalSpelling(t *testing.T) { + covered := 0 + for key := range strippedDetailKeys() { + if isDroppedOnImport(key) { + continue // dropped, not refused — see the note above + } + slug := (BundledKeyVocabulary{}).PropertySlug(key) + if slug == key { + continue // one spelling; TestValidate_ImportRefusesWhatExportStrips covers it + } + covered++ + doc := fmt.Sprintf(`{"version": 2, "id": "obj1", "properties": {%q: "x"}}`, slug) + err := Validate([]byte(doc)) + require.Error(t, err, "%q is the canonical spelling of stripped key %q, so it must be refused", slug, key) + assert.Contains(t, err.Error(), "/properties/"+slug) + } + require.NotZero(t, covered, "the bundled slug table no longer differs from the stored spellings — this test went dead") +} + +// The legend is consulted before any vocabulary (§3), so without a check it +// was an unchecked rebind primitive: any spelling could be bound to any stored +// key, denied ones included — {"prio": "uniqueKey"} landed a uniqueKey detail, +// and {"myid": "id"} overwrote the envelope id itself (observed: details.id +// became ["boom"]). Admission runs on the resolved key, which the legend is +// part of resolving. +func TestValidate_LegendCannotRebindOntoInternalKeys(t *testing.T) { + for name, doc := range map[string]string{ + "resolution vector": `{"version": 2, "id": "o1", "property_internal_keys": {"prio": "uniqueKey"}, "properties": {"prio": "ot-page"}}`, + "envelope id": `{"version": 2, "id": "o1", "property_internal_keys": {"myid": "id"}, "properties": {"myid": "boom"}}`, + "stripped key": `{"version": 2, "id": "o1", "property_internal_keys": {"s": "spaceId"}, "properties": {"s": "other"}}`, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(doc)) + require.Error(t, err, doc) + _, _, unmErr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr, "Unmarshal must reject what Validate rejects (I2)") + }) + } + + // a legend entry that rebinds a spelling onto a HARMLESS key is the + // feature working as specified: nothing lands on an internal key + ok := `{"version": 2, "id": "o1", "property_internal_keys": {"prio": "6a32d4856761631534b22f85"}, "properties": {"prio": "high"}}` + require.NoError(t, Validate([]byte(ok))) + _, snap, err := Unmarshal([]byte(ok), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + require.Contains(t, snap.Details.Fields, "6a32d4856761631534b22f85") +} + +// {"resolved_layout": "nonsense"} validated clean and imported the raw string +// onto a number-format property, where every consumer reads it with an int +// getter and silently sees "basic" — the exact silence the layout-name check +// exists to catch, dead for every canonically-spelled document. +func TestValidate_LayoutNameCheckedInCanonicalSpelling(t *testing.T) { + err := Validate([]byte(`{"version": 2, "id": "o1", "properties": {"resolved_layout": "nonsense"}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/properties/resolved_layout") + assert.Contains(t, err.Error(), "unknown layout") + + // a real name is accepted and lands as the stored number — the import + // half always resolved the slug; only validation did not + doc := `{"version": 2, "id": "o1", "properties": {"resolved_layout": "todo"}}` + require.NoError(t, Validate([]byte(doc))) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + v := snap.Details.Fields["resolvedLayout"] + require.NotNil(t, v) + assert.Equal(t, float64(layoutNames.value("todo")), v.GetNumberValue()) +} + +// A legend value is a stored key — the string that becomes a details field — +// so it obeys the same writable-key rule as a property name (§3): non-empty, +// no control characters, at most 128 characters. Export only ever records +// values that passed that rule, so the schema bound keeps admission symmetric. +func TestValidate_LegendValueMustBeAWritableKey(t *testing.T) { + t.Run("refused", func(t *testing.T) { + for name, doc := range map[string]string{ + "empty": `{"version": 2, "property_internal_keys": {"p": ""}}`, + "over-long": fmt.Sprintf(`{"version": 2, "property_internal_keys": {"p": %q}}`, strings.Repeat("k", maxPropertyKeyLen+1)), + "control char": `{"version": 2, "property_internal_keys": {"p": "a\nb"}}`, + } { + assert.Error(t, Validate([]byte(doc)), name) + } + }) + t.Run("accepted", func(t *testing.T) { + // the shapes real stored keys have: bson-hex, and option keys carrying + // the option's own name, spaces and non-ASCII included (ANOMALIES §7) + doc := `{"version": 2, "property_internal_keys": { + "prio": "6a32d4856761631534b22f85", + "toggles": "69a56205ccba0a47d8d8eb71_тогглы"}}` + require.NoError(t, Validate([]byte(doc))) + }) +} + +// rebindingVocabulary stands in for a node-backed vocabulary whose space maps +// a slug to a stored key the bundled table never knew. Validate cannot see it +// (it takes no resolver, §13), so admission has to run AGAIN on the importer's +// final resolved key, at the seam where details are written. +type rebindingVocabulary struct{ BundledKeyVocabulary } + +func (rebindingVocabulary) PropertyKey(slug string) (string, bool) { + if slug == "prio" { + return "uniqueKey", true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func TestImport_AdmissionRunsOnTheResolvedKey(t *testing.T) { + doc := `{"version": 2, "id": "o1", "properties": {"prio": "ot-page"}}` + require.NoError(t, Validate([]byte(doc)), + "the bundled chain resolves prio verbatim, a legal custom key — Validate cannot know better") + _, _, err := Unmarshal([]byte(doc), Options{Keys: rebindingVocabulary{}, GenerateId: seqIds("g")}) + require.Error(t, err, "the wider vocabulary resolves prio onto uniqueKey, which no document may set") + assert.Contains(t, err.Error(), "/properties/prio") + assert.Contains(t, err.Error(), "uniqueKey") +} + +// unwritableSlugVocabulary produces the slug shapes a real space can mint: +// apiObjectKey is user-supplied or strcase-derived from the property name +// (objectcreator/util.go), with no length bound. buildProperties checked the +// STORED key's writability and then emitted the slug — the string that +// actually becomes the JSON property name — unchecked, so a 192-character +// slug made Marshal emit a document its own Validate rejects (I1). +type unwritableSlugVocabulary struct{ BundledKeyVocabulary } + +func (unwritableSlugVocabulary) PropertySlug(key string) string { + switch key { + case "artist": + return strings.Repeat("s", maxPropertyKeyLen+64) + case "venue": + return "ve\nue" + case "city": + return "" + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func TestExport_UnwritableSlugFallsBackToTheStoredKey(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "obj1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("obj1"), "name": str("N"), + "artist": str("x"), "venue": str("y"), "city": str("z"), + }), + } + var warns []Issue + data, err := Marshal(model.SmartBlockType_Page, snap, + Options{Keys: unwritableSlugVocabulary{}, OnWarning: func(i Issue) { warns = append(warns, i) }}) + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal must never emit what its own Validate rejects (§11):\n%s", data) + + var got struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &got)) + for _, k := range []string{"artist", "venue", "city"} { + assert.Contains(t, got.Properties, k, + "an unwritable slug falls back to the stored key, which is always its own address (§3)") + } + assert.NotEmpty(t, warns, "a vocabulary producing unwritable slugs is worth telling the caller about") +} + +// The same raw-spelling defect had a fourth instance in the same loop's +// neighbourhood: the property_definitions-vs-recommended-lists ambiguity check +// indexed properties by the STORED list keys, so the canonical spelling +// carried both representations without a word. The canonical spelling is the +// display name "Recommended properties" (bundledname.go); the stored key +// still resolves verbatim; and the derived-slug shape of either — the retired +// `recommended_relations` and the v0.38 alias `recommended_properties` alike +// — lands in the same fold class, so every spelling of the key is checked. +func TestValidate_RecommendedListConflictCheckedInCanonicalSpelling(t *testing.T) { + for _, spelling := range []string{ + "recommendedRelations", "Recommended properties", + "recommended_relations", "recommended_properties", + } { + doc := fmt.Sprintf(`{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "page", + "type_settings": {"property_definitions": [{"property": "due_date", "format": "date"}]}, + "properties": {%q: ["a"]}}`, spelling) + err := Validate([]byte(doc)) + require.Error(t, err, spelling) + assert.Contains(t, err.Error(), "/properties/"+spelling) + assert.Contains(t, err.Error(), "type_settings.property_definitions") + } +} diff --git a/pkg/lib/anyblockjson/profilepage.go b/pkg/lib/anyblockjson/profilepage.go new file mode 100644 index 0000000000..8210e505f9 --- /dev/null +++ b/pkg/lib/anyblockjson/profilepage.go @@ -0,0 +1,42 @@ +package anyblockjson + +// profilepage.go — the deprecated per-space profile object, which a bundle +// does not carry (§2c). +// +// `kind: "profile_page"` is the pre-participant representation of a person in +// a space. A `participant` document does that job now, and every space that +// still holds a profile object also holds participants — from 1 to 1,856 of +// them across a 77-space export. +// +// What survives in a real account is not the account owner's own profile. It +// is the profile object of whoever built each imported space, dragged along +// by the import. Measured over 77 spaces, 8 remain, and every one of them +// +// - is `isHidden: true`, +// - carries `importType` and `origin`, so it ARRIVED rather than being made, +// - carries `oldAnytypeID`, so it predates the current data model, +// - holds nothing: seven have no blocks at all, the eighth an empty +// paragraph — the one the editor leaves on any object ever opened, +// - and is named after someone or something else: four are literally +// "Onboarding 2.2", one is a space's name, three are other people. +// +// A bundle is shareable, and a hidden object carrying a stranger's name is +// not something a reader wants restored. +// +// UNCONDITIONAL, and deliberately unlike the omissions beside it. Those are +// fail-closed — a space document with real content on its page still travels +// — because a space object is a live thing that merely happens to be empty. +// A profile object is not: the kind is DEPRECATED, nothing creates one any +// more, and whatever a particular one holds is residue from a data model that +// no longer exists. Keeping the richest of them would preserve exactly the +// thing least worth preserving. + +import ( + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// OmittedProfilePage reports the deprecated profile object, which a bundle +// never writes (§2c). +func OmittedProfilePage(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase) bool { + return sbType == model.SmartBlockType_ProfilePage && base != nil +} diff --git a/pkg/lib/anyblockjson/profilepage_test.go b/pkg/lib/anyblockjson/profilepage_test.go new file mode 100644 index 0000000000..e50cad2423 --- /dev/null +++ b/pkg/lib/anyblockjson/profilepage_test.go @@ -0,0 +1,61 @@ +package anyblockjson + +// profilepage_test.go — the deprecated profile object never travels (§2c). + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The kind is deprecated: a `participant` document represents a person in a +// space now, and every space in a 77-space export that still holds a profile +// object also holds participants — 1 to 1,856 of them. What survives is the +// profile of whoever built each imported space, hidden, carrying +// `oldAnytypeID`, named after someone else. +// +// The drop is UNCONDITIONAL, unlike the space-document omission beside it. A +// space object is live and merely happens to be empty, so that one fails +// closed on any content. Nothing creates a profile object any more, so +// whatever one holds is residue from a data model that is gone — keeping the +// richest of them would preserve exactly the thing least worth preserving. +// +// How this can fail: make it conditional on emptiness and the account's +// eight become seven, kept by an empty paragraph the editor left behind; +// widen the kind gate and live objects stop being exported. +func TestProfilePage_NeverTravels(t *testing.T) { + base := func(det map[string]*types.Value, blocks ...*model.Block) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{Details: fields(det), Blocks: blocks} + } + + t.Run("an empty one", func(t *testing.T) { + assert.True(t, OmittedProfilePage(model.SmartBlockType_ProfilePage, + base(map[string]*types.Value{"name": str("Onboarding 2.2"), "isHidden": boolVal(true)}))) + }) + + t.Run("one with content goes too — the kind is what is deprecated", func(t *testing.T) { + assert.True(t, OmittedProfilePage(model.SmartBlockType_ProfilePage, + base(map[string]*types.Value{"name": str("Abby"), "description": str("a real note")}, + &model.Block{Id: "p", Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "something someone wrote"}}}))) + }) + + t.Run("no other kind is touched", func(t *testing.T) { + for _, k := range []model.SmartBlockType{ + model.SmartBlockType_Page, + model.SmartBlockType_Participant, + model.SmartBlockType_Workspace, + model.SmartBlockType_STType, + } { + assert.Falsef(t, OmittedProfilePage(k, base(map[string]*types.Value{"name": str("x")})), + "kind %v must still be exported", k) + } + }) + + t.Run("a nil snapshot is not an omission", func(t *testing.T) { + assert.False(t, OmittedProfilePage(model.SmartBlockType_ProfilePage, nil)) + }) +} diff --git a/pkg/lib/anyblockjson/propertydefinition_test.go b/pkg/lib/anyblockjson/propertydefinition_test.go new file mode 100644 index 0000000000..7f6d0c89f9 --- /dev/null +++ b/pkg/lib/anyblockjson/propertydefinition_test.go @@ -0,0 +1,268 @@ +package anyblockjson + +// propertydefinition_test.go pins the ONE-SHAPE rule: a property is described +// by `$defs/propertyDefinition` wherever it is described, and every home +// REFERENCES that shape rather than restating it — the same discipline +// TestPropertyFormatEnum_MatchesFormatNames applies to the format vocabulary. +// A fourth spelling of "a property definition" is the §15 #14 disease this +// wave exists to end, and a restated member is how one starts: two lists that +// agree today and drift tomorrow. + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// sharedPropertyMembers is the decided propertyDefinition surface: the +// identity pair the key/spelling split produced (`property` the spelling, +// `internal_key` the stored id — one word no longer carries both meanings), +// the four members every home speaks beside it, plus the five the dictionary +// lifts (description, include_time, max_count, readonly, default_value). The +// test restates it ON PURPOSE — the schema is the implementation and this +// list is the specification, so a member added to one and not the other +// fails here instead of shipping as a home-local extension. +var sharedPropertyMembers = []string{ + "property", "internal_key", "name", "format", "options", "object_types", + "description", "include_time", "max_count", "readonly", "default_value", +} + +// The published schema states the property-definition shape once — +// $defs/propertyDefinition — and each home layers over a $ref to it: its own +// `properties` may only NARROW a shared member (typeProperty pins `format` to +// authorableFormat and `object_types` to a real array) or add the one member +// that belongs to the home rather than the property (`section`). The shared +// shape itself stays open (no `required`, no unevaluated/additional gate), so +// homes can close themselves without the allOf-vs-additionalProperties trap. +// +// How this can fail: drop the $ref from typeProperty and restate the ten +// members locally (the shape validates identically today and drifts +// tomorrow); add an eleventh member to propertyDefinition without adding it +// here; close propertyDefinition itself, which would break every layered +// home at once; or reopen typeProperty by removing its unevaluatedProperties +// gate. +func TestPropertyDefinition_OneSharedShapeThreeHomes(t *testing.T) { + type schemaNode struct { + AllOf []json.RawMessage `json:"allOf"` + Properties map[string]json.RawMessage `json:"properties"` + Required []string `json:"required"` + Additional json.RawMessage `json:"additionalProperties"` + Uneval json.RawMessage `json:"unevaluatedProperties"` + } + var schema struct { + Properties map[string]schemaNode `json:"properties"` + Defs map[string]schemaNode `json:"$defs"` + } + require.NoError(t, json.Unmarshal(SchemaJSON(), &schema)) + + def, ok := schema.Defs["propertyDefinition"] + require.True(t, ok, "the schema must publish $defs/propertyDefinition") + + want := map[string]bool{} + for _, m := range sharedPropertyMembers { + want[m] = true + } + got := map[string]bool{} + for m := range def.Properties { + got[m] = true + } + assert.Equal(t, want, got, "propertyDefinition carries the decided ten members, no more, no fewer") + + // the shared shape is the extension point, so it must stay open: each + // home states its own `required` and closes itself + assert.Empty(t, def.Required, "requiredness is home-specific; the shared shape demands nothing") + assert.Empty(t, def.Additional, "the shared shape must stay open for its homes to layer over") + assert.Empty(t, def.Uneval, "the shared shape must stay open for its homes to layer over") + + // each home: a $ref to the shared shape, a local layer of narrowings, + // refusals (`false` members whose fact lives elsewhere) and home-owned + // members ONLY, and its own closure. The two in-file homes are checked + // through this map; the third home — the dictionary entry, which lives + // in properties.schema.json — is checked below with the same rules. A + // home missing from either is a fourth spelling. + typeProperty, foundTypeProperty := schema.Defs["typeProperty"] + relationSettings, foundRelationSettings := schema.Properties["property_settings"] + for home, tc := range map[string]struct { + node schemaNode + found bool + localMembers []string // narrowings and home-owned members the layer may hold + }{ + "typeProperty": { + node: typeProperty, found: foundTypeProperty, + localMembers: []string{"format", "object_types", "section"}, + }, + "property_settings": { + node: relationSettings, found: foundRelationSettings, + localMembers: nil, // nothing to narrow; its layer is all refusals + }, + } { + require.Truef(t, tc.found, "home %s must exist", home) + h := tc.node + refFound := false + for _, a := range h.AllOf { + var ref struct { + Ref string `json:"$ref"` + } + if json.Unmarshal(a, &ref) == nil && ref.Ref == "#/$defs/propertyDefinition" { + refFound = true + } + } + assert.Truef(t, refFound, "%s must reference propertyDefinition, not restate it", home) + allowed := map[string]bool{} + for _, m := range tc.localMembers { + allowed[m] = true + } + for m, raw := range h.Properties { + if string(raw) == "false" { + // a refusal, not a restatement: the member's fact has a home + // elsewhere on this document (§2d) + continue + } + assert.Truef(t, allowed[m], "%s restates %q — a shared member may only be narrowed, and only where the home must", home, m) + } + assert.Equalf(t, "false", string(h.Uneval), "%s must close itself with unevaluatedProperties: false", home) + } + + // the third home lives in its own schema FILE — the dictionary entry + // (§2f) — and references the shape across files by its published URL, + // the way the index schema references plainIcon. Same discipline: a + // layer of narrowings (`object_types` back to a real array) plus the + // home's own requirements, closed with unevaluatedProperties. + // + // How this can fail: restate the ten members inside + // properties.schema.json instead of the $ref (drift starts), widen the + // entry's layer beyond the one narrowing, or reopen the entry by + // deleting its unevaluatedProperties gate. + var propSchema struct { + Defs map[string]schemaNode `json:"$defs"` + } + require.NoError(t, json.Unmarshal(propertiesSchemaJSON, &propSchema)) + entry, foundEntry := propSchema.Defs["dictionaryEntry"] + require.True(t, foundEntry, "the properties schema must publish $defs/dictionaryEntry") + refFound := false + for _, a := range entry.AllOf { + var ref struct { + Ref string `json:"$ref"` + } + if json.Unmarshal(a, &ref) == nil && ref.Ref == SchemaURL+"#/$defs/propertyDefinition" { + refFound = true + } + } + assert.True(t, refFound, "a dictionary entry must reference propertyDefinition by its published URL, not restate it") + for m, raw := range entry.Properties { + if string(raw) == "false" { + continue + } + assert.Truef(t, m == "object_types", "dictionaryEntry restates %q — its layer holds the one narrowing only", m) + } + // `format` alone is required outright: self-sufficiency (§2f) means an + // entry states what the property holds. Identity is required through + // anyOf instead — a key, OR a `name` the spelling derives from — because + // demanding a key asks an author to invent an identifier only a real + // space can mint. + assert.ElementsMatch(t, []string{"format"}, entry.Required, + "an entry requires its format outright; identity is the anyOf beside it") + assert.Equal(t, "false", string(entry.Uneval), "dictionaryEntry must close itself with unevaluatedProperties: false") +} + +// The layered closure has a classic failure mode: `additionalProperties: +// false` beside an allOf-$ref refuses EVERYTHING the ref admits, and +// swapping it for unevaluatedProperties without a working annotation flow +// silently admits every unknown member instead. Both ends are pinned through +// the real validator: an unknown member on a type_properties entry is still +// refused, and every shared member is still admitted. +// +// How this can fail: replace typeProperty's unevaluatedProperties with +// additionalProperties (every entry with a key fails, second case red), or +// delete the gate entirely (first case goes green on a member nothing reads). +func TestPropertyDefinition_LayeredClosureHoldsBothWays(t *testing.T) { + t.Run("an unknown member is still refused through the layer", func(t *testing.T) { + err := Validate([]byte(`{"version":2,"kind":"object_type","internal_key":"task", + "type_settings":{"property_definitions": [{"property":"due_date","sections":"featured"}]}}`)) + require.Error(t, err, "`sections` names nothing; the closure must catch it") + }) + t.Run("a null object_types stays a relation-only shape", func(t *testing.T) { + // the shared shape admits null because a relation's STORED value can + // hold one (§2d); a type declares targets or omits the member, so the + // home narrows it back to an array + err := Validate([]byte(`{"version":2,"kind":"object_type","internal_key":"task", + "type_settings":{"property_definitions": [{"property":"assignee","object_types":null}]}}`)) + require.Error(t, err) + }) + t.Run("every shared member is admitted on an entry", func(t *testing.T) { + err := Validate([]byte(`{"version":2,"kind":"object_type","internal_key":"task", + "type_settings":{"property_definitions": [{"property":"budget","name":"Budget","format":"number", + "description":"Planned spend","include_time":false,"max_count":1, + "readonly":true,"default_value":100,"section":"featured"}]}}`)) + require.NoError(t, err) + }) +} + +// capturingPropertyResolver records the definitions PropertyId receives, so a +// test can see exactly what crossed the codec seam. +type capturingPropertyResolver struct { + defs []PropertyDefinition +} + +func (r *capturingPropertyResolver) PropertyById(id string) (PropertyDefinition, bool) { + return PropertyDefinition{}, false +} + +func (r *capturingPropertyResolver) PropertyId(def PropertyDefinition) (string, bool) { + r.defs = append(r.defs, def) + return "relid-" + string(def.Key), true +} + +// A member the schema admits and the codec sheds is worse than one the schema +// refuses: the document validates, imports, and quietly means less than it +// says. So the whole decoded definition must reach the resolver's create +// path, through BOTH doors the §2a array arrives by — the document +// (applyTypeProperties) and the PATCH channel (BuildRecommendedLists) — which +// share TypeProperty.definition precisely so they cannot disagree. +// +// How this can fail: shed one of the five members in TypeProperty.definition, +// or rebuild the def by hand in one door and forget a member there. +func TestPropertyDefinition_SharedMembersReachTheResolver(t *testing.T) { + doc := []byte(`{"version":2,"kind":"object_type","internal_key":"task", + "type_settings":{"property_definitions": [{"property":"budget","name":"Budget","format":"number", + "description":"Planned spend","include_time":false,"max_count":1, + "readonly":true,"default_value":100,"section":"featured"}]}}`) + + check := func(t *testing.T, defs []PropertyDefinition) { + require.Len(t, defs, 1) + def := defs[0] + assert.Equal(t, domain.RelationKey("budget"), def.Key) + assert.Equal(t, model.RelationFormat_number, def.Format) + assert.Equal(t, "Planned spend", def.Description) + require.NotNil(t, def.IncludeTime, "include_time false is a declaration, not an absence") + assert.False(t, *def.IncludeTime) + assert.Equal(t, int64(1), def.MaxCount) + assert.True(t, def.Readonly) + assert.Equal(t, float64(100), def.DefaultValue) + } + + t.Run("the document door", func(t *testing.T) { + r := &capturingPropertyResolver{} + _, _, err := Unmarshal(doc, Options{ResolveProperties: r}) + require.NoError(t, err) + check(t, r.defs) + }) + + t.Run("the PATCH door", func(t *testing.T) { + r := &capturingPropertyResolver{} + var parsed struct { + TypeSettings struct { + PropertyDefinitions []TypeProperty `json:"property_definitions"` + } `json:"type_settings"` + } + require.NoError(t, json.Unmarshal(doc, &parsed)) + _, err := BuildRecommendedLists(parsed.TypeSettings.PropertyDefinitions, Options{ResolveProperties: r}) + require.NoError(t, err) + check(t, r.defs) + }) +} diff --git a/pkg/lib/anyblockjson/propertykeys_test.go b/pkg/lib/anyblockjson/propertykeys_test.go new file mode 100644 index 0000000000..ee342bd537 --- /dev/null +++ b/pkg/lib/anyblockjson/propertykeys_test.go @@ -0,0 +1,193 @@ +package anyblockjson + +// The slug layer is a compaction of key spelling, and like every compaction in +// this format it has to be invertible from the document alone (§9a's rule for +// object ids). It was not: a node-backed vocabulary slugs a custom key +// `6a32d485…` to `priority`, and a reader without that space reads `priority` +// back as the key `priority` — a different relation. A 36 808-object sweep +// found 12 objects whose dataview named a relation this way; that this layer +// was the mechanism behind them was established afterwards, by these tests +// and storeresolver/keyvocab_test.go — those 12 objects have not been swept +// again since the fix landed. + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// spaceVocabulary is a node-backed vocabulary: it knows the space's stored +// slugs, which the bundled table cannot. +type spaceVocabulary struct{ slugOf map[string]string } + +func (v spaceVocabulary) PropertySlug(key string) string { + if slug, ok := v.slugOf[key]; ok { + return slug + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (v spaceVocabulary) PropertyKey(slug string) (string, bool) { + for key, s := range v.slugOf { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (v spaceVocabulary) TypeSlug(key string) string { return BundledKeyVocabulary{}.TypeSlug(key) } +func (v spaceVocabulary) TypeKey(slug string) (string, bool) { + return BundledKeyVocabulary{}.TypeKey(slug) +} + +func customKeySnapshot(details map[string]*types.Value) *model.SmartBlockSnapshotBase { + details["id"] = str("o1") + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(details), + } +} + +// The legend carries exactly what the bundled table cannot invert — no more: +// a bundled key needs no entry, because every reader ships the table. +func TestExport_PropertyKeysLegendCarriesWhatTheTableCannot(t *testing.T) { + vocab := spaceVocabulary{slugOf: map[string]string{"6a32d4856761631534b22f85": "priority"}} + snap := customKeySnapshot(map[string]*types.Value{ + "6a32d4856761631534b22f85": {Kind: &types.Value_NumberValue{NumberValue: 3}}, + "dueDate": str("2026-07-06T08:44:05Z"), + }) + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + + var doc struct { + Properties map[string]any `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Contains(t, doc.Properties, "priority", "the custom key is spelled as its slug") + assert.Contains(t, doc.Properties, "Due date", "a bundled key is spelled as its display name") + assert.Equal(t, map[string]string{"priority": "6a32d4856761631534b22f85"}, doc.PropertyKeys, + "only the entry a package-only reader could not invert") +} + +// The point of the legend: a reader with no space gets the stored keys back. +func TestImport_PropertyKeysLegendInvertsWithoutTheSpace(t *testing.T) { + doc := `{"version": 2, "id": "o1", + "property_internal_keys": {"priority": "6a32d4856761631534b22f85"}, + "properties": {"priority": 3, "due_date": "2026-07-06T08:44:05Z"}}` + + // a package-only reader — no vocabulary at all + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Contains(t, snap.Details.Fields, "6a32d4856761631534b22f85", + "the legend is what makes the custom slug invertible offline") + assert.NotContains(t, snap.Details.Fields, "priority") + assert.Contains(t, snap.Details.Fields, "dueDate", "the bundled table still applies") +} + +// The collision the sweep found: a custom key slugs onto a term that is +// another property's stored key. A stored key is always its own address +// (§3 chain step 1), so it keeps the term and the custom key stays verbatim — +// the document can name both, and neither moves. +func TestExport_StoredKeyKeepsItsOwnTerm(t *testing.T) { + vocab := spaceVocabulary{slugOf: map[string]string{"6a32d4856761631534b22f85": "priority"}} + snap := customKeySnapshot(map[string]*types.Value{ + "6a32d4856761631534b22f85": {Kind: &types.Value_NumberValue{NumberValue: 3}}, + "priority": {Kind: &types.Value_NumberValue{NumberValue: 7}}, + }) + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, float64(7), back.Details.Fields["priority"].GetNumberValue(), + "the relation actually keyed priority keeps its value") + assert.Equal(t, float64(3), back.Details.Fields["6a32d4856761631534b22f85"].GetNumberValue(), + "and the custom one is still itself") +} + +// Every key slot is a slug slot, and the legend has to cover all of them. A +// link's `properties` and a `property` block's `property` name relations exactly +// as `/properties` does, so a space-slugged key in one of them needs the same +// inverse — and it needs it whether or not some other surface in the same +// document happened to record the entry. +func TestExport_LegendCoversBlockKeySlots(t *testing.T) { + vocab := spaceVocabulary{slugOf: map[string]string{"6a32d4856761631534b22f85": "priority"}} + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "o1", ChildrenIds: []string{"lnk", "rel"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "lnk", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: "target1", Relations: []string{"6a32d4856761631534b22f85"}}}}, + {Id: "rel", Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{Key: "6a32d4856761631534b22f85"}}}, + }, + // no property of that key on the object: nothing else can record the + // entry, which is what made the legend look like it worked + Details: fields(map[string]*types.Value{"id": str("o1"), "name": str("x")}), + } + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + + var doc struct { + PropertyKeys map[string]string `json:"property_internal_keys"` + Blocks []struct { + Type string `json:"type"` + Key string `json:"property"` + Properties []string `json:"properties"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + require.Len(t, doc.Blocks, 2) + assert.Equal(t, []string{"priority"}, doc.Blocks[0].Properties) + assert.Equal(t, "priority", doc.Blocks[1].Key) + assert.Equal(t, map[string]string{"priority": "6a32d4856761631534b22f85"}, doc.PropertyKeys, + "the slug in a block key slot is no more invertible than one in /properties") + + // and a reader with no space gets the stored key back for both slots + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + var gotLink, gotRel string + for _, b := range back.Blocks { + switch c := b.Content.(type) { + case *model.BlockContentOfLink: + require.Len(t, c.Link.Relations, 1) + gotLink = c.Link.Relations[0] + case *model.BlockContentOfRelation: + gotRel = c.Relation.Key + } + } + assert.Equal(t, "6a32d4856761631534b22f85", gotLink, "link relations must invert") + assert.Equal(t, "6a32d4856761631534b22f85", gotRel, "the property block key must invert") +} + +// The accept side of the same slot: the legend is the document's own +// statement about its spellings and is consulted first, wherever a key is +// read (§3). A link block bypassing it never inverted even when the entry +// was present. +func TestImport_LinkPropertiesConsultTheLegend(t *testing.T) { + doc := `{"version": 2, "id": "o1", + "property_internal_keys": {"priority": "6a32d4856761631534b22f85"}, + "blocks": [{"type": "link", "id": "lnk", "object_id": "target1", + "properties": ["priority", "due_date"]}]}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + + for _, b := range snap.Blocks { + if c, ok := b.Content.(*model.BlockContentOfLink); ok { + assert.Equal(t, []string{"6a32d4856761631534b22f85", "dueDate"}, c.Link.Relations) + } + } +} diff --git a/pkg/lib/anyblockjson/propertyslot_test.go b/pkg/lib/anyblockjson/propertyslot_test.go new file mode 100644 index 0000000000..6ffb2d71d1 --- /dev/null +++ b/pkg/lib/anyblockjson/propertyslot_test.go @@ -0,0 +1,125 @@ +package anyblockjson + +// propertyslot_test.go — the slot that names a property is spelled `property`, +// everywhere, and the vacated spelling `key` is refused with the repair named. +// +// Measured over 28,599 real exported documents, the slot had TWO member names +// twelve lines apart inside one dataview block, each a hard schema error in +// the other's position: `properties[]` required `key` (28,034 slots) while +// the columns, sorts and filters beside it required `property` (46,710), and +// the `property` block spelled `key` too (67,808). 2,504 real dataview blocks +// wrote the SAME spelling under both names. Generalising a member name across +// sibling structures inside one block is what generalisation IS, so a model +// that learned either spelling was rejected at the other slot — few-shot +// prompting on the corpus reproduced the split rather than resolving it. +// +// v0.41 collapses the slot onto `property`. No input alias: the pre-freeze +// rule (v0.37, v0.38) is that an old spelling is refused like any unknown +// member — a second legal spelling would keep the two-name confusion alive in +// every example an agent learns from — but refused WITH the repair named, +// because 95,842 corpus slots spell `key` and an agent prompted on an old +// export will write it. + +import ( + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The two documents of the measured defect, spelled the ONE way: the member +// name an author learned at any sibling slot works at every other. +func TestValidate_OnePropertySpellingAcrossTheDataview(t *testing.T) { + for name, doc := range map[string]string{ + "properties entry": `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","object_id":"t1", + "properties":[{"property":"name","format":"text"}],"views":[{"id":"v"}]}]}`, + "view column": `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","object_id":"t1", + "views":[{"id":"v","columns":[{"property":"name"}]}]}]}`, + "property block": `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"property","property":"name"}]}`, + } { + t.Run(name, func(t *testing.T) { + require.NoError(t, Validate([]byte(doc)), doc) + _, _, err := Unmarshal([]byte(doc), testOptions()) + require.NoError(t, err, "what Validate accepts, Unmarshal accepts (§12 I2)") + }) + } +} + +// The vacated spelling is refused at both slots that carried it, addressed at +// the member (§12), with the repair named — and Unmarshal refuses the same +// documents (I2). These scenarios pinned `key` as the REQUIRED member until +// v0.41; they now pin its refusal, because the corpus guarantees old-corpus +// agents will keep writing it. +func TestValidate_TheVacatedKeySpellingIsRefusedWithTheRepairNamed(t *testing.T) { + for name, tc := range map[string]struct{ doc, path string }{ + "dataview properties entry": { + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","object_id":"t1", + "properties":[{"key":"name","format":"text"}],"views":[{"id":"v"}]}]}`, + "/blocks/0/properties/0/key"}, + "property block": { + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"property","key":"prio"}]}`, + "/blocks/0/key"}, + "view column, where key never belonged": { + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"dataview","object_id":"t1", + "views":[{"id":"v","columns":[{"key":"name"}]}]}]}`, + "/blocks/0/views/0/columns/0/key"}, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(tc.doc)) + require.Error(t, err, "accepted the vacated spelling:\n%s", tc.doc) + assert.Contains(t, issuePaths(t, err), tc.path, + "the refusal has to name the member it is about (§12): %v", err) + assert.Contains(t, err.Error(), `spelled "property"`, + "told only \"not allowed\", the obvious wrong repair is deleting the member: %v", err) + + _, _, err = Unmarshal([]byte(tc.doc), testOptions()) + require.Error(t, err, "Unmarshal must refuse what Validate refuses (§12 I2)") + }) + } +} + +// The I1 side of the collapse: export writes the one spelling at every slot +// that names a property, and no member spelled `key` anywhere — on a snapshot +// exercising all four slots of the measured defect at once (a dataview's +// properties list, a column, a sort, a filter, and a property block beside +// them). Validate then accepts exactly what export emits. +func TestExport_EmitsOnlyThePropertySpelling(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "root", ChildrenIds: []string{"rel", "dv"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "rel", Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{Key: "note"}}}, + {Id: "dv", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + RelationLinks: []*model.RelationLink{ + {Key: "note", Format: model.RelationFormat_longtext}, + {Key: "due_date", Format: model.RelationFormat_date}, + }, + Views: []*model.BlockContentDataviewView{{Id: "v1", Name: "All", + Relations: []*model.BlockContentDataviewRelation{{Key: "note", Width: 120}}, + Sorts: []*model.BlockContentDataviewSort{{RelationKey: "due_date"}}, + Filters: []*model.BlockContentDataviewFilter{{ + RelationKey: "note", + Condition: model.BlockContentDataviewFilter_Equal, + Value: str("x"), + }}, + }}, + }}}, + }, + Details: fields(map[string]*types.Value{"id": str("root")}), + } + + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + + got := string(data) + assert.NotContains(t, got, `"key"`, + "no slot spells the vacated member — one concept, one spelling (§15 #14)") + assert.Equal(t, 6, strings.Count(got, `"property":`), + "the two properties entries, the column, the sort, the filter and the property block all spell it:\n%s", got) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (§11 I1)") +} diff --git a/pkg/lib/anyblockjson/propname_test.go b/pkg/lib/anyblockjson/propname_test.go new file mode 100644 index 0000000000..8d1a2c93f5 --- /dev/null +++ b/pkg/lib/anyblockjson/propname_test.go @@ -0,0 +1,53 @@ +package anyblockjson + +// typeProperties[].name is used only when the property has to be created +// (§2a). For a key that already exists — every bundled one — the existing +// name wins, so a document asking for a different label reads as working and +// silently does nothing. + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func typeDoc(tp string) string { + return `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [` + tp + `]}}` +} + +func TestValidate_BundledPropertyRenameWarns(t *testing.T) { + for _, tc := range []struct{ key, want, bundled string }{ + {"description", "Summary", "Description"}, + {"createdInContext", "Parent page", "Created in context"}, + } { + t.Run(tc.key, func(t *testing.T) { + var got []Issue + require.NoError(t, ValidateWarn( + []byte(typeDoc(`{"property": "`+tc.key+`", "name": "`+tc.want+`"}`)), + func(i Issue) { got = append(got, i) })) + require.Len(t, got, 1) + assert.Contains(t, got[0].Message, tc.bundled) + assert.Contains(t, got[0].Path, "/type_settings/property_definitions/0/name") + }) + } +} + +func TestValidate_PropertyNameNonTriggers(t *testing.T) { + noWarn := func(t *testing.T, doc string) { + t.Helper() + var got []Issue + require.NoError(t, ValidateWarn([]byte(doc), func(i Issue) { got = append(got, i) })) + assert.Empty(t, got) + } + t.Run("custom key keeps its name", func(t *testing.T) { + noWarn(t, typeDoc(`{"property": "verifiedUntil", "name": "Verified until", "format": "date"}`)) + }) + t.Run("bundled key with the bundled name", func(t *testing.T) { + noWarn(t, typeDoc(`{"property": "description", "name": "Description"}`)) + }) + t.Run("bundled key with no name at all", func(t *testing.T) { + noWarn(t, typeDoc(`{"property": "description"}`)) + }) +} diff --git a/pkg/lib/anyblockjson/rawnames_test.go b/pkg/lib/anyblockjson/rawnames_test.go new file mode 100644 index 0000000000..21084a8cf9 --- /dev/null +++ b/pkg/lib/anyblockjson/rawnames_test.go @@ -0,0 +1,660 @@ +package anyblockjson + +// rawnames_test.go — the raw-name spelling rule under attack: the names the +// rule newly admits as keys (`#`, `☕`, `C++`, `50% done`, non-Latin), two +// properties colliding inside one document, a name equal to another live +// stored key, and the map-less reader's type-scoped resolution with its loud +// error. I1 (Marshal never emits what its own Validate rejects) and I2 +// (Validate and Unmarshal agree) are asserted on every arm, plus the §11 +// fixpoint: a second export of the re-import is byte-identical. + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +// nameVocab is the stand-in for a SPACE-backed vocabulary in this package's +// own tests, and it is held to the shipped one's contract rather than to a +// convenient approximation of it. storeresolver is the implementation it +// mirrors, method for method: +// +// - the emit side runs grant's degradation ladder, so a name that is some +// other live entity's stored key degrades to ` ()` here too, +// instead of silently falling back to the plain stored key; +// - the candidate lists are SETS, sorted, keyed by the GRANTED LABEL — a +// claimant that degraded is listed under the spelling it actually writes; +// - an exact live stored key outranks every table (verbatim-first); +// - an ambiguous candidate set is REFUSED. Nothing behind it is consulted, +// the fold layer included. +// +// The last two are what a double gets wrong for free. Falling through to +// BundledKeyVocabulary after an ambiguous set — which this double used to do +// — means every test that thought it was pinning "the importer refuses to +// guess" was really pinning "the double guessed the bundled twin", and no +// assertion in the file could tell the difference. Duplicate-blind candidate +// lists are the same shape of blindness one layer down: the importer reads a +// candidate list as a COUNT, so a double that cannot produce a wrong count +// cannot test what the importer does with one (keycandidates_test.go supplies +// a deliberately duplicating vocabulary for exactly that). +type nameVocab struct { + names map[string]string // stored key -> display name + typeNames map[string]string // stored type key -> display name + typeProps map[string][]string // stored type key -> its property keys +} + +func (v nameVocab) storedKey(term string) bool { + _, ok := v.names[term] + return ok +} + +func (v nameVocab) storedTypeKey(term string) bool { + _, ok := v.typeNames[term] + return ok +} + +// grantedLabel is keyMaps.grant's ladder, the half a double can drop without +// any assertion noticing: a name that is some OTHER live entity's stored key +// could never resolve back to its owner, because an exact stored key outranks +// every table at every reader. Its holder therefore degrades to +// ` ()`, and to no label at all when even that string is taken or +// cannot be built — in which case the stored key is the spelling. +func grantedLabel(name, key string, storedKey func(string) bool) string { + if name == "" || !storedKey(name) { + return name + } + label := DisambiguatedKeySpelling(name, key) + if label == "" || storedKey(label) { + return "" + } + return label +} + +// propertyLabel / typeLabel are labelByKey: the spelling one live, visible, +// non-bundled entity is granted. A bundled key takes the code table's word in +// every space, so the space row never speaks for it. +func (v nameVocab) propertyLabel(key string) string { + if bundle.HasRelation(domain.RelationKey(key)) { + return "" + } + return grantedLabel(PropertyLabel(key, v.names[key]), key, v.storedKey) +} + +func (v nameVocab) typeLabel(key string) string { + if bundle.HasObjectTypeByKey(domain.TypeKey(key)) { + return "" + } + return grantedLabel(TypeLabel(key, v.typeNames[key]), key, v.storedTypeKey) +} + +func (v nameVocab) PropertySlug(key string) string { + if bundle.HasRelation(domain.RelationKey(key)) { + // the bundled table is the authority in every space and offline — + // unless a live stored key owns the very string, which outranks it + if spelling := (BundledKeyVocabulary{}).PropertySlug(key); spelling != key && !v.storedKey(spelling) { + return spelling + } + return key + } + if label := v.propertyLabel(key); label != "" { + return label + } + return key +} + +func (v nameVocab) TypeSlug(key string) string { + if bundle.HasObjectTypeByKey(domain.TypeKey(key)) { + if spelling := (BundledKeyVocabulary{}).TypeSlug(key); spelling != key && !v.storedTypeKey(spelling) { + return spelling + } + return key + } + if label := v.typeLabel(key); label != "" { + return label + } + return key +} + +func (v nameVocab) PropertyKey(spelling string) (string, bool) { + if v.storedKey(spelling) { + return spelling, false // verbatim-first: an exact stored key wins + } + switch cands := v.PropertyKeyCandidates(spelling); len(cands) { + case 1: + return cands[0], true + case 0: + return BundledKeyVocabulary{}.PropertyKey(spelling) // the forgiving fold + default: + // several live claimants: refused outright, and the fold layer is not + // consulted either. A caller with type context may resolve this; a + // caller without one degrades to the verbatim term, never to a guess + return spelling, false + } +} + +func (v nameVocab) TypeKey(spelling string) (string, bool) { + if v.storedTypeKey(spelling) { + return spelling, false + } + switch cands := v.TypeKeyCandidates(spelling); len(cands) { + case 1: + return cands[0], true + case 0: + return BundledKeyVocabulary{}.TypeKey(spelling) + default: + return spelling, false + } +} + +// PropertyKeyCandidates / TypeKeyCandidates are keysByLabel plus the bundled +// table's binding: every live claimant of the exact spelling, as a sorted SET. +// They are keyed by the granted LABEL, not by the raw name — a claimant that +// degraded through the ladder answers to the spelling it actually writes, and +// no longer to the one it lost. +func (v nameVocab) PropertyKeyCandidates(spelling string) []string { + out := newTestKeySet() + for key := range v.names { + if v.propertyLabel(key) == spelling { + out.add(key) + } + } + if key, ok := BundledPropertyKeyByName(spelling); ok { + out.add(key) + } + return out.sorted() +} + +func (v nameVocab) TypeKeyCandidates(spelling string) []string { + out := newTestKeySet() + for key := range v.typeNames { + if v.typeLabel(key) == spelling { + out.add(key) + } + } + if key, ok := BundledTypeKeyByName(spelling); ok { + out.add(key) + } + return out.sorted() +} + +// TypePropertyKeys is a set too: the importer intersects it with the candidate +// list and counts what survives, so a property the type named twice would stop +// the type from singling out its own property. +func (v nameVocab) TypePropertyKeys(typeKey string) []string { + out := newTestKeySet() + for _, key := range v.typeProps[typeKey] { + out.add(key) + } + return out.keys +} + +func (v nameVocab) PropertyTermFacts(term string) KeyTermFacts { + facts := KeyTermFacts{LiveStoredKey: v.storedKey(term)} + facts.ExtendsName = extendedLiveName(term, v.names, PropertyLabel) + return facts +} + +func (v nameVocab) TypeTermFacts(term string) KeyTermFacts { + return KeyTermFacts{ + LiveStoredKey: v.storedTypeKey(term), + ExtendsName: extendedLiveName(term, v.typeNames, TypeLabel), + } +} + +// extendedLiveName is extendsLiveName: the live name the term extends past a +// word boundary, longest first, ties broken lexicographically so the answer +// does not depend on Go's map order. +func extendedLiveName(term string, names map[string]string, label func(key, name string) string) string { + var best string + for key, name := range names { + name = label(key, name) + if name == "" || !KeyTermExtendsName(term, name) { + continue + } + if len(name) > len(best) || (len(name) == len(best) && name < best) { + best = name + } + } + return best +} + +// testKeySet accumulates stored keys in first-seen order, dropping repeats. +type testKeySet struct { + keys []string + seen map[string]bool +} + +func newTestKeySet() *testKeySet { + return &testKeySet{seen: map[string]bool{}} +} + +func (s *testKeySet) add(key string) { + if key == "" || s.seen[key] { + return + } + s.seen[key] = true + s.keys = append(s.keys, key) +} + +func (s *testKeySet) sorted() []string { + sortStrings(s.keys) + return s.keys +} + +func sortStrings(s []string) { + for i := 1; i < len(s); i++ { + for j := i; j > 0 && s[j] < s[j-1]; j-- { + s[j], s[j-1] = s[j-1], s[j] + } + } +} + +// The names the old normalized spelling could not carry at all — each used +// to need a fallback or an escape, and each is now a plain key. The whole +// codec survives every one: Marshal validates (I1), the document inverts to +// the stored keys (I2's substance), and the round trip is a fixpoint. +func TestRawNames_TheNamesNormalizationCouldNotCarry(t *testing.T) { + keys := map[string]string{ + "6a7663db61fab21cd4b90001": "#", + "6a7663db61fab21cd4b90002": "☕", + "6a7663db61fab21cd4b90003": "C++", + "6a7663db61fab21cd4b90004": "50% done", + "6a7663db61fab21cd4b90005": "Дата выполнения", + "6a7663db61fab21cd4b90006": "作業内容", + "6a7663db61fab21cd4b90007": "All", + "6a7663db61fab21cd4b90008": "What's missing", + } + vocab := nameVocab{names: keys} + details := map[string]*types.Value{"id": pbtypes.String("o1")} + for key := range keys { + details[key] = pbtypes.String("v:" + keys[key]) + } + snap := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: details}} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + + // then — I1, and every name is a member key exactly as written + require.NoError(t, Validate(data), "I1:\n%s", data) + var doc struct { + Properties map[string]string `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + for key, name := range keys { + assert.Equalf(t, "v:"+name, doc.Properties[name], "the name %q is the member key", name) + assert.Equalf(t, key, doc.PropertyKeys[name], "and the legend inverts it") + } + + // and back onto the stored keys — through the legend alone, no vocabulary + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + for key, name := range keys { + require.NotNilf(t, back.Details.Fields[key], "name %q lost its key", name) + assert.Equal(t, "v:"+name, back.Details.Fields[key].GetStringValue()) + } + + // fixpoint + again, err := Marshal(model.SmartBlockType_Page, back, Options{Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, string(data), string(again), "the round trip is byte-stable") +} + +// Two properties colliding inside one document: EVERY claimant degrades +// through the ladder, deterministically, and the document still carries both +// values and inverts both keys. The same document also plants a name equal +// to a live stored key, which is refused the same way. +func TestRawNames_TwoPropertiesCollideInsideOneDocument(t *testing.T) { + const ( + keyA = "6a7663db61fab21cd4b90011" + keyB = "6a7663db61fab21cd4b90022" + ) + vocab := nameVocab{names: map[string]string{ + keyA: "Projects", + keyB: "Projects", + }} + snap := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("o1"), + keyA: pbtypes.String("value of A"), + keyB: pbtypes.String("value of B"), + }}} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + + // then — I1, both claimants suffixed off their own tails, both values kept + require.NoError(t, Validate(data), "I1:\n%s", data) + var doc struct { + Properties map[string]string `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "value of A", doc.Properties["Projects (b90011)"]) + assert.Equal(t, "value of B", doc.Properties["Projects (b90022)"]) + assert.NotContains(t, doc.Properties, "Projects", + "the plain name is written for NOBODY: a plain spelling must never be one of two same-named claimants") + assert.Equal(t, map[string]string{ + "Projects (b90011)": keyA, + "Projects (b90022)": keyB, + }, doc.PropertyKeys) + + // I2's substance: both values come home on their own keys + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, "value of A", back.Details.Fields[keyA].GetStringValue()) + assert.Equal(t, "value of B", back.Details.Fields[keyB].GetStringValue()) + + // fixpoint — the suffix is deterministic off the name and the key's own + // tail, so a second generation re-derives the identical spellings + again, err := Marshal(model.SmartBlockType_Page, back, Options{Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, string(data), string(again)) +} + +// A name that IS some other live property's stored key. Verbatim-first +// outranks every table at every reader, so that spelling can never resolve to +// the entity merely NAMED it — and the holder degrades through grant's ladder +// rather than writing a spelling that lands on somebody else's row. The bson +// key is unreadable, so the rung taken is the disambiguated name, not the raw +// key; the entity that OWNS the string keeps its own plain name, because +// nothing contests it. +// +// This is the arm this file's header has always claimed and the test double +// could not previously produce: the double skipped the ladder's middle rung +// and answered with the raw 24-hex stored key, which is a spelling the shipped +// vocabulary never writes for a bson id. +func TestRawNames_ANameThatIsAnotherLivePropertysStoredKey(t *testing.T) { + // given — one relation whose STORED KEY is the string "Projects", and + // another whose display NAME is "Projects" + const holder = "6a7663db61fab21cd4b90044" + vocab := nameVocab{names: map[string]string{ + "Projects": "Task list", + holder: "Projects", + }} + snap := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("o1"), + "Projects": pbtypes.String("value of the key holder"), + holder: pbtypes.String("value of the named one"), + }}} + want := map[string]string{ + "Task list": "value of the key holder", + "Projects (b90044)": "value of the named one", + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + + // then — I1, and neither claimant wrote the contested string + require.NoError(t, Validate(data), "I1:\n%s", data) + var doc struct { + Properties map[string]string `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, want, doc.Properties) + assert.NotContains(t, doc.Properties, "Projects", + "the string is one entity's address and the other's lost name: it is written for neither") + assert.Equal(t, holder, doc.PropertyKeys["Projects (b90044)"], + "the degraded spelling is nobody's chain, so the legend states it") + + // and back onto both stored keys + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, "value of the key holder", back.Details.Fields["Projects"].GetStringValue()) + assert.Equal(t, "value of the named one", back.Details.Fields[holder].GetStringValue()) + + // fixpoint — the tail6 suffix is derived, so generation 2 re-derives it + again, err := Marshal(model.SmartBlockType_Page, back, Options{Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, string(data), string(again)) +} + +// An edge-whitespace name is carried verbatim — the format warns (§12) and +// does not trim — and Validate's hygiene warning names it without refusing +// the document. +func TestRawNames_EdgeWhitespaceIsCarriedAndWarned(t *testing.T) { + const key = "6a7663db61fab21cd4b90033" + vocab := nameVocab{names: map[string]string{key: "Email 📧 "}} + snap := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("o1"), + key: pbtypes.String("x@y.z"), + }}} + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + + var warns []Issue + require.NoError(t, ValidateWarn(data, func(i Issue) { warns = append(warns, i) }), + "warned, never refused — one stored name must not make an object unexportable") + var hygiene []string + for _, w := range warns { + if strings.Contains(w.Message, "edge whitespace") { + hygiene = append(hygiene, w.Message) + } + } + require.NotEmpty(t, hygiene, "the invisible byte is worth one line to the caller") + assert.Contains(t, hygiene[0], `"Email 📧 "`) + + // and the invisible-code-point arm: a variation selector in a legend key + doc := `{"version":2,"id":"o1","property_internal_keys":{"Star️":"` + key + `"},` + + `"properties":{"Star️":1}}` + warns = nil + require.NoError(t, ValidateWarn([]byte(doc), func(i Issue) { warns = append(warns, i) })) + found := false + for _, w := range warns { + if strings.Contains(w.Message, "invisible code point") { + found = true + } + } + assert.True(t, found, "a default-ignorable code point is named, with its code") +} + +// The map-less reader resolves a shared name within the declared type — the +// overwhelming case — and raises a LOUD error when the type is not enough: +// it never guesses and never mints a phantom key while two live properties +// bear that exact name. +func TestRawNames_TypeScopedResolution(t *testing.T) { + const ( + keyA = "6a7663db61fab21cd4b90011" + keyB = "6a7663db61fab21cd4b90022" + taskType = "6a7663db61fab21cd4b90099" + ) + vocab := nameVocab{ + names: map[string]string{keyA: "Projects", keyB: "Projects"}, + typeNames: map[string]string{taskType: "Sprint"}, + typeProps: map[string][]string{taskType: {keyA, "name"}}, + } + + t.Run("the declared type singles the claimant out", func(t *testing.T) { + doc := `{"version":2,"id":"o1","type":"Sprint","properties":{"Projects":"resolved"}}` + _, back, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, "resolved", back.Details.Fields[keyA].GetStringValue(), + "unambiguous among the type's own properties — resolved, silently") + assert.Nil(t, back.Details.Fields[keyB]) + }) + + t.Run("a type that cannot place the name errors loudly", func(t *testing.T) { + doc := `{"version":2,"id":"o1","type":"Page","properties":{"Projects":"?"}}` + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + require.Error(t, err, "never a guess, never a phantom while two live properties bear the name") + assert.Contains(t, err.Error(), `"Projects"`) + assert.Contains(t, err.Error(), memberPropertyInternalKeys, + "the error asks for the legend — the one statement that settles it") + }) + + t.Run("the legend outranks the whole question", func(t *testing.T) { + doc := `{"version":2,"id":"o1","type":"Page",` + + `"property_internal_keys":{"Projects":"` + keyB + `"},` + + `"properties":{"Projects":"stated"}}` + _, back, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, "stated", back.Details.Fields[keyB].GetStringValue()) + }) + + t.Run("a shared TYPE name errors loudly too", func(t *testing.T) { + shared := nameVocab{typeNames: map[string]string{ + "6a7663db61fab21cd4b90777": "Meeting", + "6a7663db61fab21cd4b90888": "Meeting", + }} + doc := `{"version":2,"id":"o1","type":"Meeting"}` + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: shared}) + require.Error(t, err, "the type is the scope — there is nothing wider to resolve inside") + assert.Contains(t, err.Error(), memberTypeInternalKeys) + }) +} + +// The two verbatim-resolution warnings, once per term: the phantom (a term +// no live entity answers to, stored verbatim as a new key) and the glued +// annotation (a term extending a live or bundled name past a word boundary). +func TestRawNames_VerbatimTermWarnings(t *testing.T) { + const key = "6a7663db61fab21cd4b90011" + vocab := nameVocab{names: map[string]string{key: "Lists [in work]"}} + + collect := func(doc string, opts Options) []string { + var msgs []string + opts.GenerateId = seqIds("g") + opts.OnWarning = func(i Issue) { msgs = append(msgs, i.Message) } + _, _, err := Unmarshal([]byte(doc), opts) + require.NoError(t, err) + return msgs + } + + t.Run("a stale or guessed name mints a phantom, and says so", func(t *testing.T) { + msgs := collect(`{"version":2,"id":"o1","properties":{"Budget":"1", "b1": {"x": 1}}}`, + Options{Keys: vocab}) + joined := strings.Join(msgs, "\n") + assert.Contains(t, joined, `"Budget"`) + assert.Contains(t, joined, "phantom") + }) + + t.Run("the glued annotation names the live name it extends", func(t *testing.T) { + msgs := collect(`{"version":2,"id":"o1","properties":{"Lists [in work] (text)":"x"}}`, + Options{Keys: vocab}) + joined := strings.Join(msgs, "\n") + assert.Contains(t, joined, `"Lists [in work] (text)"`) + assert.Contains(t, joined, `"Lists [in work]"`) + assert.Contains(t, joined, "glued") + }) + + t.Run("a bundled name's glue is caught with no vocabulary at all", func(t *testing.T) { + msgs := collect(`{"version":2,"id":"o1","properties":{"Creation date (text)":"x"}}`, + Options{}) + joined := strings.Join(msgs, "\n") + assert.Contains(t, joined, `"Creation date"`) + assert.Contains(t, joined, "glued") + }) + + t.Run("one term, one warning, however many slots name it", func(t *testing.T) { + doc := `{"version":2,"id":"o1","properties":{"Budget":"1"},"blocks":[ + {"id":"dv","type":"dataview","properties":[{"property":"Budget","format":"number"}], + "views":[{"id":"v1","sorts":[{"property":"Budget"}]}]}]}` + msgs := collect(doc, Options{Keys: vocab}) + count := 0 + for _, m := range msgs { + if strings.Contains(m, `"Budget"`) && strings.Contains(m, "phantom") { + count++ + } + } + assert.Equal(t, 1, count, "the diagnosis is a fact about the term, not about any one slot") + }) +} + +// The fixpoint under a SPACE-shaped collision that includes a bundled twin: +// a custom property named "Description" beside the bundled one, both in one +// document. Both degrade — the bundled key to its readable stored key, the +// custom one to its suffix — and the whole document round-trips. +func TestRawNames_BundledAndCustomShareOneName(t *testing.T) { + const custom = "6a7663db61fab21cd4b90055" + vocab := nameVocab{names: map[string]string{custom: "Description"}} + snap := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("o1"), + "description": pbtypes.String("the bundled one"), + custom: pbtypes.String("the custom one"), + }}} + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + require.NoError(t, Validate(data), "I1:\n%s", data) + + var doc struct { + Properties map[string]string `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "the bundled one", doc.Properties["description"], + "the bundled claimant's stored key is readable: rung (a)") + assert.Equal(t, "the custom one", doc.Properties[fmt.Sprintf("Description (%s)", custom[len(custom)-6:])]) + assert.NotContains(t, doc.PropertyKeys, "description", + "no entry owed: the term is the bundled key's own stored key, and the bundled "+ + "fold inverts it in every reader that ships the table") + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, "the bundled one", back.Details.Fields["description"].GetStringValue()) + assert.Equal(t, "the custom one", back.Details.Fields[custom].GetStringValue()) + + again, err := Marshal(model.SmartBlockType_Page, back, Options{Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, string(data), string(again)) +} + +// An attribution key never contests a spelling: export writes it and import +// DROPS it, so a generation-2 census will not hold it — a claimant it had +// suffixed would un-suffix, and the round trip stopped being byte-stable in +// exactly the spaces holding a custom name-twin of "Created by" (a real +// production space does). The attribution claimant yields its plain name +// (its own bundled stored key is readable), and the normal claimant keeps +// the verdict it will re-derive without it. +func TestRawNames_AttributionNeverContestsASpelling(t *testing.T) { + const custom = "6a7663db61fab21cd4b90077" + vocab := nameVocab{names: map[string]string{custom: "Created by"}} + snap := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("o1"), + "creator": pbtypes.String("_participant_a_b_A5qTLyde3S1q9NRyFeSeN6UWwa6VwwXEJbMACJwMfez3BGVD"), + custom: pbtypes.String("the twin's value"), + }}} + opts := Options{Keys: vocab, SpaceId: "a.b"} + + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + require.NoError(t, Validate(data), "I1:\n%s", data) + + var doc struct { + Properties map[string]any `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "the twin's value", doc.Properties["Created by"], + "the normal claimant keeps the plain name — no suffix that generation 2 would drop") + assert.Contains(t, doc.Properties, "creator", + "the attribution line yields to its own stored key, always its own address") + assert.Equal(t, custom, doc.PropertyKeys["Created by"]) + + // the fixpoint half: the re-import drops the attribution line, and the + // next export still spells the twin identically + _, back, err := Unmarshal(data, opts) + require.NoError(t, err) + assert.Nil(t, back.Details.Fields["creator"], "attribution does not survive a round trip") + again, err := Marshal(model.SmartBlockType_Page, back, opts) + require.NoError(t, err) + var doc2 struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(again, &doc2)) + assert.Equal(t, "the twin's value", doc2.Properties["Created by"], + "generation 2 re-derives the same spelling — the fixpoint the yield exists for") +} diff --git a/pkg/lib/anyblockjson/refname_test.go b/pkg/lib/anyblockjson/refname_test.go new file mode 100644 index 0000000000..e0c62a5499 --- /dev/null +++ b/pkg/lib/anyblockjson/refname_test.go @@ -0,0 +1,135 @@ +package anyblockjson + +import ( + "math/rand" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/filterstring" +) + +// refNameNormalize (refs.go) is the identifier normalization that used to +// mint KEY labels, surviving for its one remaining surface: the informative +// `#name` reference suffix (§9). Key spellings are raw names now and need no +// normalization; the suffix still does, because its grammar is what makes +// the `#` split safe. The rules — and these pins — are unchanged from the +// key-label era on purpose: the suffix is informative and trimmed unread, +// and keeping the bytes stable keeps every already-written reference +// identical on its next export. +// +// Every case here is a decision with a plausible alternative, and the +// alternative is named in the case's own comment — a table that only pinned +// the happy path would pass with a transliterating normalizer. +func TestRefNameNormalize(t *testing.T) { + for _, tc := range []struct { + in string + want string + why string + }{ + {"Publish Date", "publish_date", "a name separates its own words, and this one does"}, + {"Original creation date", "original_creation_date", ""}, + // camelCase is a KEY phenomenon; a person types "Due Date". Hump + // splitting is what turned "P2P Sync" into `p_2_p_sync`. + {"iconEmoji", "iconemoji", "a camelCase NAME is a key someone pasted into a name field"}, + {"mediaArtistURL", "mediaartisturl", "no acronym rule can tell SDKs from XMLParser anyway"}, + {"P2P Sync", "p2p_sync", "a letter and a digit are one word"}, + {"GitHub Stars", "github_stars", ""}, + {"Platform SDKs", "platform_sdks", ""}, + // a leading `_` run is CONTENT: 20 production relations from two + // integrations namespace themselves this way. + {"__amemory_salience", "__amemory_salience", "a namespace prefix survives"}, + {"trailing_", "trailing", "a trailing run is still a gap between a word and nothing"}, + {"a__b", "a_b", "an interior run still collapses"}, + {"___", "", "underscores alone name nothing"}, + {" spaced name ", "spaced_name", "separator runs collapse and edges trim"}, + + // non-Latin scripts are kept, never transliterated + {"Тоггл", "тоггл", "no transliteration"}, + {"日本語のプロパティ", "日本語のプロパティ", "no transliteration"}, + {"tiếng Việt", "tiếng_việt", "no transliteration"}, + + // emoji, punctuation and symbols are separators, not letters + {"Priority 📌", "priority", ""}, + {"C#", "c", "the suffix grammar admits no `#` — the split guarantee"}, + {"#", "", "nothing left to name means no suffix, never a dangling `#`"}, + {"", "", ""}, + + // the leading-`_` escape for the two grammar faults — kept for byte + // stability with every reference already written + {"50% done", "_50_done", "identStart is a letter or `_`, never a digit"}, + {"All", "_all", "`all` is a reserved word of the filter grammar"}, + + // A combining mark modifies the letter before it: neither a + // separator (which would cut the word at every virama) nor + // droppable (this script writes its VOWELS as marks, and + // मिल/मूल/मल/मैल would all become मल). + {"क्षत्रिय", "क्षत्रिय", "marks are kept: they carry the word"}, + {"हिन्दी", "हिन्दी", ""}, + {"İstanbul", "istanbul", "lowercasing İ leaves a combining dot behind"}, + } { + t.Run(tc.in, func(t *testing.T) { + assert.Equal(t, tc.want, refNameNormalize(tc.in), tc.why) + }) + } + + // two visually identical names can be different byte sequences, and two + // exports of one reference must not differ by normalization form + t.Run("NFD and NFC forms of one name normalize to one suffix", func(t *testing.T) { + nfc := "Ünïcødé" + nfd := norm.NFD.String(nfc) + assert.NotEqual(t, nfc, nfd, "the fixture has to actually differ in bytes") + assert.Equal(t, refNameNormalize(nfc), refNameNormalize(nfd)) + assert.Equal(t, "ünïcødé", refNameNormalize(nfd)) + }) + + // the control that makes the marks rows meaningful: four words that + // differ ONLY by their marks must stay four different suffixes + t.Run("marks distinguish words rather than collapsing them", func(t *testing.T) { + seen := map[string]string{} + for _, name := range []string{"मिल", "मूल", "मल", "मैल"} { + label := refNameNormalize(name) + if prev, clash := seen[label]; clash { + t.Fatalf("%q and %q both normalize to %q — the mark carries the meaning", prev, name, label) + } + seen[label] = name + } + assert.Len(t, seen, 4, "four words, four suffixes") + }) +} + +// The suffix grammar's whole contract: whatever refNameNormalize mints, when +// it is not empty, is a bare identifier the filter grammar accepts — which +// is a strictly narrower shape than "contains no `#`", so the split +// guarantee (§9) rides along. Asserted as a PROPERTY over hostile input +// rather than as a case list, so it keeps holding when either side grows. +func TestRefNameNormalize_EverySuffixIsABareIdentifier(t *testing.T) { + inputs := []string{ + "", " ", "#", "\U0001F389", "50% done", "007", "_", "__", "-", + "All", "NOT", "in", "id", "type", "Cost & type", "What's missing", + "Тоггл", "日本語のプロパティ", "क्षत्रिय", "C#", "C++", "a.b", "a#b#c", + "\t\n\v", "a\nb", "­", "​", "é", "ß", "fi", "Ⅻ", "½", "①", + strings.Repeat("a", 200), strings.Repeat("é ", 90), + } + alphabet := []rune("aZ_-. 0б日ế\U0001F389#/\\\"'\t́é½Ⅻ") + rnd := rand.New(rand.NewSource(7383)) + for i := 0; i < 4000; i++ { + var b strings.Builder + for n := rnd.Intn(12); n >= 0; n-- { + b.WriteRune(alphabet[rnd.Intn(len(alphabet))]) + } + inputs = append(inputs, b.String()) + } + for _, in := range inputs { + label := refNameNormalize(in) + if label == "" { + continue // "no suffix" is always a legal answer + } + require.Truef(t, filterstring.IsBareKey(label), + "suffix %q minted from %q is not a bare identifier", label, in) + require.NotContainsf(t, label, "#", "minted from %q — the split guarantee", in) + } +} diff --git a/pkg/lib/anyblockjson/refs.go b/pkg/lib/anyblockjson/refs.go new file mode 100644 index 0000000000..66c62a468b --- /dev/null +++ b/pkg/lib/anyblockjson/refs.go @@ -0,0 +1,521 @@ +package anyblockjson + +// refs.go — object references (§9): the informative `#name` suffix and the +// participant fold. +// +// An object reference in this format is a full id, always (§9a deleted the +// compaction legend). Two amendments make one readable without ceasing to be +// an address: +// +// - **The `#name` suffix.** A reference MAY carry `#` after the id — +// `bafyrei…#local_first_ux` — where the name is the referenced object's +// display name normalized into an identifier grammar (refNameNormalize: +// letters, digits, `_`, combining marks, nothing else). Key spellings +// stopped being normalized when raw naming landed; the suffix still is, +// because its grammar is what keeps the `#` split safe. +// The suffix is INFORMATIVE ONLY: import trims it at the first `#` and +// never resolves it, so a stale name costs nothing and two objects +// sharing one name collide on nothing. It exists so a human or a model +// reading a document sees what a reference points at instead of a +// 59-character CID. A bare id with no suffix is equally valid, and is +// what a writer with no name in hand writes. +// +// - **The participant fold.** `_participant__` is a +// derived id: the space id is the document's own space restated, and the +// identity is the whole of the content. When Options.SpaceId names the +// space, export folds the composite down to the bare identity and import +// rebuilds the composite (domain.NewParticipantId) — 135 characters down +// to 48, and the same member re-addresses correctly when a document +// crosses spaces, because the reader rebuilds against ITS space. +// +// The split at `#` is unconditional and safe from both ends, verified rather +// than assumed: no id form this format writes can contain `#` (CIDs are +// base32 `[a-z2-7]`, participant ids base32+base58, `_ot`/`_br` ids are +// `[a-zA-Z0-9_]` across all 223 bundled keys, `_date_…`/`_missing_object` +// are fixed shapes; measured over 37,429 production documents: zero +// id-shaped values contain `#`) — and the name half is normalized through a +// grammar that admits no `#` either. + +import ( + "strings" + "unicode" + + "github.com/anyproto/any-sync/util/crypto" + "github.com/ipfs/go-cid" + "golang.org/x/text/unicode/norm" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson/filterstring" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// ObjectNameResolver names an object for the informative reference suffix +// (§9). It is the object-namespace sibling of ParticipantResolver, and it is +// export-only: import trims the suffix without ever asking anyone. +// +// A resolver that cannot name an id returns false and the reference is +// written bare — never with a partial or invented suffix. An empty or +// whitespace name is treated as no name at the seam (refNameLabel), the same +// discipline the participant seam applies, so an implementation answering +// ("", true) cannot put a dangling `#` on every reference in an export. +type ObjectNameResolver interface { + ObjectName(id string) (string, bool) +} + +// ObjectExistenceResolver answers whether the space's store holds an object +// under an id — the question behind the missing-reference rule (§9): a +// reference to an object that does not exist in the SPACE is not written as +// if it did. It is an optional capability of Options.ResolveObjectNames, +// discovered by type assertion (the TypeResolver pattern, §2d): the resolver +// that can NAME an object — one point lookup on the space index — is the one +// that can also say whether the row is there at all, and a caller without it +// keeps a well-defined degradation: nothing is rewritten and nothing is +// dropped, because the absence of an answer is not evidence of absence. +// +// ObjectName is NOT this question and must never stand in for it: its ok is +// `name != ""`, so it answers "no" for an object that exists UNTITLED — and +// untitled objects are common. An export that conflated the two would +// rewrite live references to `_missing_object`. +// +// known=false means the resolver could not ask (a store failure): the caller +// treats the reference exactly as if the capability were absent. exists is +// a statement about the store's rows, tombstones included — a deleted +// object keeps an index row, so a reference to it is NOT missing: the id +// still means something in this space. +type ObjectExistenceResolver interface { + ObjectExists(id string) (exists, known bool) +} + +// ObjectDeletionResolver answers whether an id names an object the space +// DELETED — a tombstone: the index keeps a row stripped to its bookkeeping +// (`{id, spaceId, isDeleted, sync*}`) and nothing else. +// +// It is deliberately separate from ObjectExists, which counts a tombstone as +// existing and says so: "a deleted object keeps an index row, so a reference +// to it is NOT missing: the id still means something in this space". That +// rule stands for every reference slot but one. An ICON is the exception, +// because an icon is OPTIONAL: a link or a mention block must have a target, +// so a dangling one is rewritten to the sentinel rather than dropped, but an +// object with no icon is an ordinary object. Measured over a 77-space +// export, 134 bookmark documents shipped an icon pointing at a favicon whose +// file object had been deleted — every one of the 134 confirmed a tombstone +// in its own space's store. +// +// known=false means the resolver could not ask; the caller then treats the +// reference as live, so a store failure never removes an icon. +type ObjectDeletionResolver interface { + ObjectDeleted(id string) (deleted, known bool) +} + +// DroppedDeletedIconRef reports that an icon image reference names an object +// the space deleted, so export drops the icon and falls through to whatever +// channel is left (§2b) — the same fall-through an image that is not an +// object id already takes. +// +// Exported because snapshotdiff must apply the SAME predicate: `iconImage` +// is a DETAIL, so without this the comparator reads every dropped icon as +// data loss — the drift class that once produced 1,344 false failures in a +// single sweep (§11). +func DroppedDeletedIconRef(opts Options, id string) bool { + if !isObjectIdShaped(id) { + return false + } + res, ok := opts.ResolveObjectNames.(ObjectDeletionResolver) + if !ok { + return false + } + deleted, known := res.ObjectDeleted(id) + return known && deleted +} + +// isObjectIdShaped reports whether s parses as a content id (CID) — the +// shape of every object and file id a space actually mints. It is the gate +// that keeps the existence question OFF everything that is not a space +// store row's address: derived ids (`_date_…` is virtual, `_ot…`/`_br…` +// bundled urls and cross-space participant composites resolve against other +// authorities than this space's index), account identities, type and +// property keys, doc-local block ids — none of these parse as a CID, so +// none can be declared missing by a store that was never their authority. +// The cheap length gate mirrors isAccountIdentity's: no CID is shorter than +// 46 characters, and nearly every non-id fails there. +func isObjectIdShaped(s string) bool { + if len(s) < 46 { + return false + } + _, err := cid.Decode(s) + return err == nil +} + +// missingFromSpace reports that id names an object the wired store says the +// space does not hold — the only fact that may rewrite or drop a reference +// (§9). Three gates, each fail-safe toward "not missing": the id must be +// object-id-shaped (isObjectIdShaped — an id the space index was never the +// authority for cannot be missing from it), the existence capability must be +// wired (a package-only export has no store to ask, and "missing from this +// EXPORT" is not "missing from the space"), and the store must actually +// answer (known) — a store failure leaves the reference untouched. +func missingFromSpace(opts Options, id string) bool { + if !isObjectIdShaped(id) { + return false + } + res, ok := opts.ResolveObjectNames.(ObjectExistenceResolver) + if !ok { + return false + } + exists, known := res.ObjectExists(id) + return known && !exists +} + +// DroppedMissingObjectRef reports whether export drops entry from a +// LIST-valued reference slot — an objects/files property value (§3), a +// property document's `object_types` (§2d): the stored `_missing_object` +// sentinel, or an object id the wired store says the space does not hold. +// A list expresses absence by being shorter; singular slots rewrite to the +// sentinel instead (§9) and are not this predicate's business. +// +// Exported because snapshotdiff — the comparator behind the corpus sweep — +// must apply the SAME predicate to both sides, or every dropped-by-design +// entry reports as data loss (the drift class that once produced 1,344 +// false failures in one sweep, §11). With no capability wired it drops +// nothing, sentinel included: a package-only export passes every entry +// through verbatim. +func DroppedMissingObjectRef(opts Options, entry string) bool { + if entry == missingObjectId { + _, ok := opts.ResolveObjectNames.(ObjectExistenceResolver) + return ok + } + return missingFromSpace(opts, entry) +} + +// refNameSep splits an object reference from its informative name suffix. +// The FIRST occurrence splits (§9): the id half can never contain one, and +// the name half never does either once normalized, so first-vs-last is not a +// choice between behaviours — it is the same answer stated defensively. +const refNameSep = "#" + +// maxRefNameLen bounds the suffix. The suffix is a glanceable hint, not an +// address, so a name that normalizes past the bound is truncated rather than +// dropped — truncation invents nothing here, unlike a key label (label.go), +// which IS an address and refuses instead. +const maxRefNameLen = 64 + +// splitRefName splits a reference at the first `#` into the id and the +// informative name. A reference with no `#`, and the degenerate `#…` whose +// id half would be empty, split into themselves and no name: import never +// invents an empty id out of a malformed reference. +func splitRefName(ref string) (id, name string) { + if i := strings.Index(ref, refNameSep); i > 0 { + return ref[:i], ref[i+1:] + } + return ref, "" +} + +// trimRefName is the import half of the suffix: the id, with the informative +// name dropped unread (§9). +func trimRefName(ref string) string { + id, _ := splitRefName(ref) + return id +} + +// refNameLabel normalizes a display name into the suffix grammar +// (refNameNormalize below), bounded by maxRefNameLen. An empty answer means +// no suffix. The grammar admits no `#`, which is the writer's half of the +// split guarantee: a raw display name here would break the split from both +// ends. +func refNameLabel(name string) string { + label := refNameNormalize(name) + if runes := []rune(label); len(runes) > maxRefNameLen { + label = strings.TrimRight(string(runes[:maxRefNameLen]), "_") + } + return label +} + +// refNameNormalize turns a display name into the `#name` suffix grammar — +// letters of any script, digits, `_`, combining marks — or "" when nothing +// is left to name. +// +// This is the identifier normalization that used to mint KEY labels +// (label.go), surviving here for its one remaining surface. Key spellings +// are raw names now and need no normalization at all; the ref suffix still +// does, because its grammar is what makes the `#` split safe — a raw +// display name may contain `#`, and the suffix must not. The rules are +// unchanged from the key-label era on purpose: the suffix is informative +// and trimmed unread, so nothing depends on its exact shape, and keeping +// the bytes stable keeps every already-written reference identical on its +// next export. +// +// Three decisions worth keeping stated, because each has a plausible +// alternative: +// +// - **NFC, lowercase, separators collapse to `_`.** Two visually +// identical names must not suffix differently between exports. +// - **Combining marks are kept with their letter.** In Devanagari, Thai, +// Bengali, Tamil, Khmer and Myanmar the vowels ARE marks; dropping them +// does not shorten a word, it changes it — मिल/मूल/मल/मैल would all +// become मल. +// - **A leading `_` run is content, not a gap** — integrations namespace +// themselves `__amemory_…` in their names — while interior runs +// collapse and a trailing run trims; and a result that starts with a +// digit or is a filter-grammar keyword takes a leading `_`, the escape +// the suffix inherited from the key grammar and keeps for byte +// stability. +func refNameNormalize(s string) string { + if s == "" { + return "" + } + lead := 0 + for _, r := range s { + if r != '_' { + break + } + lead++ + } + var b strings.Builder + gap := false // a separator run is pending, emitted only before the next letter + for _, r := range norm.NFC.String(s) { + switch { + case unicode.IsLetter(r) || unicode.IsDigit(r): + if gap && b.Len() > 0 { + b.WriteRune('_') + } + gap = false + b.WriteRune(unicode.ToLower(r)) + case unicode.Is(unicode.Mn, r) || unicode.Is(unicode.Mc, r): + // a mark cannot start a token, and one arriving with a pending + // separator is malformed input, not a word + if b.Len() > 0 && !gap { + b.WriteRune(r) + } + default: + gap = true // `_` included: runs collapse and edges trim + } + } + label := strings.Repeat("_", lead) + b.String() + if label == "" || strings.Trim(label, "_") == "" { + return "" + } + if !filterstring.IsBareKey(label) { + label = "_" + label + } + if !filterstring.IsBareKey(label) { + // unreachable by construction — every rune is already an identPart, + // so the only faults are a leading digit and a keyword, both cured + // above. It is a guard rather than a path: IsBareKey is another + // package's rule and may grow one, and the honest degradation is no + // suffix at all. + return "" + } + return label +} + +// isAccountIdentity reports whether s is a member's account identity — the +// base58 strkey form with its version byte and crc16 checksum intact +// (any-sync util/crypto). The checksum is what makes this a CLASSIFIER +// rather than a heuristic: no CID, bson id, or `_`-prefixed derived id can +// decode as one, so a bare identity in a reference slot is unambiguous. +func isAccountIdentity(s string) bool { + if len(s) < 40 || len(s) > 64 { + return false // cheap gate: real identities are 48 characters + } + _, err := crypto.DecodeAccountAddress(s) + return err == nil +} + +// foldParticipantRef is the export half of the participant fold (§9): +// `_participant__` becomes the bare identity. It folds +// ONLY what unfoldParticipantRef provably rebuilds — the space embedded in +// the id must be this run's own SpaceId (a cross-space participant ref would +// otherwise silently re-home on import), the identity must classify as one, +// and the composite must round-trip through domain.NewParticipantId +// byte-identically. With no SpaceId the fold is off in both directions: +// folding on export without the paired import being able to rebuild would +// land a bare identity where a composite belongs. +func (o Options) foldParticipantRef(id string) string { + if o.SpaceId == "" || !strings.HasPrefix(id, domain.ParticipantPrefix) { + return id + } + spaceId, identity, err := domain.ParseParticipantId(id) + if err != nil || spaceId != o.SpaceId || !isAccountIdentity(identity) { + return id + } + if domain.NewParticipantId(o.SpaceId, identity) != id { + return id + } + return identity +} + +// FoldParticipantId is the exported form of the participant fold, for +// callers that must agree with the envelope id Marshal writes WITHOUT +// marshalling: the exporter's path plan names a document file by its +// envelope id (EXPORTER_DESIGN.md §1.3), and a participant document's +// envelope id is its folded bare identity. Same gates as the internal fold +// — no spaceId, a foreign space, a non-identity tail, or a composite that +// does not round-trip all decline and return id unchanged — which is +// exactly when Marshal keeps the composite as the envelope id, so the plan +// and the envelope cannot disagree. +func FoldParticipantId(spaceId, id string) string { + return Options{SpaceId: spaceId}.foldParticipantRef(id) +} + +// unfoldParticipantRef is the import half: a bare identity in an object +// reference slot rebuilds this space's participant id. Gated on the exact +// classifier the fold used, so unfold(fold(x)) == x and fold(unfold(y)) == y +// for every id either side touches. +func (o Options) unfoldParticipantRef(id string) string { + if o.SpaceId == "" || !isAccountIdentity(id) { + return id + } + return domain.NewParticipantId(o.SpaceId, id) +} + +// objectRef renders one object reference for a document slot (§9): the +// participant fold first, then the informative `#name` suffix when the +// shape asks for it (Options.RefNames) and a resolver names the target. The +// resolver is asked about the STORED id — the composite participant id, not +// the folded identity — because that is the id the space indexes. With no +// resolver, or no name, the reference is written bare — never with a +// partial or invented suffix. +func (e *exporter) objectRef(id string) string { + out := e.opts.foldParticipantRef(id) + if !e.opts.RefNames || e.opts.ResolveObjectNames == nil || id == "" { + return out + } + if !suffixableRef(id) { + return out + } + name, ok := e.opts.ResolveObjectNames.ObjectName(id) + if !ok { + return out + } + if label := refNameLabel(name); label != "" { + return out + refNameSep + label + } + return out +} + +// suffixableRef reports the ids a name suffix belongs on. A date id and the +// missing-object sentinel already say everything they mean, and a dynamic +// filter placeholder (§6.2) is not an object id at all — a suffix on any of +// them would be decoration on a value some other layer must read verbatim. +// +// An id that already carries a `#` is excluded for a different reason: the +// suffix is only written where it is REVERSIBLE. No id this format writes +// contains one, but a snapshot is untrusted (§11) and may hold anything, and +// `x#y` + `#name` reads back as `x` — a different id from the one exported. +// Worse where the id half is empty: `#name` refuses to split at index 0 +// (splitRefName), so import returns it whole and the next export appends +// again, one name per generation without bound. Writing such an id bare +// costs a caption on a reference that could not resolve anyway, and buys +// back §11 guarantee 2. +func suffixableRef(id string) bool { + return !strings.HasPrefix(id, dateIdPrefix) && + id != missingObjectId && + !isFilterTemplate(id) && + !strings.Contains(id, refNameSep) +} + +// singularObjectRef renders a SINGULAR reference slot — a block's +// `object_id` (link, bookmark, file kinds, dataview) — under the +// missing-reference rule (§9): a target the space does not hold is written +// as the `_missing_object` sentinel, because omission cannot express "no +// target" here — only deleting the block could, and that would lose the +// fact that a link existed. A target the store DOES hold, the store cannot +// speak for (missingFromSpace's gates), or that already IS the sentinel +// passes to the ordinary objectRef untouched. +// +// The rewrite warns, naming the id: unlike the sentinel — which says +// nothing beyond "gone" — the id is real information, and the warning is +// its last appearance anywhere. After one round trip the slot is a +// fixpoint: the sentinel is kept as-is, so re-exports are byte-stable. +func (e *exporter) singularObjectRef(path, slot, id string) string { + if missingFromSpace(e.opts, id) { + e.warn(path, "%s %q names no object in this space and is written as %q — "+ + "the slot cannot say \"no target\" without deleting the block, and the sentinel "+ + "keeps the fact that a reference existed", slot, id, missingObjectId) + return e.objectRef(missingObjectId) + } + return e.objectRef(id) +} + +// droppedMissingListEntry is the LIST half of the missing-reference rule +// (§9): an objects/files property value entry, or an `object_types` entry, +// that the space does not hold is dropped — a list expresses absence by +// being shorter. The predicate is the exported DroppedMissingObjectRef, so +// the comparator applies exactly what export applied. +// +// Only a REAL id warns. A stored `_missing_object` sentinel drops silently: +// it carries nothing — which object it was is already gone — and the corpus +// holds ~990 of them in property values alone, which would triple a warning +// channel that was just cut down to what is worth reading (§12). +func (e *exporter) droppedMissingListEntry(path, id string) bool { + if !DroppedMissingObjectRef(e.opts, id) { + return false + } + if id != missingObjectId { + e.warn(path, "%q names no object in this space and is dropped — "+ + "a list expresses absence by being shorter", id) + } + return true +} + +// exportMarks applies the missing-reference rule to inline markup (§8, §9): +// a `` whose target the space does not hold is +// rewritten to the `_missing_object` sentinel — a mention is a singular +// slot; dropping the mark would lose the fact that a mention existed while +// its text stayed. Copy-on-write: the snapshot's own marks are caller-owned +// state and are never mutated, and the common case — nothing missing — +// returns the input slice untouched. Object-link marks (`[label](anytype://…)`) +// keep their ids verbatim, as §9 states for them. +func (e *exporter) exportMarks(path string, marks []*model.BlockContentTextMark) []*model.BlockContentTextMark { + out := marks + copied := false + for i, m := range marks { + if m == nil || m.Type != model.BlockContentTextMark_Mention || !missingFromSpace(e.opts, m.Param) { + continue + } + e.warn(path, "mention target %q names no object in this space and is written as %q — "+ + "the mention's own text stays; only its address is gone", m.Param, missingObjectId) + if !copied { + out = append([]*model.BlockContentTextMark(nil), marks...) + copied = true + } + clone := *m + clone.Param = missingObjectId + out[i] = &clone + } + return out +} + +// dateIdPrefix marks a virtual date object id (pkg/lib/localstore/addr). +const dateIdPrefix = "_date_" + +// missingObjectId is the dangling-reference sentinel stored details carry +// (pkg/lib/localstore/addr.MissingObject). +const missingObjectId = "_missing_object" + +// MissingObjectId is missingObjectId for the round-trip comparator, which +// lives in its own package and must apply the very sentinel export applies — +// the two cannot be allowed to spell it differently. +const MissingObjectId = missingObjectId + +// objectRef reads one object reference back (§9): the informative suffix is +// trimmed at the first `#`, unread, and a bare identity unfolds into this +// space's participant id. Everything else passes verbatim, exactly as +// before the suffix existed — which is what keeps a bare id and a suffixed +// id importing identically. +func (imp *importer) objectRef(ref string) string { + id := trimRefName(ref) + // A bare account identity in a reference slot is the folded half of a + // participant id (§9), and only a space can rebuild it. A reader that + // names none would store the identity where the composite belongs — a + // reference to an object that does not exist, in silence. The classifier + // is exact (a strkey checksum), so the reader KNOWS this has happened + // and says so, once, in build. It may not refuse: Validate never sees + // Options, so refusing here would put the two surfaces into + // disagreement over one document (§12 I2). + if imp.opts.SpaceId == "" && isAccountIdentity(id) { + imp.foldedUnrebuilt = true + return id + } + return imp.opts.unfoldParticipantRef(id) +} diff --git a/pkg/lib/anyblockjson/refs_test.go b/pkg/lib/anyblockjson/refs_test.go new file mode 100644 index 0000000000..ab8fa9578c --- /dev/null +++ b/pkg/lib/anyblockjson/refs_test.go @@ -0,0 +1,541 @@ +package anyblockjson + +// refs_test.go — the informative `#name` reference suffix (§9): written on +// export behind RefNames, trimmed unread on import, and never required. + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// testObjectNames answers from a fixed table — the ObjectNameResolver shape. +type testObjectNames map[string]string + +func (m testObjectNames) ObjectName(id string) (string, bool) { + n, ok := m[id] + return n, ok +} + +// refSnapshot exercises every slot the suffix rides: an object-format +// property, collection items, link/file/bookmark blocks, and a dataview with +// an object-valued filter, a custom order, and a kanban object order. +func refSnapshot() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + { + Id: "bafyreirefroot", + ChildrenIds: []string{"lnk", "fil", "bmk", "dv1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }, + {Id: "lnk", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: "bafyreilinked", + }}}, + {Id: "fil", Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + Type: model.BlockContentFile_Image, TargetObjectId: "bafyreipicture", + }}}, + {Id: "bmk", Content: &model.BlockContentOfBookmark{Bookmark: &model.BlockContentBookmark{ + Url: "https://anytype.io", TargetObjectId: "bafyreibookmarked", + }}}, + {Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + TargetObjectId: "bafyreitargeted", + Views: []*model.BlockContentDataviewView{{ + Id: "view1", + Name: "All", + Filters: []*model.BlockContentDataviewFilter{{ + Id: "f1", + RelationKey: "assignee", + Condition: model.BlockContentDataviewFilter_In, + Value: strList("bafyreifiltered"), + }}, + Sorts: []*model.BlockContentDataviewSort{{ + Id: "s1", + RelationKey: "assignee", + CustomOrder: []*types.Value{str("bafyreiordered")}, + }}, + }}, + ObjectOrders: []*model.BlockContentDataviewObjectOrder{{ + ViewId: "view1", + ObjectIds: []string{"bafyreikanban"}, + }}, + }}}, + }, + Details: fields(map[string]*types.Value{ + "id": str("bafyreirefroot"), + "name": str("Ref host"), + "related": strList("bafyreitopic"), + "assignee": strList("bafyreiassigned"), + }), + Collections: fields(map[string]*types.Value{ + storeKeyItems: strList("bafyreicollected"), + }), + } +} + +// refNames names every referenced object in refSnapshot. +var refNames = testObjectNames{ + "bafyreitopic": "Local-first UX", + "bafyreiassigned": "Roma Kha", + "bafyreicollected": "Collected Page", + "bafyreilinked": "Linked Page", + "bafyreipicture": "Cat Photo", + "bafyreibookmarked": "Bookmarked Page", + "bafyreitargeted": "Task Tracker", + "bafyreifiltered": "Filter Target", + "bafyreiordered": "Order Target", + "bafyreikanban": "Kanban Card", +} + +func refOptions() Options { + o := testOptions() + o.ResolveFormat = func(key domain.RelationKey) (model.RelationFormat, bool) { + if key == "related" { + return model.RelationFormat_object, true + } + return testFormatResolver(key) + } + return o +} + +// Every slot §9 lists gains the suffix when the shape asks for it, and the +// output stays a document this package's own Validate accepts (I1). +// +// How this can fail: unhook exporter.objectRef from any one slot and that +// slot's assertion finds the bare id; break the normalizer and the expected +// spellings differ; emit a suffix Validate rejects and the I1 check fails. +// Nothing here re-implements the suffix — the expectations are literal +// strings. +func TestRefNames_SuffixOnEverySlot(t *testing.T) { + // given + opts := refOptions() + opts.RefNames = true + opts.ResolveObjectNames = refNames + + // when + data, err := Marshal(model.SmartBlockType_Page, refSnapshot(), opts) + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (§11 I1)") + doc := string(data) + + // then — one literal expectation per slot + for slot, want := range map[string]string{ + "property value (custom objects format)": `"bafyreitopic#local_first_ux"`, + "property value (bundled objects format)": `"bafyreiassigned#roma_kha"`, + "items": `"bafyreicollected#collected_page"`, + "link block": `"object_id": "bafyreilinked#linked_page"`, + "file block": `"object_id": "bafyreipicture#cat_photo"`, + "bookmark block": `"object_id": "bafyreibookmarked#bookmarked_page"`, + "dataview target": `"object_id": "bafyreitargeted#task_tracker"`, + "filter value": `"bafyreifiltered#filter_target"`, + "sort custom order": `"bafyreiordered#order_target"`, + "object_orders object ids": `"bafyreikanban#kanban_card"`, + } { + assert.Contains(t, doc, want, slot) + } +} + +// The suffix is opt-in per shape: RefNames off (the default, the +// export/backup shape) writes every reference bare even with a resolver +// wired, and RefNames on with no resolver writes them bare too — never a +// partial or invented suffix. +// +// How this can fail: make the suffix unconditional and the first case finds +// a `#`; invent a suffix from the id itself and the second does. +func TestRefNames_OffByDefaultAndBareWithoutResolver(t *testing.T) { + t.Run("resolver wired, flag off", func(t *testing.T) { + // given + opts := refOptions() + opts.ResolveObjectNames = refNames + + // when + data, err := Marshal(model.SmartBlockType_Page, refSnapshot(), opts) + + // then + require.NoError(t, err) + assert.NotContains(t, string(data), "#", + "the export/backup shape writes no suffix: minimal, and stable under renames") + }) + + t.Run("flag on, no resolver", func(t *testing.T) { + // given + opts := refOptions() + opts.RefNames = true + + // when + data, err := Marshal(model.SmartBlockType_Page, refSnapshot(), opts) + + // then + require.NoError(t, err) + assert.NotContains(t, string(data), "#", "with no resolver, the bare id — nothing invented") + }) +} + +// suffixedRefDoc spells a suffixed reference in every §9 slot; bareRefDoc is +// the same document with every suffix removed. +const suffixedRefDoc = `{ + "version": 2, + "id": "bafyreirefroot", + "properties": { + "assignee": ["bafyreiassigned#roma_kha"], + "name": "Ref host" + }, + "blocks": [ + {"type": "link", "object_id": "bafyreilinked#linked_page"}, + {"type": "image", "object_id": "bafyreipicture#cat_photo"}, + {"type": "bookmark", "url": "https://anytype.io", "object_id": "bafyreibookmarked#bookmarked_page"}, + {"type": "dataview", "object_id": "bafyreitargeted#task_tracker", "views": [ + {"id": "view1", "name": "All", + "filters": [{"property": "assignee", "condition": "in", "value": ["bafyreifiltered#filter_target"]}], + "sorts": [{"property": "assignee", "custom_order": ["bafyreiordered#order_target"]}], + "object_orders": [{"object_ids": ["bafyreikanban#kanban_card"]}]} + ]} + ], + "items": ["bafyreicollected#collected_page"] +}` + +// Import trims the suffix at the first `#` in every slot, and a bare +// document imports IDENTICALLY — the §11 I2 surface: a model writing a new +// reference has no name to add, and must not need one. +// +// How this can fail: skip the trim at any slot and that slot's snapshot +// value keeps the `#name`; trim at the LAST `#` instead of the first and the +// double-# case keeps half a suffix; make the suffix load-bearing and the +// bare document stops importing equal. +func TestRefs_ImportTrimsAndBareImportsIdentically(t *testing.T) { + bare := strings.NewReplacer( + "#roma_kha", "", "#linked_page", "", "#cat_photo", "", + "#bookmarked_page", "", "#task_tracker", "", "#filter_target", "", + "#order_target", "", "#kanban_card", "", "#collected_page", "", + ).Replace(suffixedRefDoc) + require.NotContains(t, bare, "#", "the bare twin really is bare") + + // when — both forms validate and both import + require.NoError(t, Validate([]byte(suffixedRefDoc)), "a suffixed reference is valid") + require.NoError(t, Validate([]byte(bare)), "a bare reference is valid") + importOpts := func() Options { + o := testOptions() + o.GenerateId = seqIds("gen") // deterministic, so the two snapshots can be compared whole + return o + } + sbType1, suffixed, err := Unmarshal([]byte(suffixedRefDoc), importOpts()) + require.NoError(t, err) + sbType2, bareSnap, err := Unmarshal([]byte(bare), importOpts()) + require.NoError(t, err) + + // then — the suffix reached no snapshot slot + blob, err := json.Marshal(suffixed) + require.NoError(t, err) + assert.NotContains(t, string(blob), "#", "no suffix survives into the snapshot") + for slot, want := range map[string]string{ + "assignee": "bafyreiassigned", + } { + assert.Equal(t, []string{want}, + valueStringList(suffixed.GetDetails().GetFields()[slot]), slot) + } + + // and the two forms import identically + assert.Equal(t, sbType1, sbType2) + assert.Equal(t, bareSnap, suffixed, "a bare id and a suffixed id import identically (§11 I2)") +} + +// A `#` that does not follow an id is left alone: the trim never invents an +// empty reference out of a malformed one, and an option NAME containing `#` +// (a select value like "C#") is not an object reference and keeps its +// characters. +// +// How this can fail: trim unconditionally at index 0 and the leading-# +// value comes back empty; run the trim over select values and "C#" loses +// its sharp. +func TestRefs_TrimNeverInventsEmptinessAndSkipsOptionNames(t *testing.T) { + t.Run("a leading-# value stays whole", func(t *testing.T) { + // given + doc := `{"version": 2, "properties": {"assignee": ["#notanid"]}}` + + // when + _, snap, err := Unmarshal([]byte(doc), testOptions()) + + // then + require.NoError(t, err) + assert.Equal(t, []string{"#notanid"}, + valueStringList(snap.GetDetails().GetFields()["assignee"])) + }) + + t.Run("a double-# value trims at the FIRST separator", func(t *testing.T) { + // given a reference whose informative half itself spells a # + doc := `{"version": 2, "properties": {"assignee": ["bafyreiassigned#a#b"]}}` + + // when + _, snap, err := Unmarshal([]byte(doc), testOptions()) + + // then — LastIndex would hand back bafyreiassigned#a, an id that + // addresses nothing + require.NoError(t, err) + assert.Equal(t, []string{"bafyreiassigned"}, + valueStringList(snap.GetDetails().GetFields()["assignee"])) + }) + + t.Run("an option name keeps its #", func(t *testing.T) { + // given customStatus resolves to the select format (testOptions) + doc := `{"version": 2, "properties": {"customStatus": ["C#"]}}` + + // when + _, snap, err := Unmarshal([]byte(doc), testOptions()) + + // then + require.NoError(t, err) + assert.Equal(t, []string{"C#"}, + valueStringList(snap.GetDetails().GetFields()["customStatus"]), + "a select value is a name, not a reference — no trim (§9)") + }) +} + +// The round trip stays byte-stable given the same resolver: import trims the +// suffix, and the second export re-derives it from the same names. +// +// How this can fail: any slot that trims without re-deriving (or derives +// without trimming) shifts bytes between generations. +func TestRefs_RoundTripByteStableWithResolver(t *testing.T) { + // given + opts := refOptions() + opts.RefNames = true + opts.ResolveObjectNames = refNames + + // when + first, err := Marshal(model.SmartBlockType_Page, refSnapshot(), opts) + require.NoError(t, err) + sbType, imported, err := Unmarshal(first, opts) + require.NoError(t, err) + second, err := Marshal(sbType, imported, opts) + require.NoError(t, err) + + // then + assert.Equal(t, string(first), string(second), + "Export ∘ Import is byte-stable with the same resolver (§11)") +} + +// The ids that already say what they mean take no suffix: a date reference, +// the missing-object sentinel, a dynamic filter placeholder. +// +// How this can fail: drop the suffixableRef guard and each of the three +// gains a suffix the moment a resolver claims to name it. +func TestRefNames_SelfDescribingIdsTakeNoSuffix(t *testing.T) { + // given a resolver that (wrongly) has names for all three + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: "bafyreirefroot", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(map[string]*types.Value{ + "id": str("bafyreirefroot"), + "related": strList("_date_2026-08-17", "_missing_object", "_filter_template_2_"), + "assignee": strList("bafyreiassigned"), + }), + } + opts := refOptions() + opts.RefNames = true + opts.ResolveObjectNames = testObjectNames{ + "_date_2026-08-17": "17 Aug 2026", + "_missing_object": "Missing", + "_filter_template_2_": "Current user", + "bafyreiassigned": "Roma Kha", + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + doc := string(data) + + // then + assert.Contains(t, doc, `"_date_2026-08-17"`) + assert.Contains(t, doc, `"_missing_object"`) + assert.Contains(t, doc, `"_filter_template_2_"`) + assert.Contains(t, doc, `"bafyreiassigned#roma_kha"`, "the control: an ordinary ref still gains one") +} + +// The name half of the split guarantee: whatever a display name holds, the +// suffix that reaches the document contains no `#` and survives in the +// identifier grammar. +// +// How this can fail: write the raw display name after the `#` and the +// adversarial cases split wrong on read; drop the truncation and the long +// name blows the bound. +func TestRefNameLabel(t *testing.T) { + for name, tc := range map[string]struct{ in, want string }{ + "spaces snake": {"Local-first UX", "local_first_ux"}, + "plain name": {"Roma Kha", "roma_kha"}, + "hash inside": {"a#b", "a_b"}, + "only a hash": {"#", ""}, + "whitespace only": {" \t ", ""}, + "non-latin kept": {"Тоггл", "тоггл"}, + "empty": {"", ""}, + } { + t.Run(name, func(t *testing.T) { + got := refNameLabel(tc.in) + assert.Equal(t, tc.want, got) + assert.NotContains(t, got, "#", "the grammar admits no #") + }) + } + + t.Run("a long name truncates at the bound", func(t *testing.T) { + long := strings.Repeat("word ", 40) // normalizes to 199 chars of word_word_… + got := refNameLabel(long) + assert.LessOrEqual(t, len([]rune(got)), maxRefNameLen) + assert.NotEmpty(t, got) + assert.False(t, strings.HasSuffix(got, "_"), "no dangling separator after the cut") + }) +} + +// An id that already carries a `#` takes no suffix, however confidently a +// resolver names it: `x#y` + `#name` reads back as `x`, so the caption would +// be paid for with the id itself (§9). +// +// How this can fail: drop the refNameSep arm of suffixableRef and both ids +// below gain a caption the reader cannot undo. +func TestRefNames_AnIdCarryingAHashTakesNoSuffix(t *testing.T) { + // given a resolver that has a name for both hostile ids + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: "bafyreirefroot", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(map[string]*types.Value{ + "id": str("bafyreirefroot"), + "assignee": strList("bafyreiassigned#stale_name"), + "related": strList("#notanid"), + }), + } + opts := refOptions() + opts.RefNames = true + opts.ResolveObjectNames = testObjectNames{ + "bafyreiassigned#stale_name": "Roma Kha", + "#notanid": "Roma Kha", + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (§11 I1)") + doc := string(data) + + // then — both stand exactly as stored, with nothing appended + assert.Contains(t, doc, `"bafyreiassigned#stale_name"`) + assert.Contains(t, doc, `"#notanid"`) + assert.NotContains(t, doc, "roma_kha", "no caption on an unsplittable id") +} + +// A reference with no id half does not grow a name every generation (§11 +// guarantee 2). splitRefName refuses to split at index 0, so `#name` imports +// whole; if export were still willing to caption it, each round trip would +// append another and the document would diverge without bound. +// +// How this can fail: let suffixableRef admit a `#`-bearing id and generation +// 2 is `#some_name#roma_kha`, generation 3 one name longer again. +func TestRefs_ALeadingHashReferenceDoesNotGrow(t *testing.T) { + // given the mistake a writer makes copying the readable half of id#name + doc := []byte(`{"version": 2, "kind": "page", "id": "bafyreiroot", + "properties": {"assignee": ["#some_name"]}}`) + require.NoError(t, Validate(doc)) + + opts := refOptions() + opts.RefNames = true + opts.ResolveObjectNames = testObjectNames{"#some_name": "Roma Kha"} + + // when — three generations through the codec + var gens []string + cur := doc + for i := 0; i < 3; i++ { + sbType, snap, err := Unmarshal(cur, opts) + require.NoError(t, err) + assert.Equal(t, []string{"#some_name"}, + valueStringList(snap.GetDetails().GetFields()["assignee"]), + "generation %d reads back the value it was given", i+1) + cur, err = Marshal(sbType, snap, opts) + require.NoError(t, err) + require.NoError(t, Validate(cur)) + gens = append(gens, string(cur)) + } + + // then + assert.Equal(t, gens[0], gens[1], "Export ∘ Import is byte-stable (§11)") + assert.Equal(t, gens[1], gens[2]) +} + +// The format's one reference normalization (§11 N(S)): an id with a `#` +// INSIDE it loses its tail on read, because the split cannot tell that `#` +// from the one the suffix uses. Export no longer captions such an id, so the +// loss happens once and the value is a fixpoint from the second generation +// on — it does not shrink again, and it does not grow. +// +// No id this format writes contains a `#` and none was found in 81,696 +// production documents across two corpora; this test exists to state what +// happens if one ever does, rather than to leave it to be discovered. +// +// How this can fail: caption a `#`-bearing id again and generation 2 differs +// from generation 3 as the tail is eaten one segment at a time. +func TestRefs_AHashInsideAnIdIsNormalizedOnce(t *testing.T) { + // given + opts := refOptions() + opts.RefNames = true + opts.ResolveObjectNames = testObjectNames{ + "bafyreiassigned#weird": "Roma Kha", + "bafyreiassigned": "Roma Kha", + } + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: "bafyreirefroot", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(map[string]*types.Value{ + "id": str("bafyreirefroot"), + "assignee": strList("bafyreiassigned#weird"), + }), + } + + // when + gen1, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + sbType, back, err := Unmarshal(gen1, opts) + require.NoError(t, err) + gen2, err := Marshal(sbType, back, opts) + require.NoError(t, err) + _, back2, err := Unmarshal(gen2, opts) + require.NoError(t, err) + gen3, err := Marshal(sbType, back2, opts) + require.NoError(t, err) + + // then — the tail goes once, and only once + assert.Contains(t, string(gen1), `"bafyreiassigned#weird"`, "export writes the stored id whole") + assert.Equal(t, []string{"bafyreiassigned"}, + valueStringList(back.GetDetails().GetFields()["assignee"]), + "the reader cannot tell this # from the suffix's, so the tail goes (§11 N(S))") + assert.Equal(t, string(gen2), string(gen3), "and the normalized value is a fixpoint") +} + +// A reference with no id half is a warning on the way in, not a silent +// dangling value: the reader will not repair it, so the writer is told (§9). +// Warning-grade, because a document that carries one is still readable, and +// export must be able to pass through whatever a snapshot holds. +// +// How this can fail: drop the objects arm of wrongShapeForFormat and the +// document validates clean while the value addresses nothing. +func TestValidate_AReferenceWithNoIdHalfWarns(t *testing.T) { + // given assignee is a bundled objects property — no store needed to know it + doc := []byte(`{"version": 2, "properties": {"assignee": ["#roma_kha"]}}`) + + // when + var warned []Issue + err := ValidateWarn(doc, func(i Issue) { warned = append(warned, i) }) + + // then + require.NoError(t, err, "a document that carries one is still readable") + require.Len(t, warned, 1) + assert.Equal(t, "/properties/assignee", warned[0].Path) + assert.Contains(t, warned[0].Message, "no id before its") +} diff --git a/pkg/lib/anyblockjson/relationformat.go b/pkg/lib/anyblockjson/relationformat.go new file mode 100644 index 0000000000..15de20aa00 --- /dev/null +++ b/pkg/lib/anyblockjson/relationformat.go @@ -0,0 +1,544 @@ +package anyblockjson + +// relationformat.go implements §2d: the `property_settings` group of a +// `kind: "property"` document — one propertyDefinition (§2e), whose three +// travelling members are `format`, `include_time`, `object_types`. They are +// grouped rather than sitting at the document root, because the dictionary +// entry and a type's property-definition entry are groups holding the same +// shape and two patterns for one idea is §15 #14 one level up. The group +// (and the kinds) are named off "relation": the product calls these things +// properties, and the format already did in every neighbouring name. +// +// A relation object IS a property definition, and until this lift it was the +// one document that could not state its own format in the format's own +// vocabulary: `properties` carried `relation_format: 100` — a raw enum +// number — while a `type_properties` entry three sections up spelled the +// same fact `format: "objects"`. One concept, two spellings, in one format +// (§15 #14). Worse, the raw spelling was a live trap: in a 198-run +// small-model eval, 9 of 9 attempts wrote `properties: {"format": "number"}`, +// which VALIDATED — inside `properties` every key is a property spelling, so +// that line means "a custom property named format" — and imported as exactly +// that, leaving the relation with no relationFormat at all: longtext forever, +// silently. The container was the problem, not the word. +// +// The precedent is §2b: stored keys lifted into typed envelope fields, with +// the flat spellings refused where they used to sit — including the refusal, +// because a format with two legal spellings for one thing, one of which a +// small model has seen far more of in training data, defeats the whole +// point. Like the nine §2b keys, all three relations are `hidden: true`, so +// no property row loses the presence §3 makes meaningful — but unlike §2b +// the envelope fields mirror stored presence EXACTLY (false, `[]` and null +// all travel): these are the definition of the property, not decoration, and +// the §15 #14 verdict was to fix the SPELLING and leave the emptiness +// collapse to its own change. Measured over 38,061 production documents +// (10,617 of them relation documents): every one carries `relationFormat`, +// so requiring `format` refuses nothing real; `include_time` is true only on +// dates (543 of 9,035 present) and null on 80; `object_types` is non-empty +// only on objects/files (1,089 + 167 of 10,159 present); and none of the +// three keys occurs on any other kind, so the unconditional refusal costs +// nothing. + +import ( + "fmt" + "math" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The three stored detail keys the §2d envelope fields carry. Named off the +// bundle so a rename there is a compile error here rather than a silent +// un-lift (the §2b rule). +var ( + detailKeyRelationFormat = bundle.RelationKeyRelationFormat.String() + detailKeyRelationFormatIncludeTime = bundle.RelationKeyRelationFormatIncludeTime.String() + detailKeyRelationFormatObjectTypes = bundle.RelationKeyRelationFormatObjectTypes.String() +) + +// propertySettingsLiftedDetailKeys is the §2d lift list, and like liftedDetailKeys +// (§2b) it is the single source of truth for both directions: export writes +// these keys nowhere but the envelope, and import refuses them in +// `properties` (deniedPropertyKey reads this same set). The refusal is +// unconditional across kinds — there is no second way to write a relation's +// format — and export honours that everywhere: on a kind with no §2d fields +// to lift into, a present key is dropped with a warning rather than written +// as a property (never observed: 0 of 27,444 non-relation documents carry +// any of the three). +func propertySettingsLiftedDetailKeys() map[string]bool { + return map[string]bool{ + detailKeyRelationFormat: true, + detailKeyRelationFormatIncludeTime: true, + detailKeyRelationFormatObjectTypes: true, + } +} + +// propertySettingsLiftedKeyRepair names the property_settings member a refused flat +// spelling belongs in — liftedKeyRepair's rule (§2b): the refusal is worth +// twice as much said as a repair, because unlike an internal key there IS +// something to write instead. +func propertySettingsLiftedKeyRepair(key string) string { + switch key { + case detailKeyRelationFormat: + return `"format": ""` + case detailKeyRelationFormatIncludeTime: + return `"include_time": true|false` + case detailKeyRelationFormatObjectTypes: + return `"object_types": ["", …]` + } + return "" +} + +// TypeResolver translates between type OBJECT ids — what the store keeps in +// `relationFormatObjectTypes` (objectcreator.fillRelationFormatObjectTypes +// rewrites bundled urls to derived ids at creation) — and stored type KEYS, +// which are what every type-key slot in this format spells (§2a, §2d). It is +// an optional capability of Options.ResolveProperties, discovered by type +// assertion, rather than a fourth resolver field: the resolver that already +// answers PropertyById is the one with the space listing this mapping falls +// out of (storeresolver fills keyById from the same bounded query), and a +// caller without it keeps a well-defined degradation — entries pass through +// verbatim in both directions, each its own address (§3), so an offline +// round trip is byte-exact and only a resolver-wired one translates. +// +// Both directions exist so the translation is an inverse rather than a +// one-way normalization: export turns the stored ids into keys, import turns +// the keys back into this space's ids — the same policy applyTypeProperties +// applies to property definitions via PropertyId — and a key the reader's +// space does not serve stays a key, for the wiring to reconcile (§2a). +type TypeResolver interface { + TypeKeyById(id string) (string, bool) + TypeIdByKey(key string) (string, bool) +} + +// +// ---- export ---- +// + +// isPropertyDoc reports whether this export carries the §2d envelope fields. +// It is the snapshot-side half of isPropertyKind and MUST list the same +// kinds, because the schema now requires `format` on all three: an export +// that lifted for fewer than it validates for would emit a document its own +// Validate rejects (§11 I1) — which is exactly what happened when only the +// document side was widened, on `bundled_property`. +// +// `sub_object` is NOT one of them. It is deprecated and out of the format's +// support surface by decision, not by measurement — 0 of 38,061 corpus +// documents carry it either way, so nothing observable turns on it; what +// turns on it is that a deprecated kind must not acquire a new obligation +// in a format about to freeze. +func (e *exporter) isPropertyDoc() bool { + return isPropertySmartBlock(e.sbType) +} + +// isPropertySmartBlock is the SNAPSHOT-side statement of which kinds are +// property documents, and isPropertyKind is the DOCUMENT-side one. All three lists — +// these two and the schema's `if` — must name the same kinds: the schema +// requires `format` on each of them, so a half that lifts for fewer than the +// schema validates for breaks §11 I1 in one direction and drops the +// definition in the other. Both breaks happened when only one list was +// widened. +func isPropertySmartBlock(sbType model.SmartBlockType) bool { + return sbType == model.SmartBlockType_STRelation || + sbType == model.SmartBlockType_BundledRelation +} + +// buildPropertySettings writes the `property_settings` group — one +// propertyDefinition, the three §2d members that travel today — or, on a +// kind that has no such group, reports any stored value the lift leaves +// nowhere to go. Member presence mirrors stored-key presence exactly, value +// included (false, `[]`, null): the §4 omit-empty canon stops at these three +// because they are the property's definition, and §15 #14 scoped the lift to +// the SPELLING, deliberately leaving present-and-empty alone so the +// snapshot round-trips unchanged and the comparator needs no new rule. +// The three sit in a group rather than at the root: the dictionary entry +// and the type's property-definition entry are groups holding the same +// shape, and two patterns for one idea is the §15 #14 disease again. +func (e *exporter) buildPropertySettings(doc *omap) error { + if !e.isPropertyDoc() { + for _, key := range []string{detailKeyRelationFormat, + detailKeyRelationFormatIncludeTime, detailKeyRelationFormatObjectTypes} { + if e.detail(key) != nil { + e.warn("/properties", "%q describes a property definition and this is not a property document; "+ + "the value is dropped — only a property document has a property_settings member for it, and `properties` refuses the key", key) + } + } + return nil + } + + name, err := e.relationFormatName() + if err != nil { + return err + } + group := &omap{} + group.set("format", name) + + if v := e.detail(detailKeyRelationFormatIncludeTime); v != nil { + switch k := v.GetKind().(type) { + case *types.Value_BoolValue: + group.set("include_time", k.BoolValue) + case *types.Value_NullValue: + // a stored null is a value — the key was set (§3) — and 80 + // production relations hold exactly this, so dropping it would + // change the snapshot on the way round + group.set("include_time", nil) + default: + e.warn("/property_settings/include_time", "includeTime %v is neither a boolean nor null and is dropped — "+ + "there is no way to write it", protoValueToJSON(v)) + } + } + + if v := e.detail(detailKeyRelationFormatObjectTypes); v != nil { + switch v.GetKind().(type) { + case *types.Value_ListValue, *types.Value_StringValue: + // present even when empty — an empty list is a cleared target + // set, the same user-intent reading that kept + // relationFormatObjectTypes off the §15 #12 trim whitelist + group.set("object_types", stringsToAny(e.typeSlugs(e.relationTargetKeys()))) + case *types.Value_NullValue: + group.set("object_types", nil) + default: + e.warn("/property_settings/object_types", "relationFormatObjectTypes %v is not a list and is dropped — "+ + "there is no way to write it", protoValueToJSON(v)) + } + } + doc.set(memberPropertySettings, group) + return nil +} + +// relationFormatName renders the stored relationFormat as its §3 name. The +// reading mirrors what every consumer of this detail does — int32 of the +// number, absent and null both the proto zero, longtext — so the document +// states the format the system actually serves for this relation. +// +// A value that reading cannot name is an ERROR, not a fallback: `format` is +// required on a relation document, so there is nothing to omit, and writing +// "text" for a format that is not text would import as a permanent silent +// format rewrite — the exact disease the lift exists to kill. Failing the +// export instead follows buildDoc's own rule for a smartblock type with no +// kind mapping, and it is corrupt-data-only territory: formatNames is total +// over model.RelationFormat (pinned by TestFormatNames_TotalOverModelEnum), +// and all 10,617 production relation documents carry an in-enum integer. +func (e *exporter) relationFormatName() (string, error) { + v := e.detail(detailKeyRelationFormat) + var n float64 + switch k := v.GetKind().(type) { + case nil, *types.Value_NullValue: + n = 0 + case *types.Value_NumberValue: + n = k.NumberValue + default: + return "", fmt.Errorf("relation format %v is not a number: this document cannot state "+ + "what it defines", protoValueToJSON(v)) + } + if math.IsNaN(n) || math.IsInf(n, 0) || n < 0 || n > math.MaxInt32 { + return "", fmt.Errorf("relation format %v is outside the format enum: this document "+ + "cannot state what it defines", n) + } + name := formatName(model.RelationFormat(int32(n))) + if name == "" { + return "", fmt.Errorf("relation format %v has no name in this format: "+ + "this document cannot state what it defines", n) + } + return name, nil +} + +// relationTargetKeys is the stored relationFormatObjectTypes list with each +// entry translated to the stored type KEY it names, memoized because the +// type-key census (seedTypeTermLedger) and buildPropertySettings both read +// it — the same one-build rule as iconField (§2b). +// +// Translation is per entry: a type object id inverts through the +// TypeResolver capability when the resolver carries it, and a bare type key +// the legacy import paths stored directly (21 production entries) passes +// through verbatim, its own address (§3) — a key is vocabulary, and a +// vocabulary miss is never evidence of nonexistence. What no longer passes +// is an entry the SPACE's own store disowns (§9): the +// `_missing_object` sentinel, and an object id the wired existence +// capability says names no row — 56 production properties carry one, type +// ids from the account where a shipped use case was AUTHORED, and an object +// id differs in every space while a key does not. Both drop, the real id +// with a warning naming it; `object_types` is a list, and a list expresses +// absence by being shorter. The predicate is DroppedMissingObjectRef, +// shared with snapshotdiff, so the comparator drops exactly what export +// drops. Without the capability — package-only, offline — everything still +// passes through verbatim and the round trip stays byte-exact: an id the +// store merely could not be asked about is still the stored value's +// meaning, and a backup format that deletes it on export is disqualifying. +func (e *exporter) relationTargetKeys() []string { + if e.relTargetsBuilt { + return e.relTargets + } + e.relTargetsBuilt = true + entries := valueStringList(e.detail(detailKeyRelationFormatObjectTypes)) + tr, _ := e.opts.ResolveProperties.(TypeResolver) + out := make([]string, 0, len(entries)) + for _, entry := range entries { + if tr != nil { + if key, ok := tr.TypeKeyById(entry); ok && key != "" { + out = append(out, key) + continue + } + } + if e.droppedMissingListEntry("/property_settings/object_types", entry) { + continue + } + out = append(out, entry) + } + e.relTargets = out + return out +} + +// +// ---- import ---- +// + +// applyPropertySettings writes the stored keys the §2d group's members stand +// for. Presence mirrors presence: a member the document omits writes +// nothing, so the details that came out are the details that go back in. +func (imp *importer) applyPropertySettings(details *types.Struct, sbType model.SmartBlockType) error { + if !isPropertySmartBlock(sbType) { + // the schema keeps the group off every other kind (§2d), so there is + // nothing here to read — Unmarshal validates before it decodes, + // deliberately (§12 I2) + return nil + } + rs := imp.doc.PropertySettings + if rs == nil { + // unreachable off a validated document — the schema requires the + // group on both property-document kinds — but Unmarshal must not crash on a + // snapshot rebuilt by a caller that skipped Validate + return nil + } + if rs.Format != "" { + // the name resolves per key, exactly as a type_properties entry's + // format does (§3): "text" names both stored text formats, and the + // relation's own envelope `key` is what disambiguates — a bundled + // short-text relation (name, globalName, …) keeps its stored format + // across a round trip even though the document never spells it + f := declaredFormatWith(imp.opts, imp.doc.InternalKey, rs.Format) + details.Fields[detailKeyRelationFormat] = &types.Value{ + Kind: &types.Value_NumberValue{NumberValue: float64(f)}} + } + if raw := rs.IncludeTime; len(raw) > 0 { + if string(raw) == "null" { + details.Fields[detailKeyRelationFormatIncludeTime] = &types.Value{ + Kind: &types.Value_NullValue{}} + } else { + var b bool + if err := jsonUnmarshal(raw, &b); err != nil { + return fmt.Errorf("decode include_time: %w", err) + } + details.Fields[detailKeyRelationFormatIncludeTime] = &types.Value{ + Kind: &types.Value_BoolValue{BoolValue: b}} + } + } + if raw := rs.TargetTypes; len(raw) > 0 { + if string(raw) == "null" { + details.Fields[detailKeyRelationFormatObjectTypes] = &types.Value{ + Kind: &types.Value_NullValue{}} + return nil + } + var slugs []string + if err := jsonUnmarshal(raw, &slugs); err != nil { + return fmt.Errorf("decode object_types: %w", err) + } + tr, _ := imp.opts.ResolveProperties.(TypeResolver) + vals := make([]*types.Value, 0, len(slugs)) + for j, slug := range slugs { + slotPath := fmt.Sprintf("/property_settings/object_types/%d", j) + // a TYPE key slot (§2d): the document's own legend first, then + // the vocabulary — and the seam refuses a resolution onto the + // empty key, which has no written form (§3), the same refusal + // applyTypeProperties makes for its object_types + key := imp.typeKey(slug, slotPath) + if key == "" { + return &ValidationError{Issues: []Issue{{ + Path: slotPath, + Message: unwritableKeyReason("resolved type key", key), + }}} + } + // the store speaks ids; the TypeResolver capability turns the + // key back into this space's type object id, and a key the + // space does not serve stays a key for the wiring to reconcile + // — applyTypeProperties' degradation, on the type namespace + id := key + if tr != nil { + if resolved, ok := tr.TypeIdByKey(key); ok && resolved != "" { + id = resolved + } + } + vals = append(vals, &types.Value{Kind: &types.Value_StringValue{StringValue: id}}) + } + details.Fields[detailKeyRelationFormatObjectTypes] = &types.Value{ + Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: vals}}} + } + return nil +} + +// +// ---- validation ---- +// + +// propertySettingsOf reads the §2d group off a raw document, for the checks +// that run before it decodes. One reader, so the slot issue and the semantic +// pass cannot disagree about where the group lives. +func propertySettingsOf(doc map[string]any) (map[string]any, bool) { + raw, has := doc[memberPropertySettings] + group, _ := raw.(map[string]any) + return group, has +} + +// propertyFormatSlotIssue words the missing-definition verdict on a property +// document — missingFormatIssue's trade (§2b) at the §2d slot: `required` +// can say a member is missing but not what the choices are, and the author +// most likely to hit it is holding an older document whose format lives at +// the root (legacy) or in `properties` as a raw number (legacy). The +// kind is read RAW, exactly as the schema's `if` reads it, so the two +// verdicts cannot disagree about which documents owe the group — +// isPropertyKind and the schema's `if` list the same kinds. The names are +// read out of the published schema (propertyFormatEnum), never restated. +func propertyFormatSlotIssue(doc map[string]any, r *keySlotReport) { + if !isPropertyKind(doc) { + return + } + group, hasGroup := propertySettingsOf(doc) + if hasGroup { + if _, has := group["format"]; has { + return + } + } + names := propertyFormatEnum() + if len(names) == 0 { + return + } + var msg string + path := "" + if hasGroup { + path = "/property_settings" + msg = fmt.Sprintf("missing property 'format': a property document states "+ + "the format of the property it defines — one of %s", quotedList(names)) + } else { + msg = fmt.Sprintf("missing property 'property_settings': a property document states "+ + "the definition of the property it IS — at least `format`, one of %s", quotedList(names)) + } + // the migration hints, on the same reasoning as `refs` (§10): each older + // spelling is exactly one this verdict fires on, and told only that a + // member is missing, the obvious wrong repair is to invent one while + // leaving the old spelling where it sits. The hints keep the OLD member + // names because they describe what the older document in hand SPELLS. + if _, atRoot := doc["format"]; atRoot && !hasGroup { + msg += `. This document spells "format" at the root — the legacy root form: ` + + `the definition moved into the "property_settings" group, so move ` + + `"format" (and "include_time"/"object_types" beside it) in there` + } + if _, was := doc["relation_settings"]; was && !hasGroup { + // the legacy group name, before the format stopped calling a + // property a relation anywhere; the members inside are unchanged + msg += `. This document spells the group "relation_settings" — the legacy group name: ` + + `rename the group to "property_settings"` + } + if props, _ := doc["properties"].(map[string]any); props != nil { + if _, legacy := props["relation_format"]; legacy { + msg += `. This document spells "relation_format" inside properties — the legacy ` + + `form: replace that raw number with its name in property_settings` + } + // the OTHER wrong container, and the commoner one: 9 of 9 + // small-model attempts wrote `format` inside `properties`, where it + // is a custom property named "format" and the relation ends up with + // no format at all. Told only that a member is missing, the author + // has no reason to connect it to the member they DID write — and + // the warning that would say so lives in the semantic pass, which + // a schema failure never reaches. + if _, phantom := props["format"]; phantom { + msg += `. This document spells "format" inside properties, where it names a ` + + `CUSTOM property rather than the property's own format: move that member ` + + `into property_settings` + } + } + r.rejectValueAt(path, msg) +} + +// propertySettingsIssues runs the §2d checks the schema cannot express: a +// meaningful value against a format that cannot use it. WARNINGS, not +// errors, and the grade is load-bearing (wrongShapeForFormat's reasoning): +// the stored details are not authored — a real relation may carry +// `includeTime` against any format, and 8,375 production relations carry a +// false one against a non-date format — so a refusal would make Marshal +// emit what Validate rejects (I1), and an export that dropped the value +// instead would silently delete stored state. §2a's array can afford its +// errors because it is authored, never lifted from a store. +// +// Only a MEANINGFUL value warns — `include_time: true`, a non-empty +// `object_types` — because a false or empty one against the wrong format +// says nothing the reader would act on, and warning on it would fire on +// most of the corpus (8,375 present-and-false alone), burying the case an +// author can actually fix. +func propertySettingsIssues(doc map[string]any, warn func(path, format string, args ...any)) { + if !isPropertyKind(doc) { + return // the schema refuses the group on every other kind + } + group, _ := propertySettingsOf(doc) + format, _ := group["format"].(string) + if format == "" { + // required and missing: the schema's error already says so, and it + // is the one that names the wrong container too + // (propertyFormatSlotIssue) — this pass never runs on a document + // the schema rejected, so it cannot be the place that says it. + return + } + propertyPhantomIssues(doc, warn) + if v, has := group["include_time"]; has && format != "date" { + if b, isBool := v.(bool); isBool && b { + warn("/property_settings/include_time", "include_time is only meaningful on date, not %q — "+ + "it is carried but nothing reads it", format) + } + } + if v, has := group["object_types"]; has && format != "objects" && format != "files" { + if list, isList := v.([]any); isList && len(list) > 0 { + warn("/property_settings/object_types", "object_types is only meaningful on objects/files, not %q — "+ + "it is carried but nothing reads it", format) + } + } +} + +// propertyPhantomIssues reports a `properties` member spelling one of the +// three FIELD names on a relation document. It is almost certainly the +// envelope field written in the wrong container — the exact shape 9 of 9 +// small-model attempts wrote, which validated silently and imported as a +// custom property literally named `format`, leaving the relation with no +// relationFormat at all. +// +// It stays a WARNING, because the spelling is a legitimate custom property +// key (a media space really can have a "Format" column) and a relation +// object carrying one must stay exportable (§11 I1). +// +// The kind gate is the property-document SET, not `property` alone. Export +// writes `bundled_property` — 0 of 38,061 corpus +// documents — but the schema's `kind` enum offers it beside `property` +// with nothing marking it non-authorable, and an author who picks it +// walks straight back into the §2d bug with every gate silent. A kind +// nothing emits is exactly the kind nobody thought to guard. +func propertyPhantomIssues(doc map[string]any, warn func(path, format string, args ...any)) { + props, _ := doc["properties"].(map[string]any) + if props == nil { + return + } + for _, member := range []string{"format", "include_time", "object_types"} { + if _, has := props[member]; has { + warn("/properties/"+member, "on a property document %q names a CUSTOM property, "+ + "not this property's own %s — that lives in property_settings; "+ + "drop this member unless a property literally named %q is meant", + member, member, member) + } + } +} + +// isPropertyKind reports the kinds whose document IS a property definition: +// the one export writes, plus the one the schema's enum offers beside it. +func isPropertyKind(doc map[string]any) bool { + kind, _ := doc["kind"].(string) + return kind == kindNames.name(model.SmartBlockType_STRelation) || + kind == kindNames.name(model.SmartBlockType_BundledRelation) +} diff --git a/pkg/lib/anyblockjson/relationformat_test.go b/pkg/lib/anyblockjson/relationformat_test.go new file mode 100644 index 0000000000..f778ba2bf1 --- /dev/null +++ b/pkg/lib/anyblockjson/relationformat_test.go @@ -0,0 +1,1191 @@ +package anyblockjson + +// relationformat_test.go — the §2d relation-definition group: +// `property_settings` with `format`, `include_time`, `object_types` on +// kind:property documents, the refusal of their flat spellings in +// `properties`, and the presence-mirror round trip. +// +// v0.38 renamed the wire vocabulary here — kinds `relation`→`property` / +// `bundled_relation`→`bundled_property`, the group +// `relation_settings`→`property_settings` — so every fixture and pinned +// message in this file moved to the new spelling. The RULES pinned are +// unchanged: same gates, same repairs, same warnings, one spelling later. + +import ( + "encoding/json" + "math" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func nullValue() *types.Value { + return &types.Value{Kind: &types.Value_NullValue{}} +} + +// relationSnapshot is a minimal kind:property snapshot — the details a real +// relation object carries, minus the install noise this test does not need. +func relationSnapshot(details map[string]*types.Value) *model.SmartBlockSnapshotBase { + if details == nil { + details = map[string]*types.Value{} + } + details["id"] = str("relObjectId") + if _, has := details["name"]; !has { + details["name"] = str("Budget") + } + return &model.SmartBlockSnapshotBase{ + Key: "budget", + Details: fields(details), + Blocks: []*model.Block{{ + Id: "relObjectId", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + } +} + +// typeIdVocabulary is a TypeResolver-capable property resolver: the shape +// storeresolver has, reduced to the id↔key translation the §2d slot needs. +type typeIdVocabulary struct { + testPropertyResolver + keyById map[string]string + idByKey map[string]string +} + +func (r *typeIdVocabulary) TypeKeyById(id string) (string, bool) { + key, ok := r.keyById[id] + return key, ok +} + +func (r *typeIdVocabulary) TypeIdByKey(key string) (string, bool) { + id, ok := r.idByKey[key] + return id, ok +} + +func newTypeIdVocabulary() *typeIdVocabulary { + return &typeIdVocabulary{ + testPropertyResolver: *newTestPropertyResolver(), + keyById: map[string]string{"typeid-page": "page", "typeid-wine": "wine"}, + idByKey: map[string]string{"page": "typeid-page", "wine": "typeid-wine"}, + } +} + +// The three stored details travel on the envelope, never in `properties`. +// +// How this can fail: drop propertySettingsLiftedDetailKeys from envelopeLiftedKeys +// and the flat spellings reappear in properties; drop the +// buildPropertySettings call from buildDoc and the envelope fields vanish. +func TestRelationEnvelope_LiftsTheThreeDetails(t *testing.T) { + // given + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(2), + "relationFormatIncludeTime": boolValue(false), + "relationFormatObjectTypes": strList("typeid-page"), + }) + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, testOptions()) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"format": "number"`) + assert.Contains(t, string(data), `"include_time": false`) + assert.Contains(t, string(data), `"object_types"`) + for _, spelling := range []string{`"relation_format"`, `"relation_format_include_time"`, + `"relation_format_object_types"`} { + assert.NotContains(t, string(data), spelling, + "the flat spelling must not survive anywhere — properties refuses it (§2d)") + } + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (§11 I1)") +} + +// `format` is required on a relation document, so every relation export +// writes one — including a stored 0 (longtext, a real format) and a snapshot +// with no relationFormat detail at all, both of which write "text". +// +// How this can fail: emit `format` with setNonEmpty, or skip it when the +// detail is absent, and the exported document is one Validate refuses (I1). +func TestRelationEnvelope_FormatIsAlwaysWritten(t *testing.T) { + for name, details := range map[string]map[string]*types.Value{ + "stored zero": {"relationFormat": num(0)}, + "absent detail": {}, + } { + t.Run(name, func(t *testing.T) { + // when + data, err := Marshal(model.SmartBlockType_STRelation, relationSnapshot(details), testOptions()) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"format": "text"`) + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + }) + } +} + +// formatNames is total over model.RelationFormat — the property that makes +// the required §2d `format` safe for every stored enum value. shorttext is +// the one deliberate hole; formatName folds it into "text". +// +// How this can fail: add a value to the model enum without naming it here +// (the way "map" was missing before v0.31, on 72 production documents), or +// remove a name from formatNames. +func TestFormatNames_TotalOverModelEnum(t *testing.T) { + for raw, enumName := range model.RelationFormat_name { + f := model.RelationFormat(raw) + assert.NotEmpty(t, formatName(f), + "stored format %s (%d) has no §3 name: a relation object carrying it cannot be exported (§2d)", + enumName, raw) + } +} + +// Format "map" (102) is real data — 72 production relation documents, every +// one the bundled templatePlaceholders relation — and round-trips by name. +// +// How this can fail: remove RelationFormat_map from formatNames and export +// errors; remove "map" from the schema enum and the exported document fails +// its own validation. +func TestRelationEnvelope_MapFormatRoundTrips(t *testing.T) { + // given + snap := relationSnapshot(map[string]*types.Value{"relationFormat": num(102)}) + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + sbType, got, err := Unmarshal(data, testOptions()) + + // then + require.NoError(t, err) + assert.Contains(t, string(data), `"format": "map"`) + assert.Equal(t, model.SmartBlockType_STRelation, sbType) + assert.Equal(t, float64(102), got.Details.Fields["relationFormat"].GetNumberValue()) +} + +// A stored format the vocabulary cannot name fails the export by name, +// rather than being rewritten to "text": `format` is required, so there is +// nothing to omit, and a false format claim imports as a permanent silent +// format rewrite — the disease the lift exists to kill. +// +// How this can fail: make relationFormatName fall back to "text" for an +// unnameable value and the error disappears. +func TestRelationEnvelope_UnnameableFormatFailsExport(t *testing.T) { + for name, v := range map[string]*types.Value{ + "outside the enum": num(47), + "not a number": str("weird"), + } { + t.Run(name, func(t *testing.T) { + // when + _, err := Marshal(model.SmartBlockType_STRelation, + relationSnapshot(map[string]*types.Value{"relationFormat": v}), testOptions()) + + // then + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot state what it defines", + "the failure has to say the document could not be written, not merely that a value was odd") + }) + } +} + +// A stored relationFormat that is NaN, infinite, or outside int32 fails the +// export BY THE GUARD, before the float ever reaches int32: what int32(n) +// yields for such values is implementation-dependent (Go spec, Conversions), +// and measured here int32(NaN) is 0 — so without the guard a NaN format +// exports as `"format": "text"`, a false claim about what the property is +// that imports as a permanent silent rewrite to longtext, the exact disease +// the §2d lift exists to kill. The other inputs happen to land outside the +// enum on this architecture and would still error by name, which is why +// every subtest pins the GUARD's own message rather than merely +// require.Error: the pin must hold on every architecture, never on the +// accident of what the conversion produces. Corrupt-data-only territory — +// all 10,617 production relation documents carry an in-enum integer — but +// corrupt data is exactly what an export must refuse to launder into a +// well-formed lie. +// +// How this can fail: drop the IsNaN/IsInf/int32-range guard from +// relationFormatName. The NaN subtest then exports `"format": "text"` +// cleanly, and the rest degrade to the no-name error — the wrong statement +// about a value the reading never legitimately produced. +func TestRelationEnvelope_NonFiniteFormatFailsExportByTheGuard(t *testing.T) { + for name, raw := range map[string]float64{ + "NaN": math.NaN(), + "+Inf": math.Inf(1), + "-Inf": math.Inf(-1), + "beyond int32": 1e10, + "negative": -1, + } { + t.Run(name, func(t *testing.T) { + // when + _, err := Marshal(model.SmartBlockType_STRelation, + relationSnapshot(map[string]*types.Value{"relationFormat": num(raw)}), testOptions()) + + // then + require.Error(t, err, "a value the enum cannot hold must fail export, never become text") + assert.Contains(t, err.Error(), "outside the format enum", + "the guard, not the int32 conversion's accident, must be what refuses this value") + }) + } +} + +// The flat spellings are refused in `properties`, by Validate AND by +// Unmarshal (§12 I2), with the envelope repair named. This is also the whole +// of the legacy story (§10): a legacy document spells `relation_format` +// here and is refused loudly instead of read. +// +// How this can fail: drop the propertySettingsLiftedDetailKeys arm from +// deniedPropertyKey and both doors accept the raw number again — a phantom +// property in Validate's case, a silent second spelling in Unmarshal's. +func TestRelationEnvelope_RefusedInProperties(t *testing.T) { + // The spellings that resolve onto the three lifted stored keys are their + // display NAMES (bundledname.go); the old `relation_*` and v0.38 + // `property_*` slugs no longer resolve to anything — a denied key's fold + // class answers nothing, deliberately — so they are ordinary custom keys + // and cannot trip this refusal: a legacy document is refused earlier, at + // the kind enum and the missing property_settings verdict. + for spelling, value := range map[string]string{ + "Format": "100", + "IncludeTime": "true", + "Property's target object types": `["Page"]`, + } { + t.Run(spelling, func(t *testing.T) { + doc := `{"version":2,"kind":"property","id":"o1","internal_key":"budget",` + + `"property_settings":{"format":"number"},` + + `"properties":{"` + spelling + `":` + value + `}}` + + err := Validate([]byte(doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/properties/"+spelling) + assert.Contains(t, err.Error(), "in property_settings", + "the refusal names the repair") + + _, _, err = Unmarshal([]byte(doc), testOptions()) + require.Error(t, err, "Unmarshal must refuse what Validate refuses (§12 I2)") + }) + } +} + +// Envelope presence mirrors stored presence EXACTLY — false, `[]` and null +// all travel, absence stays absence — so the same details go in and out and +// the snapshot comparator needs no new rule (§2d, §11). +// +// How this can fail: emit include_time or object_types with setNonEmpty +// (false and [] vanish), drop the null arms (80 production relations hold a +// null includeTime), or make import invent a detail the document does not +// carry. +func TestRelationEnvelope_PresenceMirrorsTheStore(t *testing.T) { + for name, tc := range map[string]struct { + details map[string]*types.Value + wire []string // substrings the document must carry + notOnDoc []string // members the document must NOT carry + }{ + "present and false/empty": { + details: map[string]*types.Value{ + "relationFormat": num(6), + "relationFormatIncludeTime": boolValue(false), + "relationFormatObjectTypes": strList(), + }, + wire: []string{`"include_time": false`, `"object_types": []`}, + }, + "present and null": { + details: map[string]*types.Value{ + "relationFormat": num(4), + "relationFormatIncludeTime": nullValue(), + }, + wire: []string{`"include_time": null`}, + notOnDoc: []string{`"object_types"`}, + }, + // object_types stored as a NULL is its own arm on both sides, and + // it is the one that can break §11 I1: export writes + // `"object_types": null`, so the schema's type union has to admit + // null or Marshal emits what Validate rejects. Nothing reached this + // path before — the include_time null case above exercises a + // different arm. + "object_types present and null": { + details: map[string]*types.Value{ + "relationFormat": num(100), + "relationFormatObjectTypes": nullValue(), + }, + wire: []string{`"object_types": null`}, + notOnDoc: []string{`"include_time"`}, + }, + "absent": { + details: map[string]*types.Value{"relationFormat": num(2)}, + notOnDoc: []string{`"include_time"`, `"object_types"`}, + }, + } { + t.Run(name, func(t *testing.T) { + // given + snap := relationSnapshot(tc.details) + want := map[string]*types.Value{} + for k, v := range snap.Details.Fields { + if propertySettingsLiftedDetailKeys()[k] { + want[k] = v + } + } + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + _, got, err := Unmarshal(data, testOptions()) + require.NoError(t, err) + + // then + for _, w := range tc.wire { + assert.Contains(t, string(data), w) + } + for _, w := range tc.notOnDoc { + assert.NotContains(t, string(data), w, + "an absent stored key writes no envelope member") + } + for k, v := range want { + assert.Equal(t, v, got.Details.Fields[k], "detail %q changed on the way round", k) + } + for k := range propertySettingsLiftedDetailKeys() { + if _, wanted := want[k]; !wanted { + assert.Nil(t, got.Details.Fields[k], "the round trip invented detail %q", k) + } + } + }) + } +} + +// With the TypeResolver capability wired, the stored target-type ids are +// spelled as type keys on the wire and come back as the same ids — the id↔key +// translation is an inverse, so the snapshot round-trips byte-exactly. An +// entry the resolver cannot answer passes through verbatim, its own address. +// +// How this can fail: drop the TypeKeyById arm from relationTargetKeys and +// the wire carries raw ids under a resolver; drop the TypeIdByKey arm from +// applyPropertySettings and the round trip stores keys where ids were. +func TestRelationEnvelope_TargetTypesTranslateThroughTheResolver(t *testing.T) { + // given + opts := testOptions() + opts.ResolveProperties = newTypeIdVocabulary() + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(100), + "relationFormatObjectTypes": strList("typeid-page", "bafyreidangling"), + }) + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, opts) + require.NoError(t, err) + _, got, err := Unmarshal(data, opts) + require.NoError(t, err) + + // then + assert.Equal(t, []string{"Page", "bafyreidangling"}, docObjectTypes(t, data), + "a resolvable id spells its type's name; an unresolvable one passes through verbatim (§3)") + assert.Equal(t, strList("typeid-page", "bafyreidangling"), + got.Details.Fields["relationFormatObjectTypes"], + "the translation must invert: ids in, ids out") +} + +// Without the capability, entries pass through verbatim in both directions — +// the offline round trip is byte-exact, and nothing is invented or dropped. +// +// How this can fail: make relationTargetKeys drop entries it cannot +// translate (the §2a dangling-id policy, wrong here: the stored value IS the +// meaning) and the ids vanish from the wire. +func TestRelationEnvelope_TargetTypesPassThroughWithoutResolver(t *testing.T) { + // given + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(100), + "relationFormatObjectTypes": strList("bafyreitypeone", "wine"), + }) + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, testOptions()) + require.NoError(t, err) + _, got, err := Unmarshal(data, testOptions()) + require.NoError(t, err) + + // then + assert.Equal(t, []string{"bafyreitypeone", "wine"}, docObjectTypes(t, data)) + assert.Equal(t, strList("bafyreitypeone", "wine"), + got.Details.Fields["relationFormatObjectTypes"]) +} + +// blankTypeResolver answers every translation with ("", true) — the shape a +// resolver bug really produces when its map carries an entry whose value was +// never filled (storeresolver builds keyById from a bounded space query, and +// a type row with an empty stored key lands exactly this). The same defect +// class already bit the property seam once: a vocabulary resolving a slug to +// "" put details[""] in the store (TestImport_SeamRefusesAnUnwritableResolvedKey). +type blankTypeResolver struct{ *testPropertyResolver } + +func (blankTypeResolver) TypeKeyById(id string) (string, bool) { return "", true } +func (blankTypeResolver) TypeIdByKey(key string) (string, bool) { return "", true } + +// The TypeResolver contract for an EMPTY answer: ok-with-"" is NO answer, in +// both directions. The empty string has no written form (§3) and is no +// store address either, so trusting the ok flag alone would let the +// translation DESTROY the value it was asked to translate — export would put +// "" in a type-key slot where the stored id was, import would store "" where +// the key was — when the §2d rule is that an entry the resolver cannot +// answer passes through verbatim, its own address, for the wiring to +// reconcile. Pass-through, never a refusal: the stored value IS the meaning, +// and a backup format that loses it to a resolver bug is disqualifying. +// +// How this can fail: drop `key != ""` from relationTargetKeys' TypeKeyById +// arm and the wire carries "" where the stored id was; drop `resolved != ""` +// from applyPropertySettings's TypeIdByKey arm and the store receives "" +// where the key was. Each drop is silent on every other test — the working +// resolvers never answer ok-with-empty. +func TestRelationEnvelope_AResolverAnsweringEmptyIsNoAnswer(t *testing.T) { + // given + opts := testOptions() + opts.ResolveProperties = blankTypeResolver{newTestPropertyResolver()} + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(100), + "relationFormatObjectTypes": strList("bafyreitypeone"), + }) + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, opts) + require.NoError(t, err) + _, got, err := Unmarshal(data, opts) + require.NoError(t, err) + + // then + assert.Equal(t, []string{"bafyreitypeone"}, docObjectTypes(t, data), + "export: ok-with-empty is no translation — the stored id is its own address (§3)") + assert.Equal(t, strList("bafyreitypeone"), + got.Details.Fields["relationFormatObjectTypes"], + "import: ok-with-empty is no translation — the key passes through verbatim") +} + +// A reference slot that legitimately NAMES a lifted key — a dataview column +// on the Property type, a type_properties entry — keeps the §3 spelling, +// which is the key's display NAME: the deny rule protects the legend, and a +// bundled-bound spelling needs no legend entry, so the rule never sees it. +// 64 production spaces carry exactly this document. +// +// How this can fail: restore writableSlug's blanket deny refusal and the +// column spells "relationFormat" camelCase-verbatim with a warning. +func TestRelationEnvelope_ReferenceSlotsKeepTheBundledSlug(t *testing.T) { + // given a set over relation objects, showing the format column — the + // Property type's own view + snap := dataviewSnapshot(&model.RelationLink{ + Key: "relationFormat", Format: model.RelationFormat_number, + }) + + // when + var warns []Issue + opts := testOptions() + opts.OnWarning = func(i Issue) { warns = append(warns, i) } + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"property": "Format"`, + "naming the property is not writing its value — the reference keeps its spelling, the display name") + assert.NotContains(t, string(data), `"relationFormat"`, + "the verbatim fallback is for keys whose slug would need a legend entry") + assert.NotContains(t, string(data), `"property_internal_keys"`, + "a bundled binding needs no legend entry — that is what makes the slug safe") + assert.Empty(t, warns) + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") +} + +// The denied-key exemption is TWO questions, and this pins the first: the +// bundled table must BIND the slug to this very key. Slugs come from +// apiObjectKey — user-editable — so a space really can spell `relationFormat` +// with a slug of its own, and that slug inverts through the vocabulary in +// force while the bundled table has never heard of it. Such a slug OWES a +// legend entry (recordPropertyKey's rule: a spelling the bundled table does +// not bind always does), and the entry's value would be the denied key, which +// the §3 deny rule refuses — so the document would spell `fmt_col` with +// nothing anywhere saying what it means, and every reader would resolve it +// verbatim (chain step 4) as a custom property named fmt_col: a silent +// repoint of the reference. The stored key is the one spelling that needs no +// entry, so it is the one written. The safe population the exemption serves +// (64 production spaces showing the Property type's format column) is +// bundled-bound by construction, so backing THIS slug off costs it nothing. +// +// How this can fail: drop the bundledBinds half from writableSlug's +// denied-key exemption. termInverts alone accepts this slug — the writer's +// own space does invert it — and the column spells "fmt_col" with no legend +// entry possible for it. +func TestRelationEnvelope_ADeniedKeySlugTheBundledTableDoesNotBindBacksOff(t *testing.T) { + // given the Property type's format column, under a vocabulary that + // spells the lifted key with a space-minted slug of its own + snap := dataviewSnapshot(&model.RelationLink{ + Key: "relationFormat", Format: model.RelationFormat_number, + }) + var warns []Issue + opts := testOptions() + opts.Keys = typedSpaceVocabulary{propSlugOf: map[string]string{"relationFormat": "fmt_col"}} + opts.OnWarning = func(i Issue) { warns = append(warns, i) } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"property": "relationFormat"`, + "a slug that would owe an unwritable legend entry backs off to the stored key (§3)") + assert.NotContains(t, string(data), "fmt_col") + assert.NotContains(t, string(data), `"property_internal_keys"`, + "the denied key can never be a legend value — that is why the slug had to go") + require.NotEmpty(t, warns, "the backed-off spelling is reported") + assert.Contains(t, warns[0].Message, "cannot be a legend value") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") +} + +// …and this pins the second question: the vocabulary IN FORCE must invert +// the spelling. The bundled table binds "Format" to relationFormat, but +// the writer's own space is a reader too — its vocabulary answers FIRST on +// import, ahead of the bundled table — and here a custom relation has +// claimed the spelling "Format" for itself (the freed-spelling +// hazard recordPropertyKey documents: measured on the property namespace, +// the same shadowing silently lands dueDate's value on the custom relation +// that wanted the spelling). Writing the slug would re-point the column to +// the shadowing relation the moment the document is read back where it was +// written, and no legend entry can correct it, because the entry's value +// would be the denied key. Only the verbatim stored key — always its own +// address (§3) — survives that reader. +// +// How this can fail: drop the termInverts half from writableSlug's +// denied-key exemption. bundledBinds alone accepts the slug, the column +// spells "Format", and the writer's own vocabulary binds it to the +// shadowing custom relation — a repoint with no error anywhere. +func TestRelationEnvelope_ADeniedKeySlugShadowedByTheVocabularyBacksOff(t *testing.T) { + // given the same format column, under a vocabulary where a custom + // relation holds the bundled slug of the lifted key + snap := dataviewSnapshot(&model.RelationLink{ + Key: "relationFormat", Format: model.RelationFormat_number, + }) + var warns []Issue + opts := testOptions() + opts.Keys = typedSpaceVocabulary{ + propSlugOf: map[string]string{"64af1efbc52a6a5ed6e9dabc": "Format"}} + opts.OnWarning = func(i Issue) { warns = append(warns, i) } + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"property": "relationFormat"`, + "the writer's own space would re-point the spelling, so the stored key is the honest one") + assert.NotContains(t, string(data), `"Format"`) + require.NotEmpty(t, warns, "the backed-off spelling is reported") + assert.Contains(t, warns[0].Message, "cannot be a legend value") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") +} + +// The target keys are in the TYPE-KEY CENSUS: verbatim-first (§3) makes each +// its own address, so no other key's slug may take one as a spelling — the +// same duty §2a's targets have carried since typeProperties shipped. +// +// How this can fail: drop the relationTargetKeys loop from +// seedTypeTermLedger. A vocabulary that slugs the document's own TYPE onto a +// target key's spelling then wins the term, the envelope `type` and a target +// entry both spell "wine", and the legend binds the spelling to the wrong +// key — a type substitution with no error anywhere. +func TestRelationEnvelope_TargetKeysAreInTheTypeCensus(t *testing.T) { + // given a vocabulary spelling the relation's own type key as `wine`, + // while the relation targets the stored type key `wine` + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(100), + "relationFormatObjectTypes": strList("wine"), + }) + snap.ObjectTypes = []string{"ot-custom123"} + opts := testOptions() + opts.Keys = typedSpaceVocabulary{typeSlugOf: map[string]string{"custom123": "wine"}} + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, opts) + require.NoError(t, err) + + // then: the stored key `wine` kept its spelling, so the vocabulary's + // binding backed off to the verbatim key + var doc struct { + Type string `json:"type"` + PropertySettings struct { + ObjectTypes []string `json:"object_types"` + } `json:"property_settings"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "custom123", doc.Type, + "the census reserved %q for the target, so the type spells its stored key", "wine") + assert.Equal(t, []string{"wine"}, doc.PropertySettings.ObjectTypes) +} + +// docObjectTypes reads the §2d target-type list out of a rendered document — +// inside the property_settings group since v0.32. +func docObjectTypes(t *testing.T, data []byte) []string { + t.Helper() + var doc struct { + PropertySettings struct { + ObjectTypes []string `json:"object_types"` + } `json:"property_settings"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + return doc.PropertySettings.ObjectTypes +} + +// A custom stored type key in `object_types` owes the type_internal_keys legend its +// identity entry, exactly as the same key would in +// type_properties[].object_types — the §2d slot is a type-key slot, not a +// free string. +// +// How this can fail: write the entries raw instead of through typeSlugs and +// the legend entry disappears — a reader whose vocabulary binds the spelling +// elsewhere then resolves the target to a different type. +func TestRelationEnvelope_TargetTypesOweTheTypeLegend(t *testing.T) { + // given a custom stored key whose spelling the bundled table cannot invert + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(100), + "relationFormatObjectTypes": strList("wine"), + }) + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, testOptions()) + require.NoError(t, err) + + // then + var doc struct { + TypeKeys map[string]string `json:"type_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "wine", doc.TypeKeys["wine"], + "a verbatim custom type key owes the identity entry (§3)") +} + +// `include_time` against a non-date format and a non-empty `object_types` +// against a non-objects/files format WARN and stay valid: the stored details +// are not authored, so a refusal would make Marshal emit what Validate +// rejects (I1). Only a MEANINGFUL value warns — a false or empty one against +// the wrong format is most of the corpus (8,375 present-and-false +// include_time alone) and says nothing an author could act on. +// +// How this can fail: drop the propertySettingsIssues call from +// semanticIssues and the warnings vanish; warn on any presence and the +// no-warning cases light up. +func TestRelationEnvelope_WrongFormatWarnsButCarries(t *testing.T) { + for name, tc := range map[string]struct { + doc string + wantWarn string // "" = no warning expected + }{ + "include_time true on number": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{"format":"number","include_time":true}}`, + wantWarn: "/property_settings/include_time", + }, + "object_types non-empty on number": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{"format":"number","object_types":["page"]}}`, + wantWarn: "/property_settings/object_types", + }, + "include_time false on number": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{"format":"number","include_time":false}}`, + }, + "object_types empty on number": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{"format":"number","object_types":[]}}`, + }, + "include_time true on date": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{"format":"date","include_time":true}}`, + }, + "object_types non-empty on objects": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{"format":"objects","object_types":["page"]}}`, + }, + } { + t.Run(name, func(t *testing.T) { + // when + var warns []Issue + err := ValidateWarn([]byte(tc.doc), func(i Issue) { warns = append(warns, i) }) + + // then + require.NoError(t, err, "a warning-grade fault must not refuse the document (§2d)") + if tc.wantWarn == "" { + assert.Empty(t, warns) + return + } + require.Len(t, warns, 1) + assert.Equal(t, tc.wantWarn, warns[0].Path) + assert.Contains(t, warns[0].Message, "only meaningful on") + }) + } +} + +// A `properties` member spelling one of the three FIELD names on a relation +// document warns: it is a custom property named "format", not the relation's +// format — the phantom-property shape the 9-of-9 eval failures wrote, which +// with the envelope field ALSO present would otherwise validate in silence. +// A warning and not a refusal, because the spelling is a legitimate custom +// key and a relation object carrying one must stay exportable (I1). +// +// How this can fail: drop the properties-member loop from +// propertySettingsIssues and the phantom shape validates with no notice; +// make it a refusal and the page case (where the member is an ordinary +// property) starts failing I1 for spaces that really have one. The members +// run one at a time, because propertyPhantomIssues lists the three +// literally and a member dropped from that list keeps the other two +// warning — the 9-of-9 eval failures happened to write `format`, but the +// bug they demonstrate (a §2d field name is an ordinary spelling inside +// `properties`) is a property of the container, so all three members walk +// into it the same way. +func TestRelationEnvelope_PhantomFieldNameInPropertiesWarns(t *testing.T) { + for member, value := range map[string]string{ + "format": `"number"`, + "include_time": `true`, + "object_types": `["vinyl"]`, + } { + t.Run(member, func(t *testing.T) { + // given the envelope format AND the phantom twin in properties + doc := `{"version":2,"kind":"property","id":"o1","internal_key":"b",` + + `"property_settings":{"format":"number"},` + + `"properties":{"name":"Budget","` + member + `":` + value + `}}` + + // when + var warns []Issue + err := ValidateWarn([]byte(doc), func(i Issue) { warns = append(warns, i) }) + + // then + require.NoError(t, err, "a custom property named %s is legal — the warning is the guard", member) + require.Len(t, warns, 1) + assert.Equal(t, "/properties/"+member, warns[0].Path) + assert.Contains(t, warns[0].Message, "CUSTOM property") + + // and on a PAGE the same member is an ordinary property: no warning + warns = nil + page := `{"version":2,"id":"o1","properties":{"` + member + `":` + value + `}}` + require.NoError(t, ValidateWarn([]byte(page), func(i Issue) { warns = append(warns, i) })) + assert.Empty(t, warns) + }) + } +} + +// The group is legal only on kind:property, `format` is required inside it, +// and every older spelling gets a message naming the repair rather than the +// mechanism — the migration story of two versions, in one gate. +// +// How this can fail: remove the allOf conditional from object.schema.json +// and the off-relation cases validate clean; remove the schemaIssueMessage +// arm and they degrade to a bare "not allowed"; drop a hint clause from +// propertyFormatSlotIssue or the root-spelling special case and the +// corresponding migration case loses its repair. +func TestRelationEnvelope_FieldsAreGatedByKind(t *testing.T) { + for name, tc := range map[string]struct{ doc, want string }{ + "property_settings on a page": { + doc: `{"version":2,"id":"o1","property_settings":{"format":"number"}}`, + want: `/property_settings: property "property_settings" is only valid on property documents`, + }, + "property_settings on a template": { + doc: `{"version":2,"kind":"template","id":"o1","type":"template","property_settings":{"format":"date"}}`, + want: `/property_settings: property "property_settings" is only valid on property documents`, + }, + "property_settings on a type": { + doc: `{"version":2,"kind":"object_type","id":"o1","property_settings":{"format":"objects"}}`, + want: `/property_settings: property "property_settings" is only valid on property documents`, + }, + "missing property_settings on a relation": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b"}`, + want: "missing property 'property_settings': a property document states the definition", + }, + "missing format inside the group": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{}}`, + want: "/property_settings: missing property 'format'", + }, + "the pre-v0.32 root spelling": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","format":"number"}`, + want: "moved off the root", + }, + "a refused member inside the group names its home": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{"format":"number","name":"Budget"}}`, + want: "the property's name is the `name` property", + }, + "legacy relation_format beside a missing definition": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","properties":{"relation_format":100}}`, + want: "the legacy form", + }, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(tc.doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + + _, _, err = Unmarshal([]byte(tc.doc), testOptions()) + assert.Error(t, err, "Unmarshal must refuse what Validate refuses (§12 I2)") + }) + } +} + +// A non-relation snapshot carrying any of the three details drops them with +// a warning: the refusal in `properties` is unconditional, so export may not +// write the flat spelling anywhere (I1), and no §2d field exists off a +// relation document to carry the value. Never observed in production — 0 of +// 27,444 non-relation documents carry any of the three — which is exactly +// why the warning is the whole guard: when the shape does appear, the drop +// is the only trace the value ever existed, so it must be per KEY, not per +// document. Each key runs alone, because the reporting loop in +// buildPropertySettings lists the three literally and a key dropped from +// that list keeps the other two warning. +// +// How this can fail: make envelopeLiftedKeys include the three keys only on +// relation documents and the page export writes the flat spelling into +// properties — a document its own Validate refuses; or drop one key from +// buildPropertySettings's off-relation reporting loop and that key's value +// vanishes in silence while the other two still warn. +func TestRelationEnvelope_NonRelationKindDropsTheDetails(t *testing.T) { + for name, tc := range map[string]struct { + storedKey string + value *types.Value + flat string + field string + }{ + "relationFormat": { + storedKey: "relationFormat", value: num(2), + flat: `"relation_format"`, field: `"format"`, + }, + "relationFormatIncludeTime": { + storedKey: "relationFormatIncludeTime", value: boolValue(true), + flat: `"relation_format_include_time"`, field: `"include_time"`, + }, + "relationFormatObjectTypes": { + storedKey: "relationFormatObjectTypes", value: strList("typeid-page"), + flat: `"relation_format_object_types"`, field: `"object_types"`, + }, + } { + t.Run(name, func(t *testing.T) { + // given + snap := trimSnapshot(map[string]*types.Value{ + tc.storedKey: tc.value, + "name": str("A page, oddly stamped"), + }) + + // when + var warns []Issue + opts := testOptions() + opts.OnWarning = func(i Issue) { warns = append(warns, i) } + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + + // then + assert.NotContains(t, string(data), tc.flat) + assert.NotContains(t, string(data), tc.field, + "a page has no §2d field to lift into") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + require.Len(t, warns, 1, "the warning is the only trace of the dropped value") + assert.Contains(t, warns[0].Message, "not a property document") + assert.Contains(t, warns[0].Message, tc.storedKey, + "the warning must name which detail was dropped") + }) + } +} + +// "text" resolves per key on the way back in, exactly as a type_properties +// entry's format does (§3): the relation's own envelope `key` disambiguates, +// so a bundled short-text relation keeps its stored format across a round +// trip even though the document never spells shorttext. +// +// How this can fail: map the envelope format name blindly through +// formatNames.value in applyPropertySettings — "text" then lands longtext on +// every relation, and the bundled `name` relation comes back reformatted. +func TestRelationEnvelope_TextFoldResolvesPerKey(t *testing.T) { + // given the bundled `name` relation, stored shorttext + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(float64(model.RelationFormat_shorttext)), + }) + snap.Key = "name" + + // when + data, err := Marshal(model.SmartBlockType_STRelation, snap, testOptions()) + require.NoError(t, err) + _, got, err := Unmarshal(data, testOptions()) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"format": "text"`, + "shorttext has no name of its own (§3)") + assert.Equal(t, float64(model.RelationFormat_shorttext), + got.Details.Fields["relationFormat"].GetNumberValue(), + "the bundled key resolves the fold — shorttext survives the trip") +} + +// Export ∘ Import is byte-stable over a relation document (§11 guarantee 2). +// +// How this can fail: any asymmetry between buildPropertySettings and +// applyPropertySettings — a field written that import drops, or one import +// rewrites into a different value — shows up as a byte diff on the second +// export. +func TestRelationEnvelope_ExportImportIsByteStable(t *testing.T) { + // given + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(100), + "relationFormatIncludeTime": boolValue(false), + "relationFormatObjectTypes": strList("bafyreitypeone"), + }) + + // when + first, err := Marshal(model.SmartBlockType_STRelation, snap, testOptions()) + require.NoError(t, err) + sbType, got, err := Unmarshal(first, testOptions()) + require.NoError(t, err) + second, err := Marshal(sbType, got, testOptions()) + require.NoError(t, err) + + // then + assert.Equal(t, string(first), string(second)) +} + +// The published schema states the format vocabulary once — +// $defs/propertyFormat — and every slot that speaks it ($2a's typeProperty, +// §6.2's dataviewProperty, the §2d envelope) references that one list, which +// must equal formatNames exactly: the schema is what an external validator +// runs, and a name in one place but not the other is a document one side +// writes and the other refuses. +// +// How this can fail: add a format name to formatNames without the schema (or +// vice versa), or point one of the three slots at a private enum again. +func TestPropertyFormatEnum_MatchesFormatNames(t *testing.T) { + // given the schema's own statement of every format the store carries + schemaNames := propertyFormatEnum() + require.NotEmpty(t, schemaNames, "the schema must publish $defs/propertyFormat") + + want := map[string]bool{} + for _, name := range formatNames.toName { + want[name] = true + } + got := map[string]bool{} + for _, name := range schemaNames { + got[name] = true + } + assert.Equal(t, want, got, "propertyFormat is total over the model enum") + + var schema struct { + Properties map[string]json.RawMessage `json:"properties"` + Defs map[string]struct { + Enum []string `json:"enum"` + Properties map[string]json.RawMessage `json:"properties"` + } `json:"$defs"` + } + require.NoError(t, json.Unmarshal(SchemaJSON(), &schema)) + + // authorableFormat is propertyFormat minus `map`, and nothing else: it + // must not drift into a second hand-maintained list. + authorable := map[string]bool{} + for _, name := range schema.Defs["authorableFormat"].Enum { + authorable[name] = true + } + delete(want, "map") + assert.Equal(t, want, authorable, + "authorableFormat is the whole vocabulary minus `map`, restated nowhere") + + // and each slot references the list it is entitled to. The shared + // propertyDefinition shape (which the §2d property_settings group is a + // reference to) states what a property IS, so it may name every format a + // store carries. The two AUTHORED slots may not invent a `map`: it + // names the shape of a hidden system relation's value, and occurs on 0 + // of 19,862 type_properties entries and 0 of 28,034 dataview property + // entries in a 38,061-document corpus. + for slot, tc := range map[string]struct { + raw json.RawMessage + ref string + }{ + "propertyDefinition.format": {schema.Defs["propertyDefinition"].Properties["format"], "propertyFormat"}, + "typeProperty.format": {schema.Defs["typeProperty"].Properties["format"], "authorableFormat"}, + "dataviewProperty.format": {schema.Defs["dataviewProperty"].Properties["format"], "authorableFormat"}, + } { + assert.Truef(t, strings.Contains(string(tc.raw), `"#/$defs/`+tc.ref+`"`), + "%s must reference %s, not restate it", slot, tc.ref) + } +} + +// A kind nothing emits is exactly the kind nobody thought to guard. Export +// writes neither `bundled_relation` nor `sub_object` — 0 of 38,061 corpus +// documents — but the schema's `kind` enum offers both beside `relation` +// with nothing marking them non-authorable, and a small model picked one: +// `{"kind":"bundled_property", …, "properties":{"format":"number"}}` +// validated clean, with no warning, and imported as a phantom property with +// no relationFormat at all. That is verbatim the §2d bug, one kind over. +// +// Two kinds are deliberately NOT in the set. `relation_option` because an +// option document is a value, not a property definition, so `format` there +// is an ordinary custom key. `sub_object` because it is deprecated: a kind +// being retired must not pick up a new obligation in a format about to +// freeze, and 0 of 38,061 corpus documents carry it either way. +// +// How this can fail: narrow isPropertyKind back to STRelation alone and the +// two side doors reopen; widen it to relation_option and the last case +// starts refusing a legitimate document. +func TestRelationEnvelope_TheSideDoorKindsAreGuardedToo(t *testing.T) { + for kind, wantValid := range map[string]bool{ + "property": false, + "bundled_property": false, + // `sub_object` is DEPRECATED and deliberately outside the set: a kind + // on its way out must not acquire a new obligation in a format about + // to freeze. It therefore accepts this shape in silence, like any + // non-relation kind — that is a decision, not an oversight, and + // re-widening it would be re-adopting a kind we are dropping. + "sub_object": true, + "property_option": true, + } { + t.Run(kind, func(t *testing.T) { + // given the shape 9 of 9 small-model attempts wrote + doc := []byte(`{"version":2,"kind":"` + kind + `","internal_key":"eh",` + + `"properties":{"name":"Estimated Hours","format":"number"}}`) + + // when + err := Validate(doc) + + // then + if wantValid { + assert.NoError(t, err, "an option document is a value, not a property definition") + return + } + require.Error(t, err, "a relation document must state its own format (§2d)") + assert.Contains(t, err.Error(), "missing property 'property_settings'") + }) + } +} + +// Told only that a member is MISSING, an author has no reason to connect +// that to the member they did write. The warning that would say so lives in +// the semantic pass, and a schema failure never reaches it — so the verdict +// that does run has to name the wrong container itself. +// +// How this can fail: drop the phantom clause from propertyFormatSlotIssue +// and the commonest authoring mistake in the corpus of small-model attempts +// gets a message that never mentions the line it is about. +func TestRelationEnvelope_TheMissingFormatVerdictNamesTheWrongContainer(t *testing.T) { + t.Run("format written into properties", func(t *testing.T) { + // given + doc := []byte(`{"version":2,"kind":"property","internal_key":"eh",` + + `"properties":{"name":"Estimated Hours","format":"number"}}`) + + // when + err := Validate(doc) + + // then + require.Error(t, err) + assert.Contains(t, err.Error(), `spells "format" inside properties`) + assert.Contains(t, err.Error(), "move that member into property_settings") + }) + + t.Run("the legacy spelling still gets its own hint", func(t *testing.T) { + // given + doc := []byte(`{"version":2,"kind":"property","internal_key":"eh",` + + `"properties":{"name":"Estimated Hours","relation_format":2}}`) + + // when + err := Validate(doc) + + // then + require.Error(t, err) + assert.Contains(t, err.Error(), "the legacy form") + }) +} + +// The phantom warning is the SEMANTIC half of the guard, and the semantic +// pass is the only place isPropertyKind is load-bearing: for a missing +// `format` the schema's own kind conditional already refuses, so a test +// there passes whether or not the Go gate agrees. This document is valid — +// it has its envelope format — so nothing but isPropertyKind decides +// whether the phantom member is reported. +// +// How this can fail: narrow isPropertyKind back to STRelation alone and the +// two side-door kinds go quiet again. +func TestRelationEnvelope_ThePhantomWarningReachesTheSideDoorKinds(t *testing.T) { + for _, kind := range []string{"property", "bundled_property"} { + t.Run(kind, func(t *testing.T) { + // given a VALID relation document that also carries the member + doc := []byte(`{"version":2,"kind":"` + kind + `","internal_key":"eh",` + + `"property_settings":{"format":"number"},` + + `"properties":{"name":"Estimated Hours","format":"number"}}`) + + // when + var warned []Issue + err := ValidateWarn(doc, func(i Issue) { warned = append(warned, i) }) + + // then + require.NoError(t, err, "the envelope format is present, so this document stands") + require.Len(t, warned, 1, "the phantom member must be reported on every kind that IS a relation") + assert.Equal(t, "/properties/format", warned[0].Path) + }) + } +} + +// A relation-shaped snapshot on a legacy kind must round-trip like any +// other. The schema requires `format` on all three relation kinds, so the +// export gate has to lift for all three or Marshal emits a document its own +// Validate rejects — which is precisely what happened when the document +// side was widened alone. +// +// Zero of these kinds come out of a live store, which is why the narrow gate +// went unnoticed. cmd/anyblockrecover reads arbitrary pb backups, where a +// relation on the legacy `sub_object` kind is the thing a recovery is for. +// +// How this can fail: narrow isPropertyDoc back to STRelation and the last +// two kinds emit no format, fail their own Validate, and lose the detail. +func TestRelationEnvelope_EveryRelationKindRoundTripsLossless(t *testing.T) { + for _, sb := range []model.SmartBlockType{ + model.SmartBlockType_STRelation, + model.SmartBlockType_BundledRelation, + } { + t.Run(sb.String(), func(t *testing.T) { + // given + snap := relationSnapshot(map[string]*types.Value{ + "relationFormat": num(4), + "relationFormatIncludeTime": boolValue(true), + }) + + // when + data, err := Marshal(sb, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (§11 I1)") + _, back, err := Unmarshal(data, Options{}) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"format": "date"`) + assert.Equal(t, num(4).String(), + back.GetDetails().GetFields()["relationFormat"].String(), + "the definition survives on every kind that IS a relation") + }) + } +} + +// `map` is not an authorable format. It names the shape of a hidden system +// relation's value — `templatePlaceholders` is the only carrier, 72 +// production documents — and no type or view declares one: 0 of 19,862 +// type_properties entries and 0 of 28,034 dataview property entries in a +// 38,061-document corpus. +// +// The relation envelope must still be able to state it, or those 72 +// documents cannot say what they define and stop exporting (§2d). +// +// How this can fail: point the authored slots back at propertyFormat and a +// model can declare a property whose values nothing but the client writes. +func TestPropertyFormat_MapIsNotAuthorable(t *testing.T) { + t.Run("a relation document may state it", func(t *testing.T) { + doc := []byte(`{"version":2,"kind":"property","internal_key":"templatePlaceholders", + "property_settings":{"format":"map"}, + "properties":{"name":"Template Placeholders"}}`) + assert.NoError(t, Validate(doc), "the only carrier of `map` must keep exporting") + }) + + t.Run("a type may not declare it", func(t *testing.T) { + doc := []byte(`{"version":2,"kind":"object_type","internal_key":"task","properties":{"name":"Task"}, + "type_settings":{"property_definitions": [{"property":"placeholders","format":"map"}]}}`) + require.Error(t, Validate(doc), "an authored property may not invent a map") + }) + + t.Run("a dataview may not declare it", func(t *testing.T) { + doc := []byte(`{"version":2,"blocks":[{"type":"dataview", + "properties":[{"property":"placeholders","format":"map"}], + "views":[{"name":"All"}]}]}`) + require.Error(t, Validate(doc), "a view may not invent a map either") + }) + + t.Run("the authored slots still take every other format", func(t *testing.T) { + for _, f := range []string{"text", "number", "date", "select", "objects", "properties"} { + doc := []byte(`{"version":2,"kind":"object_type","internal_key":"task","properties":{"name":"Task"}, + "type_settings":{"property_definitions": [{"property":"p","format":"` + f + `"}]}}`) + assert.NoErrorf(t, Validate(doc), "%q is authorable", f) + } + }) +} diff --git a/pkg/lib/anyblockjson/review_fixes_test.go b/pkg/lib/anyblockjson/review_fixes_test.go new file mode 100644 index 0000000000..8d024db84b --- /dev/null +++ b/pkg/lib/anyblockjson/review_fixes_test.go @@ -0,0 +1,323 @@ +package anyblockjson + +// Regression tests for the confirmed findings of the review pass. + +import ( + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// Finding 1: a same-param merge that extends an accepted range must re-run +// overlap resolution, or a same-type different-param overlap survives and +// the second export resolves it differently (byte-stability break). +func TestInline_MergeExtensionReresolvesOverlaps(t *testing.T) { + text := strings.Repeat("abcde", 6) // 30 chars + marks := []*model.BlockContentTextMark{ + mark(mLink, 0, 5, "http://p1"), + mark(mLink, 3, 30, "http://p2"), + mark(mLink, 4, 20, "http://p1"), + } + md1 := renderInline(text, marks) + text1, marks1, err := parseInline(md1) + require.NoError(t, err) + md2 := renderInline(text1, marks1) + require.Equal(t, md1, md2, "must be byte-stable") + // the resolved decomposition: p1 wins [0,20), p2 truncated to [20,30) + assert.Equal(t, "[abcdeabcdeabcdeabcde](http://p1)[abcdeabcde](http://p2)", md1) +} + +// Finding 5: a bare link destination starting with '<' must not be misread +// as the angle-wrapped form on re-parse. +func TestInline_DestLeadingAngle(t *testing.T) { + marks := []*model.BlockContentTextMark{mark(mLink, 0, 2, "y")} + md := renderInline("ab", marks) + text, parsed, err := parseInline(md) + require.NoError(t, err) + assert.Equal(t, "ab", text) + require.Len(t, parsed, 1) + assert.Equal(t, "y", parsed[0].Param) + assert.Equal(t, md, renderInline(text, parsed)) +} + +// Finding 9: brackets/backticks inside tag attribute values must be +// entity-encoded, or the link-label scan derails when the tag sits inside a +// link label. +func TestInline_BracketInAttrInsideLabel(t *testing.T) { + marks := []*model.BlockContentTextMark{ + mark(mLink, 0, 2, "http://u"), + mark(mColor, 0, 2, "a]b"), + } + md := renderInline("hi", marks) + text, parsed, err := parseInline(md) + require.NoError(t, err) + assert.Equal(t, "hi", text) + require.Len(t, parsed, 2) + assert.Equal(t, "http://u", parsed[0].Param) + assert.Equal(t, "a]b", parsed[1].Param) + assert.Equal(t, md, renderInline(text, parsed)) +} + +// Finding 2: verbatim property passthrough (structs, nulls inside lists) +// must produce schema-valid canonical output and round-trip. +func TestExport_VerbatimPropertyShapes(t *testing.T) { + structVal := &types.Value{Kind: &types.Value_StructValue{StructValue: fields(map[string]*types.Value{ + "a": num(1), + })}} + listWithNull := &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: []*types.Value{ + str("x"), {Kind: &types.Value_NullValue{}}, + }}}} + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{ + "id": str("obj1"), + "customWeird": structVal, + "customList": listWithNull, + }), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data), "canonical export must pass its own schema") + _, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + second, err := Marshal(model.SmartBlockType_Page, snap2, Options{}) + require.NoError(t, err) + assert.Equal(t, string(data), string(second)) +} + +// Finding 3: a sort with an empty property key is dropped instead of +// emitting a document that fails the schema's required "property"; an empty +// filter group is a no-op and is dropped too. +func TestExport_EmptyKeySortSkipped(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"dv"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dv", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{{ + Id: "v1", + Sorts: []*model.BlockContentDataviewSort{{RelationKey: ""}}, + Filters: []*model.BlockContentDataviewFilter{ + {Operator: model.BlockContentDataviewFilter_And}, + }, + }}, + }}}, + }, + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.NotContains(t, string(data), `"sorts"`) + assert.NotContains(t, string(data), `"filters"`) +} + +// Finding 4: nil inner content messages are proto-equivalent to empty ones +// and must not panic Marshal. +func TestExport_NilInnerContent(t *testing.T) { + contents := []model.IsBlockContent{ + &model.BlockContentOfText{}, + &model.BlockContentOfFile{}, + &model.BlockContentOfBookmark{}, + &model.BlockContentOfLink{}, + &model.BlockContentOfDiv{}, + &model.BlockContentOfLayout{}, + &model.BlockContentOfLatex{}, + &model.BlockContentOfRelation{}, + &model.BlockContentOfDataview{}, + &model.BlockContentOfWidget{}, + &model.BlockContentOfIcon{}, + &model.BlockContentOfTableRow{}, + } + for _, c := range contents { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"b1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "b1", Content: c}, + }, + } + require.NotPanics(t, func() { + _, _ = Marshal(model.SmartBlockType_Page, snap, Options{}) + }, "content %T", c) + } +} + +// Finding 6, revisited under raw names: a Page whose type key is `template` +// once needed an explicit kind, because the type SPELLED itself `template` — +// the byte shape that meant a template before `kind` existed. The type spells +// its display name "Template" now, which is not that byte shape, so the kind +// is derivable again and stays omitted — and the round trip must still come +// home a Page. The emission guard survives for the one spelling that still +// needs it: a writer whose vocabulary yields the raw term `template` (a +// package-only export of a custom type stored-keyed so, or a vocabulary that +// answers the key itself), which the second arm pins. +func TestExport_PageWithTemplateTypeKeepsKind(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + ObjectTypes: []string{"ot-template"}, + } + + t.Run("the display-name spelling derives its kind", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"type": "Template"`) + assert.NotContains(t, string(data), `"kind"`, "Page is derivable — the term is not the legacy byte shape") + require.NoError(t, Validate(data), "I1") + sbType, _, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + }) + + t.Run("the raw term `template` still forces the kind", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: verbatimKeys{}}) + require.NoError(t, err) + assert.Contains(t, string(data), `"type": "template"`) + assert.Contains(t, string(data), `"kind": "page"`, + "a kindless type:template document is the shape that used to mean a template (export.go's emission rule)") + require.NoError(t, Validate(data), "I1") + sbType, _, err := Unmarshal(data, Options{GenerateId: seqIds("g"), Keys: verbatimKeys{}}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + }) +} + +// verbatimKeys is the minimum conforming vocabulary: every key spells +// itself. +type verbatimKeys struct{} + +func (verbatimKeys) PropertySlug(key string) string { return key } +func (verbatimKeys) PropertyKey(s string) (string, bool) { return s, false } +func (verbatimKeys) TypeSlug(key string) string { return key } +func (verbatimKeys) TypeKey(s string) (string, bool) { return s, false } + +// Finding 7: a stray properties.id / properties.type in the document must not +// clobber the envelope-lifted details. It used to be dropped in silence, which +// left an author wondering why the id they wrote had no effect; since the +// pre-freeze pass on property-key admission (Tier 1 #5) it is refused by name, +// and the envelope stays the only place either one is set. +func TestImport_PropertiesIdDoesNotLeak(t *testing.T) { + doc := `{"version": 2, "id": "realid", "properties": {"id": "fakeid", "type": "faketype", "name": "N"}}` + err := Validate([]byte(doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), `/properties/id: "id" belongs in the envelope`) + assert.Contains(t, err.Error(), `/properties/type: "type" belongs in the envelope`) + + _, _, err = Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, err) + + // and the envelope keeps working on its own + _, snap, err := Unmarshal([]byte(`{"version": 2, "id": "realid", "properties": {"name": "N"}}`), + Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "realid", snap.Details.Fields["id"].GetStringValue()) + assert.Nil(t, snap.Details.Fields["type"]) + assert.Equal(t, "N", snap.Details.Fields["name"].GetStringValue()) +} + +// Finding 8: tables nested inside table cells join the id-uniqueness domain +// and get their inline text checked. +// A table inside a table cell is rejected at the schema level: cells use the +// non-recursive cellBlock definition — the guarantee that keeps the whole +// schema free of block recursion (§12). Cell arrays get the same treatment, +// and their inline text still reaches the markup checks. +func TestValidate_NestedTableInCell(t *testing.T) { + nestedTable := `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [ + {"type": "table", "columns": [{"id": "c2"}], "rows": []} + ]}]} + ]}` + err := Validate([]byte(nestedTable)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/blocks/0/rows/0/cells/0") + + dupIdInCellArray := `{"version": 2, "blocks": [ + {"id": "x1", "type": "paragraph"}, + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [[ + {"type": "toggle", "text": "cell"}, + {"indent": 1, "id": "x1", "type": "paragraph", "text": "dup"} + ]]}]} + ]}` + err = Validate([]byte(dupIdInCellArray)) + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate id") + + badInline := `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [[ + {"type": "toggle", "text": "cell"}, + {"indent": 1, "type": "paragraph", "text": "unclosed"} + ]]}]} + ]}` + err = Validate([]byte(badInline)) + require.Error(t, err) + assert.Contains(t, err.Error(), "inline markup") +} + +// Finding 11: a non-list "objects" value in the internal store stays in +// store instead of being dropped. +func TestExport_NonListObjectsStaysInStore(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Collections: fields(map[string]*types.Value{"objects": str("notalist")}), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + s := string(data) + assert.NotContains(t, s, `"items"`) + assert.Contains(t, s, `"objects": "notalist"`) + _, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "notalist", snap2.Collections.Fields["objects"].GetStringValue()) +} + +// Finding 12: integer-valued float versions are accepted (JSON Schema +// numeric equality). +func TestValidate_FloatVersion(t *testing.T) { + require.NoError(t, Validate([]byte(`{"version": 2.0}`))) + require.Error(t, Validate([]byte(`{"version": 2.5}`))) +} + +// Out-of-range enum values are omitted (or error for the type discriminator) +// instead of emitting schema-invalid empty strings. +func TestExport_OutOfRangeEnums(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"dv"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "dv", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{{ + Id: "v1", + Type: model.BlockContentDataviewViewType(99), + Filters: []*model.BlockContentDataviewFilter{{ + RelationKey: "k", + Condition: model.BlockContentDataviewFilterCondition(99), + QuickOption: model.BlockContentDataviewFilterQuickOption(99), + }}, + }}, + }}}, + }, + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(data), "out-of-range enums must not produce invalid output") + + // unknown text styles are an export error, not silent mangling + snap2 := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("obj1")}), + Blocks: []*model.Block{ + {Id: "obj1", ChildrenIds: []string{"t"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "t", Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Style: model.BlockContentTextStyle(99), Text: "x", + }}}, + }, + } + _, err = Marshal(model.SmartBlockType_Page, snap2, Options{}) + require.Error(t, err) +} diff --git a/pkg/lib/anyblockjson/roundtrip_test.go b/pkg/lib/anyblockjson/roundtrip_test.go new file mode 100644 index 0000000000..23864da10d --- /dev/null +++ b/pkg/lib/anyblockjson/roundtrip_test.go @@ -0,0 +1,661 @@ +package anyblockjson + +import ( + "fmt" + "math/rand" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// +// ---- helpers ---- +// + +func str(s string) *types.Value { + return &types.Value{Kind: &types.Value_StringValue{StringValue: s}} +} + +func num(f float64) *types.Value { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: f}} +} + +func boolean(b bool) *types.Value { + return &types.Value{Kind: &types.Value_BoolValue{BoolValue: b}} +} + +func strList(ss ...string) *types.Value { + vals := make([]*types.Value, 0, len(ss)) + for _, s := range ss { + vals = append(vals, str(s)) + } + return &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: vals}}} +} + +func fields(kv map[string]*types.Value) *types.Struct { + return &types.Struct{Fields: kv} +} + +func textBlock(id string, style model.BlockContentTextStyle, text string, marks ...*model.BlockContentTextMark) *model.Block { + t := &model.BlockContentText{Style: style, Text: text} + if len(marks) > 0 { + t.Marks = &model.BlockContentTextMarks{Marks: marks} + } + return &model.Block{Id: id, Content: &model.BlockContentOfText{Text: t}} +} + +// seqIds returns a deterministic id generator for import tests. +func seqIds(prefix string) func() string { + n := 0 + return func() string { + n++ + return fmt.Sprintf("%s%d", prefix, n) + } +} + +type testOptionResolver struct { + idToName map[string]string + nameToId map[string]string +} + +func (r *testOptionResolver) OptionName(_ domain.RelationKey, id string) (string, bool) { + n, ok := r.idToName[id] + return n, ok +} + +func (r *testOptionResolver) OptionId(_ domain.RelationKey, name string) (string, bool) { + id, ok := r.nameToId[name] + return id, ok +} + +var testResolver = &testOptionResolver{ + idToName: map[string]string{"opt1": "In progress", "opt2": "Done"}, + nameToId: map[string]string{"In progress": "opt1", "Done": "opt2"}, +} + +func testFormatResolver(key domain.RelationKey) (model.RelationFormat, bool) { + switch key { + case "customStatus": + return model.RelationFormat_status, true + case "customDate": + return model.RelationFormat_date, true + } + return 0, false +} + +func testOptions() Options { + return Options{ + ResolveFormat: testFormatResolver, + ResolveOptions: testResolver, + } +} + +// richSnapshot builds a snapshot exercising every §5 block family plus +// structural blocks, properties, and store content. +func richSnapshot() *model.SmartBlockSnapshotBase { + objectId := "bafyreiobject" + blocks := []*model.Block{ + { + Id: objectId, + ChildrenIds: []string{ + "header", "b1", "b2", "b3", "b5", "b6", "b7", "b8", "b9", + "row1", "table1", "b10", "dv1", + }, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }, + // structural: dropped on export (§7) + {Id: "header", ChildrenIds: []string{"title", "descr", "featured"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Header}}}, + textBlock("title", model.BlockContentText_Title, "Project Phoenix"), + textBlock("descr", model.BlockContentText_Description, "The subtitle"), + {Id: "featured", Content: &model.BlockContentOfFeaturedRelations{FeaturedRelations: &model.BlockContentFeaturedRelations{}}}, + + textBlock("b1", model.BlockContentText_Header2, "Goals"), + textBlock("b2", model.BlockContentText_Paragraph, "Ship the new export with Roman", + mark(mBold, 9, 19, ""), mark(mMention, 25, 30, "bafyreiroman")), + textBlock("b3", model.BlockContentText_Marked, "Nested item"), + textBlock("b5", model.BlockContentText_Checkbox, "Draft spec"), + {Id: "b6", Fields: fields(map[string]*types.Value{"lang": str("go")}), + Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Style: model.BlockContentText_Code, Text: "func main() {\n\tprintln(\"hi\")\n}", + }}}, + {Id: "b7", Content: &model.BlockContentOfDiv{Div: &model.BlockContentDiv{Style: model.BlockContentDiv_Dots}}}, + {Id: "b8", Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + Type: model.BlockContentFile_Image, TargetObjectId: "bafyreiimage", + Name: "cat.png", Mime: "image/png", Size_: 2048, + State: model.BlockContentFile_Done, AddedAt: 1751791445, + }}}, + {Id: "b9", Content: &model.BlockContentOfBookmark{Bookmark: &model.BlockContentBookmark{ + Url: "https://anytype.io", TargetObjectId: "bafyreibookmark", + State: model.BlockContentBookmark_Done, + }}}, + {Id: "row1", ChildrenIds: []string{"col1", "col2"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Row}}}, + {Id: "col1", ChildrenIds: []string{"b11"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Column}}}, + {Id: "col2", ChildrenIds: []string{"b12"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Column}}}, + textBlock("b11", model.BlockContentText_Paragraph, "left"), + textBlock("b12", model.BlockContentText_Paragraph, "right"), + + // table subtree (§6.1) + {Id: "table1", ChildrenIds: []string{"tcols", "trows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "tcols", ChildrenIds: []string{"c1", "c2"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: "trows", ChildrenIds: []string{"r2", "r1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + {Id: "c1", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + {Id: "c2", Fields: fields(map[string]*types.Value{"width": num(120)}), + Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + // r2 stored before r1, but r1 is the header: export reorders + {Id: "r1", ChildrenIds: []string{"r1-c1", "r1-c2"}, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{IsHeader: true}}}, + {Id: "r2", ChildrenIds: []string{"r2-c2"}, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + textBlock("r1-c1", model.BlockContentText_Paragraph, "Name"), + textBlock("r1-c2", model.BlockContentText_Paragraph, "Status"), + {Id: "r2-c2", Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Style: model.BlockContentText_Checkbox, Text: "done", Checked: true, + }}}, + + {Id: "b10", Content: &model.BlockContentOfLatex{Latex: &model.BlockContentLatex{ + Processor: model.BlockContentLatex_Mermaid, Text: "graph TD; A-->B", + }}}, + + // dataview (§6.2) + {Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + TargetObjectId: "bafyreitasks", + RelationLinks: []*model.RelationLink{ + {Key: "name", Format: model.RelationFormat_shorttext}, + {Key: "customStatus", Format: model.RelationFormat_status}, + {Key: "dueDate", Format: model.RelationFormat_date}, + }, + Views: []*model.BlockContentDataviewView{{ + Id: "v1", + Type: model.BlockContentDataviewView_Kanban, + Name: "By status", + GroupRelationKey: "customStatus", + Sorts: []*model.BlockContentDataviewSort{{ + RelationKey: "dueDate", + Format: model.RelationFormat_date, + EmptyPlacement: model.BlockContentDataviewSort_End, + Id: "s1", + }}, + Filters: []*model.BlockContentDataviewFilter{ + { + Id: "f1", + RelationKey: "dueDate", + Condition: model.BlockContentDataviewFilter_Less, + QuickOption: model.BlockContentDataviewFilter_CurrentWeek, + Format: model.RelationFormat_date, + }, + { + Operator: model.BlockContentDataviewFilter_Or, + NestedFilters: []*model.BlockContentDataviewFilter{ + { + Id: "f2", + RelationKey: "customStatus", + Condition: model.BlockContentDataviewFilter_In, + Value: strList("opt1", "opt2"), + Format: model.RelationFormat_status, + }, + { + Id: "f3", + RelationKey: "done", + Condition: model.BlockContentDataviewFilter_Empty, + Value: boolean(false), + }, + }, + }, + }, + Relations: []*model.BlockContentDataviewRelation{ + {Key: "name", IsVisible: true}, + {Key: "dueDate", IsVisible: false, Width: 120, + Formula: model.BlockContentDataviewRelation_CountDistinct, + Align: model.Block_AlignRight}, + }, + }}, + GroupOrders: []*model.BlockContentDataviewGroupOrder{{ + ViewId: "v1", + ViewGroups: []*model.BlockContentDataviewViewGroup{ + {GroupId: "g2", Index: 1, Hidden: true}, + {GroupId: "g1", Index: 0, BackgroundColor: "red"}, + }, + }}, + ObjectOrders: []*model.BlockContentDataviewObjectOrder{{ + ViewId: "v1", GroupId: "g1", ObjectIds: []string{"bafyreitask1"}, + }}, + }}}, + } + return &model.SmartBlockSnapshotBase{ + Blocks: blocks, + Details: fields(map[string]*types.Value{ + "id": str("bafyreiobject"), + "name": str("Project Phoenix"), + "description": str("The subtitle"), + "iconEmoji": str("🔥"), + "type": str("bafyreitypepage"), + "createdDate": num(1751791445), + "lastOpenedDate": num(1751791445), // local: stripped + "customStatus": str("opt1"), + "customDate": num(1751791445), + "assignee": strList("bafyreiroman"), + }), + ObjectTypes: []string{"ot-page"}, + } +} + +func TestMarshal_ProducesValidDocument(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, richSnapshot(), testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data)) +} + +// TestRoundTrip_ByteStable checks §11.2: Export ∘ Import is idempotent and +// byte-stable. +func TestRoundTrip_ByteStable(t *testing.T) { + opts := testOptions() + first, err := Marshal(model.SmartBlockType_Page, richSnapshot(), opts) + require.NoError(t, err) + require.NoError(t, Validate(first)) + + impOpts := testOptions() + impOpts.GenerateId = seqIds("gen") + sbType, snap, err := Unmarshal(first, impOpts) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + + second, err := Marshal(sbType, snap, opts) + require.NoError(t, err) + assert.Equal(t, string(first), string(second), "Export ∘ Import must be byte-stable") + + impOpts.GenerateId = seqIds("gen2") + sbType2, snap2, err := Unmarshal(second, impOpts) + require.NoError(t, err) + third, err := Marshal(sbType2, snap2, opts) + require.NoError(t, err) + assert.Equal(t, string(second), string(third)) +} + +// TestRoundTrip_State spot-checks Import(Export(S)) ≡ N(S) on the snapshot +// level (§11.1). +func TestRoundTrip_State(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, richSnapshot(), testOptions()) + require.NoError(t, err) + impOpts := testOptions() + impOpts.GenerateId = seqIds("gen") + sbType, snap, err := Unmarshal(data, impOpts) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + assert.Equal(t, []string{"ot-page"}, snap.ObjectTypes) + + byId := map[string]*model.Block{} + for _, b := range snap.Blocks { + byId[b.Id] = b + } + + // root block regenerated with the object id; structural blocks stay absent + root := byId["bafyreiobject"] + require.NotNil(t, root) + for _, id := range root.ChildrenIds { + require.NotNil(t, byId[id], "child %s missing", id) + } + + // marks: offsets and resolved mention target (§8) + b2 := byId["b2"].Content.(*model.BlockContentOfText).Text + require.NotNil(t, b2.Marks) + want := []*model.BlockContentTextMark{ + mark(mBold, 9, 19, ""), + mark(mMention, 25, 30, "bafyreiroman"), + } + assert.Equal(t, want, b2.Marks.Marks) + + // code: language back into fields.lang, literal text (§5.1, §8.4) + b6 := byId["b6"] + assert.Equal(t, "go", b6.Fields.Fields["lang"].GetStringValue()) + assert.Equal(t, "func main() {\n\tprintln(\"hi\")\n}", b6.Content.(*model.BlockContentOfText).Text.Text) + + // table subtree rebuilt with derived cell ids, header row first (§6.1) + table := byId["table1"] + require.Len(t, table.ChildrenIds, 2) + colsW, rowsW := byId[table.ChildrenIds[0]], byId[table.ChildrenIds[1]] + assert.Equal(t, model.BlockContentLayout_TableColumns, colsW.Content.(*model.BlockContentOfLayout).Layout.Style) + assert.Equal(t, []string{"c1", "c2"}, colsW.ChildrenIds) + assert.Equal(t, []string{"r1", "r2"}, rowsW.ChildrenIds) + assert.True(t, byId["r1"].Content.(*model.BlockContentOfTableRow).TableRow.IsHeader) + assert.Equal(t, float64(120), byId["c2"].Fields.Fields["width"].GetNumberValue()) + require.NotNil(t, byId["r1-c1"]) + assert.Equal(t, "Name", byId["r1-c1"].Content.(*model.BlockContentOfText).Text.Text) + // sparse cell r2-c1 stays absent + assert.Nil(t, byId["r2-c1"]) + assert.True(t, byId["r2-c2"].Content.(*model.BlockContentOfText).Text.Checked) + + // dataview: cached formats rehydrated, option names resolved back (§6.2) + dv := byId["dv1"].Content.(*model.BlockContentOfDataview).Dataview + view := dv.Views[0] + assert.Equal(t, model.RelationFormat_date, view.Sorts[0].Format) + group := view.Filters[1] + assert.Equal(t, model.BlockContentDataviewFilter_Or, group.Operator) + assert.Equal(t, strList("opt1", "opt2"), group.NestedFilters[0].Value) + assert.Equal(t, model.RelationFormat_status, group.NestedFilters[0].Format) + // value dropped on presence-only conditions (§11) + assert.Nil(t, group.NestedFilters[1].Value) + require.Len(t, dv.GroupOrders, 1) + assert.Equal(t, "g1", dv.GroupOrders[0].ViewGroups[0].GroupId) + assert.Equal(t, int32(0), dv.GroupOrders[0].ViewGroups[0].Index) + + // details: dates back to unix seconds, select names to option ids + assert.Equal(t, float64(1751791445), snap.Details.Fields["createdDate"].GetNumberValue()) + assert.Equal(t, float64(1751791445), snap.Details.Fields["customDate"].GetNumberValue()) + assert.Equal(t, strList("opt1"), snap.Details.Fields["customStatus"]) + assert.Equal(t, strList("bafyreiroman"), snap.Details.Fields["assignee"]) + // stripped local property does not resurrect + assert.Nil(t, snap.Details.Fields["lastOpenedDate"]) +} + +func TestOmitIds(t *testing.T) { + opts := testOptions() + opts.OmitIds = true + data, err := Marshal(model.SmartBlockType_Page, richSnapshot(), opts) + require.NoError(t, err) + require.NoError(t, Validate(data)) + + s := string(data) + // no block/row/column/view/sort/filter ids; id-dependent view state gone + assert.NotContains(t, s, `"id": "b1"`) + assert.NotContains(t, s, `"id": "r1"`) + assert.NotContains(t, s, `"id": "c1"`) + assert.NotContains(t, s, `"id": "v1"`) + assert.NotContains(t, s, `"id": "s1"`) + assert.NotContains(t, s, `"id": "f1"`) + assert.NotContains(t, s, `"groups"`) + assert.NotContains(t, s, `"object_orders"`) + // the envelope id stays (§9) + assert.Contains(t, s, `"id": "bafyreiobject"`) + + impOpts := testOptions() + impOpts.GenerateId = seqIds("gen") + _, snap, err := Unmarshal(data, impOpts) + require.NoError(t, err) + assert.NotEmpty(t, snap.Blocks) +} + +func TestCompactIds(t *testing.T) { + opts := testOptions() + opts.CompactIds = true + data, err := Marshal(model.SmartBlockType_Page, richSnapshot(), opts) + require.NoError(t, err) + require.NoError(t, Validate(data)) + s := string(data) + + // block ids relabel to their 5-char suffix — the one compaction left, and + // it carries no legend (§9a) + assert.Contains(t, s, `"id": "b1"`, "short authored ids serve as themselves") + // object references are written IN FULL, at every use site, on every shape + assert.Contains(t, s, ``) + assert.Contains(t, s, `"bafyreiroman"`, "the assignee property value too") + // no object-id legend, stated as the entries the deleted one would have + // written — a bare absence assertion on the field name would hold no + // matter what the export did, which is the trap this replaces (§9a) + assert.NotContains(t, s, `"roman": "bafyreiroman"`) + assert.NotContains(t, s, `"tasks": "bafyreitasks"`) + assert.NotContains(t, s, `"image": "bafyreiimage"`) + // and no short label survives at a use site either + assert.NotContains(t, s, `"object_id": "image"`) + assert.NotContains(t, s, `"object_id": "tasks"`) + assert.NotContains(t, s, `bafyreitypepage`, "stripped properties leave no legend entries either") + // the envelope id, like every other object id (§9a) + assert.Contains(t, s, `"id": "bafyreiobject"`) + + // importing it back keeps the mention on the object it named — by + // carrying the id, not by resolving a label + impOpts := testOptions() + impOpts.GenerateId = seqIds("gen") + _, snap, err := Unmarshal(data, impOpts) + require.NoError(t, err) + var mention string + for _, b := range snap.Blocks { + if txt, ok := b.Content.(*model.BlockContentOfText); ok && txt.Text.Marks != nil { + for _, m := range txt.Text.Marks.Marks { + if m.Type == model.BlockContentTextMark_Mention { + mention = m.Param + } + } + } + } + assert.Equal(t, "bafyreiroman", mention) +} + +func TestEnvelope_Variants(t *testing.T) { + t.Run("template", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("tpl1")}), + ObjectTypes: []string{"ot-template", "ot-task"}, + } + data, err := Marshal(model.SmartBlockType_Template, snap, Options{}) + require.NoError(t, err) + s := string(data) + assert.Contains(t, s, `"type": "Template"`) + assert.Contains(t, s, `"template_for": "Task"`) + // A template says so, always. `kind` used to be omitted here as + // derivable from the type term, which is what made the type term + // carry two meanings at once (§2, v0.22): the cost is ~21 bytes on a + // template document, and what it buys is that a template whose types + // do NOT begin with the template key can express its target at all. + assert.Contains(t, s, `"kind": "template"`) + + sbType, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Template, sbType) + assert.Equal(t, []string{"ot-template", "ot-task"}, snap2.ObjectTypes) + }) + + t.Run("explicit kind", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("p1")}), + } + data, err := Marshal(model.SmartBlockType_ProfilePage, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"kind": "profile_page"`) + sbType, _, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_ProfilePage, sbType) + }) + + t.Run("collection items and store", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("coll1")}), + ObjectTypes: []string{"ot-collection"}, + Collections: fields(map[string]*types.Value{ + "objects": strList("bafyreitask1", "bafyreitask2"), + "other": str("kept"), + }), + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + s := string(data) + assert.Contains(t, s, `"items"`) + assert.Contains(t, s, `"bafyreitask1"`) + assert.Contains(t, s, `"store"`) + + _, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, strList("bafyreitask1", "bafyreitask2"), snap2.Collections.Fields["objects"]) + assert.Equal(t, str("kept"), snap2.Collections.Fields["other"]) + + second, err := Marshal(model.SmartBlockType_Page, snap2, Options{}) + require.NoError(t, err) + assert.Equal(t, string(data), string(second)) + }) + + t.Run("root escape hatch", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{"id": str("p2")}), + Blocks: []*model.Block{{ + Id: "p2", + BackgroundColor: "grey", + Fields: fields(map[string]*types.Value{"custom": str("x")}), + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + } + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"root"`) + _, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "grey", snap2.Blocks[0].BackgroundColor) + assert.Equal(t, "x", snap2.Blocks[0].Fields.Fields["custom"].GetStringValue()) + }) +} + +func TestImport_Aliases(t *testing.T) { + doc := `{"version": 2, "blocks": [ + {"type": "heading_4", "text": "deep"}, + {"type": "header_4", "text": "deeper"}, + {"type": "equation", "text": "E=mc^2"}, + {"type": "embed", "processor": "youtube", "url": "https://youtu.be/x"} + ]}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + blocks := snap.Blocks[1:] // skip root + assert.Equal(t, model.BlockContentText_Header3, blocks[0].Content.(*model.BlockContentOfText).Text.Style) + assert.Equal(t, model.BlockContentText_Header3, blocks[1].Content.(*model.BlockContentOfText).Text.Style) + eq := blocks[2].Content.(*model.BlockContentOfLatex).Latex + assert.Equal(t, model.BlockContentLatex_Latex, eq.Processor) + assert.Equal(t, "E=mc^2", eq.Text) + yt := blocks[3].Content.(*model.BlockContentOfLatex).Latex + assert.Equal(t, model.BlockContentLatex_Youtube, yt.Processor) + assert.Equal(t, "https://youtu.be/x", yt.Text) +} + +func TestImport_TitleAbsorption(t *testing.T) { + doc := `{"version": 2, "blocks": [ + {"type": "title", "text": "My **Title**"}, + {"type": "description", "text": "Sub"}, + {"type": "featured_properties"}, + {"type": "paragraph", "text": "body"} + ]}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "My Title", snap.Details.Fields["name"].GetStringValue()) + assert.Equal(t, "Sub", snap.Details.Fields["description"].GetStringValue()) + // only the paragraph survives under the root + require.Len(t, snap.Blocks, 2) + assert.Equal(t, "body", snap.Blocks[1].Content.(*model.BlockContentOfText).Text.Text) + + // when the property is already set, the block is simply dropped + doc2 := `{"version": 2, "properties": {"name": "Kept"}, "blocks": [ + {"type": "title", "text": "Ignored"} + ]}` + _, snap2, err := Unmarshal([]byte(doc2), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "Kept", snap2.Details.Fields["name"].GetStringValue()) +} + +// TestExplicitIndentZero: an explicit "indent": 0 is accepted on input and +// canonicalized away on re-export (§4 omit-default canon). +func TestExplicitIndentZero(t *testing.T) { + doc := `{"version": 2, "blocks": [{"indent": 0, "id": "a", "type": "paragraph", "text": "x"}]}` + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + out, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err) + assert.NotContains(t, string(out), `"indent"`) +} + +// TestGeneratedDocs_ByteStable: for generated valid documents J, +// Export(Import(J)) is canonical and re-import/re-export is byte-identical +// (§11.2). +func TestGeneratedDocs_ByteStable(t *testing.T) { + rnd := rand.New(rand.NewSource(7)) + texts := []string{ + "plain", "**bold** and *it*", "`code` span", "a\\*b", "😀 astral 𝒜", + "under", "~~gone~~", "x P", + "[link](https://x.io)", "line\nbreak", "_alias_", "<tag>", + } + blockGens := []func(i int) string{ + func(i int) string { + return fmt.Sprintf(`{"type": "paragraph", "text": %q}`, texts[i%len(texts)]) + }, + func(i int) string { + return fmt.Sprintf(`{"type": "checkbox", "checked": %v, "text": %q}`, i%2 == 0, texts[i%len(texts)]) + }, + func(i int) string { + return fmt.Sprintf(`{"type": "heading_%d", "text": "h"},{"indent": 1, "type": "paragraph", "text": %q}`, + 1+i%3, texts[i%len(texts)]) + }, + func(i int) string { + // a deep chain: covers the real-world ~6-level maximum and beyond + depth := 4 + i%8 + parts := []string{fmt.Sprintf(`{"type": "bulleted_list_item", "text": %q}`, texts[i%len(texts)])} + for d := 1; d <= depth; d++ { + parts = append(parts, fmt.Sprintf(`{"indent": %d, "type": "bulleted_list_item", "text": %q}`, + d, texts[(i+d)%len(texts)])) + } + return strings.Join(parts, ",") + }, + func(i int) string { + // mixed wide/deep: siblings appearing after a pop back up + return fmt.Sprintf(`{"type": "toggle", "text": "t"},`+ + `{"indent": 1, "type": "paragraph", "text": %q},`+ + `{"indent": 2, "type": "paragraph", "text": "deep"},`+ + `{"indent": 1, "type": "paragraph", "text": "sibling"},`+ + `{"type": "paragraph", "text": "top"}`, texts[i%len(texts)]) + }, + func(i int) string { + return fmt.Sprintf(`{"type": "table", + "columns": [{"id": "ca%d"}, {"id": "cb%d"}], + "rows": [ + {"id": "ra%d", "is_header": true, "cells": [%q]}, + {"id": "rb%d", "cells": [null, %q]} + ]}`, i, i, i, texts[i%len(texts)], i, texts[(i+1)%len(texts)]) + }, + func(i int) string { + return fmt.Sprintf(`{"type": "dataview", "object_id": "bafyreiset%d", + "properties": [{"property": "name", "format": "text"}], + "views": [{"type": "list", "name": "v", + "sorts": [{"property": "name", "direction": "desc"}], + "filters": [{"property": "name", "condition": "contains", "value": "x"}]}]}`, i) + }, + func(i int) string { + return fmt.Sprintf(`{"type": "code", "language": "go", "text": %q}`, texts[i%len(texts)]) + }, + func(i int) string { + return fmt.Sprintf(`{"type": "callout", "icon": {"format": "emoji", "emoji": "💡"}, "text": %q}`, + texts[i%len(texts)]) + }, + } + for i := 0; i < 300; i++ { + var blocks []string + for n := 1 + rnd.Intn(5); n > 0; n-- { + blocks = append(blocks, blockGens[rnd.Intn(len(blockGens))](rnd.Intn(1000))) + } + doc := fmt.Sprintf(`{"version": 2, "properties": {"name": "Doc %d"}, "blocks": [%s]}`, + i, strings.Join(blocks, ",")) + require.NoError(t, Validate([]byte(doc)), "case %d: generated doc must be valid: %s", i, doc) + + opts := testOptions() + opts.GenerateId = seqIds(fmt.Sprintf("a%d_", i)) + sbType, snap, err := Unmarshal([]byte(doc), opts) + require.NoError(t, err, "case %d", i) + canonical, err := Marshal(sbType, snap, testOptions()) + require.NoError(t, err, "case %d", i) + // Marshal must never emit output its own Validate rejects + require.NoError(t, Validate(canonical), "case %d: canonical must validate: %s", i, canonical) + + opts2 := testOptions() + opts2.GenerateId = seqIds(fmt.Sprintf("b%d_", i)) + sbType2, snap2, err := Unmarshal(canonical, opts2) + require.NoError(t, err, "case %d: canonical must re-import: %s", i, canonical) + again, err := Marshal(sbType2, snap2, testOptions()) + require.NoError(t, err, "case %d", i) + require.Equal(t, string(canonical), string(again), "case %d: not byte-stable", i) + } +} diff --git a/pkg/lib/anyblockjson/schema/authoring/index.schema.json b/pkg/lib/anyblockjson/schema/authoring/index.schema.json new file mode 100644 index 0000000000..bdf8a189f3 --- /dev/null +++ b/pkg/lib/anyblockjson/schema/authoring/index.schema.json @@ -0,0 +1,139 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.anytype.io/anyblock/2/authoring/index.schema.json", + "title": "AnyBlock bundle index — authoring subset", + "description": "One index.json at the bundle root describes the bundle as a whole: what the space is called, what a user lands on, what the sidebar shows. A strict subset of the full index schema (../index.schema.json) — every index valid here is valid there. Every other file in the bundle is one object (authoring/object.schema.json) or the property dictionary (authoring/properties.schema.json).", + "x-app": "Anytype", + "type": "object", + "required": [ + "version", + "name", + "entrypoint" + ], + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string", + "description": "Write this file's URL: https://schemas.anytype.io/anyblock/2/authoring/index.schema.json" + }, + "version": { + "const": 2 + }, + "name": { + "type": "string", + "minLength": 1, + "description": "The space's name: \"Habit Tracker\"." + }, + "description": { + "type": "string", + "description": "One line on what the space is for." + }, + "icon": { + "type": "object", + "required": [ + "format", + "emoji" + ], + "additionalProperties": false, + "properties": { + "format": { + "const": "emoji" + }, + "emoji": { + "type": "string", + "minLength": 1 + } + }, + "description": "The space's icon, as an emoji: {\"format\": \"emoji\", \"emoji\": \"🌱\"}." + }, + "entrypoint": { + "$ref": "#/$defs/documentId", + "description": "The id of the page a new user lands on — usually a welcome page that explains the space. Also list it as the FIRST widget: the installer opens the first widget today." + }, + "homepage": { + "$ref": "#/$defs/documentId", + "description": "What opens on every later visit; omit to reuse the entrypoint. Name a page — a real page reads better than any built-in screen." + }, + "widgets": { + "type": "array", + "items": { + "$ref": "#/$defs/widget" + }, + "description": "The sidebar, top to bottom. Put the entrypoint first. The sidebar already lists every type, so give a type a widget only for fast access to its objects (layout \"view\")." + } + }, + "$defs": { + "documentId": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$", + "not": { + "enum": [ + "favorite", + "recent", + "recentOpen", + "set", + "collection", + "allObjects", + "chat", + "bin", + "widgets", + "graph" + ] + }, + "description": "The bundle-local id of a document in this bundle — the `id` its file declares. The excluded words are the built-in listings' and screens' own names; an id claiming one would capture the built-in." + }, + "widget": { + "type": "object", + "required": [ + "target" + ], + "additionalProperties": false, + "properties": { + "target": { + "anyOf": [ + { + "$ref": "#/$defs/documentId" + }, + { + "enum": [ + "_favorite", + "_recent", + "_recent_open", + "_set", + "_collection", + "_all_objects", + "_chat", + "_bin" + ] + } + ], + "description": "What the widget points at: the id of a page, type, set or collection in this bundle, or a built-in listing (_favorite, _recent, _recent_open, _set, _collection, _all_objects, _chat, _bin). Name your own pages and types first; the listings exist in every space already." + }, + "layout": { + "enum": [ + "link", + "tree", + "list", + "compact_list", + "view" + ], + "description": "How it renders: \"link\" one row (the default), \"tree\" expands children, \"list\"/\"compact_list\" show contents, \"view\" renders a type's or set's own view (a `view` widget shows the type's default view — view ids are minted on import, so there is nothing else to name)." + }, + "limit": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "description": "How many entries a listing widget shows. Ignored by \"link\"." + }, + "card_style": { + "enum": [ + "text", + "card", + "inline" + ], + "description": "How the widget's row renders; omit for a plain text row — the same choice a link block offers." + } + } + } + } +} diff --git a/pkg/lib/anyblockjson/schema/authoring/object.schema.json b/pkg/lib/anyblockjson/schema/authoring/object.schema.json new file mode 100644 index 0000000000..bebfd1d9b4 --- /dev/null +++ b/pkg/lib/anyblockjson/schema/authoring/object.schema.json @@ -0,0 +1,1255 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.anytype.io/anyblock/2/authoring/object.schema.json", + "title": "AnyBlock object — authoring subset", + "description": "One object you are creating from nothing: a page, a type, or a template. This is a strict subset of the full object schema (../object.schema.json) — every document valid here is valid there, same format, same version. What the full schema additionally admits is what only a live space can produce: block ids, attribution, timestamps, minted internal keys, legends, provenance. You never write those; a reader mints or derives them. Write what the object IS and leave the rest out.", + "x-app": "Anytype", + "type": "object", + "properties": { + "$schema": { + "type": "string", + "description": "Write this file's URL: https://schemas.anytype.io/anyblock/2/authoring/object.schema.json" + }, + "version": { + "const": 2 + }, + "kind": { + "enum": [ + "page", + "object_type", + "template" + ], + "description": "OMIT for an ordinary object — absent means page. Write \"object_type\" for a type document and \"template\" for a template; nothing else is authorable." + }, + "id": { + "$ref": "#/$defs/documentId", + "description": "This document's bundle-local id, a slug you pick: \"page-welcome\", \"type-habit\". Other files point here with it — a widget target, a link block, a collection's items. Give every document one." + }, + "type": { + "type": "string", + "minLength": 1, + "description": "The object's type, by name: a built-in type's display name (\"Page\", \"Task\", \"Collection\") or the internal_key of a type this bundle declares. Resolution folds away case and separators, so the stored key (\"page\") reaches the same type. On a template, the literal \"template\"." + }, + "template_for": { + "type": "string", + "minLength": 1, + "description": "Templates only: the key of the type this template is for — a type this bundle declares." + }, + "internal_key": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[^\u0000-\u001f]+$", + "description": "Type documents only: the type's spelling — its display name, written exactly (\"Habit\"). Objects name the type by writing this spelling in their `type`. Pick a fresh name — a built-in type's name (\"Task\", \"Book\") names the built-in type instead." + }, + "icon": { + "$ref": "#/$defs/icon" + }, + "cover": { + "$ref": "#/$defs/cover" + }, + "properties": { + "$ref": "#/$defs/propertyMap" + }, + "type_settings": { + "type": "object", + "description": "Type documents only: what the type is. Everything here is optional except that a useful type declares a layout and its properties.", + "properties": { + "layout": { + "enum": [ + "basic", + "profile", + "todo", + "note" + ], + "description": "How objects of this type look: \"basic\" (title + content), \"profile\" (round avatar), \"todo\" (checkbox in the title), \"note\" (no title)." + }, + "plural_name": { + "type": "string", + "description": "The plural display name: \"Habits\"." + }, + "default_template": { + "$ref": "#/$defs/documentId", + "description": "The id of a template document in this bundle that new objects of this type start from." + }, + "default_view": { + "enum": [ + "table", + "list", + "gallery", + "kanban", + "calendar" + ], + "description": "Which view the type opens with. Declare the views themselves in a dataview block on this document." + }, + "property_definitions": { + "type": "array", + "items": { + "$ref": "#/$defs/propertyDefinition" + }, + "description": "The type's properties, in display order. An entry names a property (by `property` or `name`) and says where it shows (`section`); its definition — format, options — can live here or in the bundle's properties.json." + } + }, + "additionalProperties": false + }, + "blocks": { + "type": "array", + "items": { + "$ref": "#/$defs/block" + }, + "description": "The document's content, top to bottom, as one flat array. Nesting is the per-block `indent`. No ids — the app assigns them." + }, + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/documentId" + }, + "description": "Collections only (type \"Collection\"): the member objects, by bundle-local id, in order." + } + }, + "required": [ + "version" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "kind": { + "const": "object_type" + } + }, + "required": [ + "kind" + ] + }, + "then": { + "required": [ + "internal_key", + "type_settings", + "properties" + ] + }, + "else": { + "properties": { + "internal_key": false, + "type_settings": false + } + } + }, + { + "if": { + "properties": { + "kind": { + "const": "template" + } + }, + "required": [ + "kind" + ] + }, + "then": { + "required": [ + "type", + "template_for" + ] + }, + "else": { + "properties": { + "template_for": false + } + } + }, + { + "if": { + "not": { + "required": [ + "kind" + ] + } + }, + "then": { + "properties": { + "type": { + "not": { + "const": "template" + } + } + } + } + } + ], + "$defs": { + "documentId": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$", + "not": { + "enum": [ + "favorite", + "recent", + "recentOpen", + "set", + "collection", + "allObjects", + "chat", + "bin", + "widgets", + "graph" + ] + }, + "description": "A bundle-local id: letters, digits, `-`, `_`; never starting with `_` (that prefix is the platform's) and never one of the ten reserved listing words — the index authoring schema beside this one reserves the same ten, and `chat` and `bin` are the two most common listing widgets." + }, + "propertyMap": { + "type": "object", + "description": "The object's property values, keyed by property NAME: `Name` (the title), `Description`, and the names this bundle's types and properties.json declare. Value shape follows the property's format — text/url/email/phone: a string; number: a number; checkbox: a boolean; date: an RFC 3339 UTC string (\"2026-07-06T15:04:05Z\"); select and multi_select: an ARRAY of option names; objects/files: an array of bundle-local ids; `Layout align`: an alignment NAME (see $defs/blockAlign) — the stored value is a number, but an author writes the name, exactly as blocks and views spell alignment. Only write a key to say something — an absent key means \"not set\". Property keys resolve case- and separator-insensitively, so \"Creation date\", `created_date` and `createdDate` are ONE key: the refusals below are stated per KEY, and every spelling of a refused key is refused. `Favorited: true` pins the object to Favorites. Do NOT write ids, icons, covers, timestamps or attribution here: the icon and cover are envelope fields, and the rest is the app's to derive.", + "propertyNames": { + "pattern": "^[^\u0000-\u001f]+$", + "maxLength": 128, + "not": { + "enum": [ + "id", + "type", + "layout", + "resolved_layout", + "resolvedLayout", + "icon_emoji", + "iconEmoji", + "icon_image", + "iconImage", + "icon_name", + "iconName", + "icon_option", + "iconOption", + "cover_id", + "coverId", + "cover_type", + "coverType", + "cover_scale", + "coverScale", + "cover_x", + "coverX", + "cover_y", + "coverY", + "creator", + "Created by", + "created_date", + "createdDate", + "Creation date", + "last_modified_by", + "lastModifiedBy", + "Last modified by", + "last_modified_date", + "lastModifiedDate", + "Last modified date", + "added_date", + "addedDate", + "Added date", + "space_id", + "spaceId", + "unique_key", + "uniqueKey", + "snippet", + "backlinks", + "links", + "mentions", + "origin", + "import_type", + "importType", + "old_anytype_id", + "oldAnytypeID", + "revision", + "Revision", + "restrictions", + "source_file_path", + "sourceFilePath", + "internal_flags", + "internalFlags", + "Internal flags", + "featured_properties", + "featuredRelations", + "Featured properties", + "is_archived", + "isArchived", + "Archived", + "is_deleted", + "isDeleted", + "sync_status", + "syncStatus" + ] + } + }, + "properties": { + "Layout align": { + "$ref": "#/$defs/blockAlign", + "description": "The object's own page alignment, by NAME. An author writes the name; raw numbers are the full format's stored-value pass-through, which the subset removes. This is the canonical spelling, and property keys resolve case- and separator-insensitively, so `layout_align` and `layoutAlign` reach the same property and are narrowed the same way." + }, + "layout_align": { + "$ref": "#/$defs/blockAlign", + "description": "`Layout align` under another spelling the codec resolves; see that member." + } + }, + "additionalProperties": { + "anyOf": [ + { + "type": [ + "string", + "number", + "boolean", + "null" + ] + }, + { + "type": "array" + } + ] + } + }, + "blockAlign": { + "enum": [ + "left", + "center", + "right", + "justify" + ], + "description": "Horizontal alignment, by NAME — the one vocabulary the format spells alignment in." + }, + "icon": { + "type": "object", + "description": "The object's icon — ONE of three kinds, chosen by `format`: an emoji, a built-in named icon, or a bare palette colour.", + "required": [ + "format" + ], + "properties": { + "format": { + "enum": [ + "emoji", + "icon", + "color" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "format": { + "const": "emoji" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "emoji" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "emoji": { + "type": "string", + "minLength": 1, + "description": "One emoji: \"🌱\"." + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "icon" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "name" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[^/]+$", + "description": "A built-in icon name (\"book\", \"repeat\", \"calendar\"). The usual choice for a type's icon. A name the app does not know shows as no icon." + }, + "color": { + "$ref": "#/$defs/paletteColor" + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "color" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "color" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "color": { + "$ref": "#/$defs/paletteColor" + } + } + } + } + ] + }, + "cover": { + "type": "object", + "description": "The object's cover — a flat colour or a gradient, chosen by `format`. The names are the app's own; observed colours: black, ice, blue; observed gradients: pinkOrange, red, sky, blue, bluePink, greenOrange.", + "required": [ + "format" + ], + "properties": { + "format": { + "enum": [ + "color", + "gradient" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "format": { + "const": "color" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "color" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "color": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[^/]+$" + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "gradient" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "gradient" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "gradient": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[^/]+$" + } + } + } + } + ] + }, + "paletteColor": { + "enum": [ + "grey", + "yellow", + "orange", + "red", + "pink", + "purple", + "blue", + "ice", + "teal", + "lime" + ] + }, + "propertyKey": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$", + "description": "A property spelling — a display name, written exactly: a built-in property's name (\"Name\", \"Due date\", \"Done\") or one this bundle declares (\"Streak\")." + }, + "propertyFormat": { + "enum": [ + "text", + "number", + "select", + "multi_select", + "date", + "checkbox", + "url", + "email", + "phone", + "objects", + "files" + ], + "description": "What kind of value the property holds. \"select\" is one choice from a vocabulary, \"multi_select\" several; \"objects\" points at other objects." + }, + "option": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "color": { + "$ref": "#/$defs/paletteColor" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + ], + "description": "One select option: a bare name (\"Daily\"), or {\"name\", \"color\"} when the colour is part of the design. Declaration order is display order." + }, + "propertyDefinition": { + "type": "object", + "description": "One property of a type. Identify it by `property` (the spelling — the property's name) or by `name` (the display name — \"Cooking Time\" declares a property spelled exactly that). Never invent stored ids: the app mints those on import.", + "properties": { + "property": { + "$ref": "#/$defs/propertyKey" + }, + "name": { + "type": "string", + "description": "Display name. Read only when the property is created here; a built-in key keeps its own name." + }, + "format": { + "$ref": "#/$defs/propertyFormat" + }, + "options": { + "type": "array", + "items": { + "$ref": "#/$defs/option" + }, + "description": "select/multi_select only: the vocabulary, in display order." + }, + "object_types": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "description": "objects/files only: which type keys the property may point at. Empty or absent = any object." + }, + "include_time": { + "type": "boolean", + "description": "date only: whether values carry a time of day." + }, + "section": { + "enum": [ + "featured", + "hidden" + ], + "description": "\"featured\" shows under the title; \"hidden\" hides it; absent = the regular sidebar list." + } + }, + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "property" + ] + }, + { + "required": [ + "name" + ] + } + ], + "allOf": [ + { + "if": { + "required": [ + "options" + ] + }, + "then": { + "required": [ + "format" + ], + "properties": { + "format": { + "enum": [ + "select", + "multi_select" + ] + } + } + } + }, + { + "if": { + "required": [ + "object_types" + ] + }, + "then": { + "required": [ + "format" + ], + "properties": { + "format": { + "enum": [ + "objects", + "files" + ] + } + } + } + }, + { + "if": { + "required": [ + "include_time" + ] + }, + "then": { + "required": [ + "format" + ], + "properties": { + "format": { + "const": "date" + } + } + } + } + ] + }, + "richText": { + "type": "string", + "description": "The block's text. Formatting goes inline, in the string: **bold**, *italic*, ~~strikethrough~~, `code`, [text](https://example.com) for an external link, [text](anytype://object?objectId=) for a link to a document in this bundle, underline. Everything else is literal; \\n is a soft line break. Escape a literal *, `, [, ], ~, < or \\ with a backslash." + }, + "literalText": { + "type": "string", + "description": "Raw content, verbatim — no inline markup is parsed here." + }, + "block": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "indent": { + "type": "integer", + "minimum": 0, + "maximum": 32, + "description": "Nesting depth; omit for top level. A block may be at most ONE level deeper than the block before it, and the first block is at 0." + }, + "type": { + "enum": [ + "paragraph", + "heading_1", + "heading_2", + "heading_3", + "quote", + "code", + "checkbox", + "bulleted_list_item", + "numbered_list_item", + "toggle", + "callout", + "toggle_heading_1", + "toggle_heading_2", + "toggle_heading_3", + "bookmark", + "link", + "divider", + "row", + "column", + "table", + "embed", + "table_of_contents", + "dataview" + ], + "description": "There is no title block — the title is the `name` property — and no image block: this format cannot fetch. A `row` contains only `column` blocks (indented under it); content nests under a column." + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "enum": [ + "paragraph", + "heading_1", + "heading_2", + "heading_3", + "quote", + "checkbox", + "bulleted_list_item", + "numbered_list_item", + "toggle", + "callout", + "toggle_heading_1", + "toggle_heading_2", + "toggle_heading_3" + ] + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "text": { + "$ref": "#/$defs/richText" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "checkbox" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "checked": { + "type": "boolean" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "callout" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "icon": { + "type": "object", + "required": [ + "format", + "emoji" + ], + "additionalProperties": false, + "properties": { + "format": { + "const": "emoji" + }, + "emoji": { + "type": "string", + "minLength": 1 + } + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "code" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "language": { + "type": "string", + "description": "\"go\", \"python\", \"json\", …" + }, + "text": { + "$ref": "#/$defs/literalText" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "embed" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "processor": { + "enum": [ + "latex", + "mermaid", + "chart", + "youtube", + "vimeo", + "soundcloud", + "google_maps", + "miro", + "figma", + "twitter", + "open_street_map", + "reddit", + "facebook", + "instagram", + "telegram", + "github_gist", + "codepen", + "bilibili", + "excalidraw", + "kroki", + "graphviz", + "sketchfab", + "image", + "drawio", + "spotify" + ], + "description": "What renders the embed. Defaults to latex." + }, + "text": { + "$ref": "#/$defs/literalText", + "description": "Source code for a renderer (latex, mermaid, chart, graphviz, kroki, excalidraw, drawio); the URL for everything else." + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "bookmark" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "type", + "url" + ], + "properties": { + "url": { + "type": "string", + "minLength": 1, + "description": "The bookmarked web page." + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "link" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "type", + "object_id" + ], + "properties": { + "object_id": { + "$ref": "#/$defs/documentId", + "description": "The bundle-local id of the document this links to." + }, + "card_style": { + "enum": [ + "text", + "card", + "inline" + ], + "description": "How the link renders; omit for a plain text row." + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "divider" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "style": { + "enum": [ + "line", + "dots" + ] + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "table" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "type", + "columns", + "rows" + ], + "properties": { + "columns": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "properties": {}, + "additionalProperties": false + }, + "description": "One `{}` per column — the entry only counts; the app assigns ids and widths. A row may not have more cells than there are columns." + }, + "rows": { + "type": "array", + "items": { + "$ref": "#/$defs/tableRow" + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "dataview" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "object_id": { + "$ref": "#/$defs/documentId", + "description": "Only on a PAGE embedding a view of a set or collection elsewhere in the bundle: that document's id. Omit on the dataview a type, set or collection carries as its own." + }, + "is_collection": { + "type": "boolean", + "description": "true when this views a collection (a curated list) rather than a set (a live query)." + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/dataviewProperty" + }, + "description": "The properties available to the views — every property a view's columns, sorts, filters or group_by name, with its format." + }, + "views": { + "type": "array", + "items": { + "$ref": "#/$defs/view" + } + } + } + } + } + ], + "unevaluatedProperties": false + }, + "tableRow": { + "type": "object", + "properties": { + "is_header": { + "type": "boolean" + }, + "cells": { + "type": "array", + "items": { + "type": [ + "string", + "null" + ] + }, + "description": "One entry per column, in column order: the cell's text, or null for an empty cell. Trailing empties may be left off." + } + }, + "additionalProperties": false + }, + "dataviewProperty": { + "type": "object", + "description": "One property available to the views, with its format. `property` is the same member name — and the same spelling — the view's columns, sorts, filters and group_by use to refer to it.", + "required": [ + "property", + "format" + ], + "additionalProperties": false, + "properties": { + "property": { + "$ref": "#/$defs/propertyKey" + }, + "format": { + "$ref": "#/$defs/propertyFormat" + } + } + }, + "view": { + "type": "object", + "description": "One way to look at the objects. No ids — the app assigns them.", + "properties": { + "type": { + "enum": [ + "table", + "list", + "gallery", + "kanban", + "calendar" + ], + "description": "Omit for table. kanban needs `group_by` on a select property; calendar groups by a date property." + }, + "name": { + "type": "string", + "description": "The view's tab label: \"By status\"." + }, + "group_by": { + "$ref": "#/$defs/propertyKey", + "description": "kanban/calendar: the property that makes the columns or places the cards." + }, + "card_size": { + "enum": [ + "small", + "medium", + "large" + ], + "description": "gallery card size; omit for small." + }, + "sorts": { + "type": "array", + "items": { + "$ref": "#/$defs/sort" + } + }, + "filters": { + "type": "array", + "items": { + "$ref": "#/$defs/filterNode" + }, + "description": "Top-level entries combine with AND; use one group node for OR." + }, + "columns": { + "type": "array", + "items": { + "$ref": "#/$defs/viewColumn" + }, + "description": "Which properties the view shows, in order." + } + }, + "additionalProperties": false + }, + "sort": { + "type": "object", + "required": [ + "property" + ], + "additionalProperties": false, + "properties": { + "property": { + "$ref": "#/$defs/propertyKey" + }, + "direction": { + "enum": [ + "asc", + "desc" + ], + "description": "Omit for asc." + }, + "empty_placement": { + "enum": [ + "start", + "end" + ], + "description": "Where objects with no value go." + } + } + }, + "filterNode": { + "oneOf": [ + { + "type": "object", + "required": [ + "operator", + "filters" + ], + "additionalProperties": false, + "properties": { + "operator": { + "enum": [ + "and", + "or" + ] + }, + "filters": { + "type": "array", + "items": { + "$ref": "#/$defs/filterNode" + } + } + } + }, + { + "type": "object", + "required": [ + "property" + ], + "additionalProperties": false, + "properties": { + "property": { + "$ref": "#/$defs/propertyKey" + }, + "condition": { + "enum": [ + "equal", + "not_equal", + "greater", + "less", + "greater_or_equal", + "less_or_equal", + "contains", + "not_contains", + "in", + "not_in", + "all_in", + "empty", + "not_empty" + ] + }, + "value": { + "description": "What to compare against. For select/multi_select: an array of option names. Omit on empty/not_empty." + }, + "date_preset": { + "enum": [ + "yesterday", + "today", + "tomorrow", + "last_week", + "current_week", + "next_week", + "last_month", + "current_month", + "next_month", + "last_year", + "current_year", + "next_year" + ], + "description": "Date properties, with equal/in/less/greater/less_or_equal/greater_or_equal: a rolling range instead of a fixed `value`. An \"overdue\" filter (`less` + a date) must sit in an `and` group with a `not_empty` on the same property, or undated objects match too." + } + } + } + ] + }, + "viewColumn": { + "type": "object", + "required": [ + "property" + ], + "additionalProperties": false, + "properties": { + "property": { + "$ref": "#/$defs/propertyKey" + }, + "hidden": { + "type": "boolean" + }, + "aggregation": { + "enum": [ + "count", + "count_value", + "count_distinct", + "count_empty", + "count_not_empty", + "percent_empty", + "percent_not_empty", + "sum", + "average", + "median", + "min", + "max", + "range" + ], + "description": "A footer over the column's values; omit for none." + } + } + } + } +} diff --git a/pkg/lib/anyblockjson/schema/authoring/properties.schema.json b/pkg/lib/anyblockjson/schema/authoring/properties.schema.json new file mode 100644 index 0000000000..864a8b0e32 --- /dev/null +++ b/pkg/lib/anyblockjson/schema/authoring/properties.schema.json @@ -0,0 +1,193 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.anytype.io/anyblock/2/authoring/properties.schema.json", + "title": "AnyBlock property dictionary — authoring subset", + "description": "One properties.json at the bundle root, beside index.json: the bundle's own properties, declared once, in one place. A strict subset of the full dictionary schema (../properties.schema.json) — every dictionary valid here is valid there. Declare a property here, reference it from a type's property_definitions by its key, and write its values in objects under the same key; the app mints the stored identity on import.", + "x-app": "Anytype", + "type": "object", + "required": [ + "version", + "properties" + ], + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string", + "description": "Write this file's URL: https://schemas.anytype.io/anyblock/2/authoring/properties.schema.json" + }, + "version": { + "const": 2 + }, + "installed": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "description": "Built-in properties this bundle wants available, by name: [\"Due date\", \"Tag\"]. Presence only — never redefine a built-in here." + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/property" + }, + "description": "The bundle's own properties, one entry each." + } + }, + "$defs": { + "property": { + "type": "object", + "description": "One property. Identify it by `property` (the spelling your documents will write — the property's name: \"Streak\") or by `name` (\"Cooking Time\" declares a property spelled exactly that); `format` is required. Never invent stored ids — the app mints those on import.", + "required": [ + "format" + ], + "anyOf": [ + { + "required": [ + "property" + ] + }, + { + "required": [ + "name" + ] + } + ], + "allOf": [ + { + "if": { + "required": [ + "options" + ] + }, + "then": { + "properties": { + "format": { + "enum": [ + "select", + "multi_select" + ] + } + } + } + }, + { + "if": { + "required": [ + "object_types" + ] + }, + "then": { + "properties": { + "format": { + "enum": [ + "objects", + "files" + ] + } + } + } + }, + { + "if": { + "required": [ + "include_time" + ] + }, + "then": { + "properties": { + "format": { + "const": "date" + } + } + } + } + ], + "additionalProperties": false, + "properties": { + "property": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$", + "description": "The property's spelling — its display name, written exactly: \"Streak\", \"Last done\". A fresh name — a built-in property's name (\"Due date\") names the built-in property instead." + }, + "name": { + "type": "string", + "description": "Display name: \"Last done\"." + }, + "format": { + "enum": [ + "text", + "number", + "select", + "multi_select", + "date", + "checkbox", + "url", + "email", + "phone", + "objects", + "files" + ], + "description": "What kind of value the property holds. \"select\" is one choice from a vocabulary, \"multi_select\" several; \"objects\" points at other objects." + }, + "options": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "color": { + "enum": [ + "grey", + "yellow", + "orange", + "red", + "pink", + "purple", + "blue", + "ice", + "teal", + "lime" + ] + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + ] + }, + "description": "select/multi_select only: the vocabulary, in display order. An entry is a bare name (\"Daily\") or {\"name\", \"color\"} when the colour is part of the design." + }, + "object_types": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "description": "objects/files only: which type keys the property may point at. Empty or absent = any object." + }, + "description": { + "type": "string", + "description": "One line on what the property means." + }, + "include_time": { + "type": "boolean", + "description": "date only: whether values carry a time of day." + } + } + } + } +} diff --git a/pkg/lib/anyblockjson/schema/index.schema.json b/pkg/lib/anyblockjson/schema/index.schema.json new file mode 100644 index 0000000000..7ffbbb8281 --- /dev/null +++ b/pkg/lib/anyblockjson/schema/index.schema.json @@ -0,0 +1,181 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.anytype.io/anyblock/2/index.schema.json", + "title": "AnyBlock JSON bundle index", + "description": "One index.json at the root of a bundle describes the bundle as a whole: what the space is called, where a user lands after installing it, and what the sidebar shows. Everything else in a bundle describes a single object; this is the only document that describes the set.", + "type": "object", + "required": [ + "version" + ], + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string", + "description": "Which of the format's three grammars this document follows — an object document, a bundle index, or a property dictionary. OPTIONAL and DECORATIVE for validity: `version` alone gates the format, so a stale or invented URL here never makes a document invalid (a bundle written against a later schema still reads). What it does do is DISPATCH: a reader handed a file with no filename to go by uses this to pick the grammar, matching on the trailing `object|index|properties.schema.json` and ignoring the version segment. Without it a reader falls back to the document's shape, which can only tell the grammars apart when the document happens to carry a member unique to one. Every document this format writes carries it, and a hand-written one is easier for a reader to place if it does too." + }, + "version": { + "const": 2 + }, + "name": { + "type": "string", + "minLength": 1, + "description": "The space's name, applied on install." + }, + "description": { + "type": "string", + "description": "One line on what the space is for. Author-facing and applied to the space where the client shows one." + }, + "icon": { + "$ref": "https://schemas.anytype.io/anyblock/2/object.schema.json#/$defs/icon", + "description": "The space's icon — exactly the shape `icon` has on any object (§2b), so an author never has to remember a second convention: an emoji, the **object id** of an image in the bundle, a built-in icon name, or a bare colour. Two flat keys used to stand here with no rule for which one wins, and they disagreed with the object surface about whether an image is a scalar or a list. This field was then narrowed to emoji-or-image, which silently dropped the icon of a space whose icon is a bare colour — 20 of 77 real spaces, the letter avatars the client draws when no image was ever set. An image needs the image object AND its file in the archive, which is why a generated bundle uses an emoji. (The installer resolves the space icon by image *name*, not id — the wiring does that lookup, so this field stays consistent with every other id in the format; an emoji or colour icon reaches no installer field at all, and is carried so a round trip does not lose it.)" + }, + "entrypoint": { + "type": "string", + "minLength": 1, + "pattern": "^[^_]", + "description": "The object opened once, immediately after the space is created from this bundle — the first thing a user ever sees. Must be an object id from this bundle, so it may not begin with `_`: that prefix is the platform's own address space (built-in screens, listings and bundled objects), and no reserved screen can be the thing a space opens *with*. TEMPORARY: the installer does not read this yet — it opens the first widget instead (pb.Profile has no field for it). Until the heart-side profile handling is improved, also list the entrypoint as your first widget, or what opens will not be what you declared." + }, + "homepage": { + "type": "string", + "minLength": 1, + "anyOf": [ + { + "enum": [ + "_widgets", + "_graph" + ] + }, + { + "pattern": "^[^_]" + } + ], + "description": "What opens whenever the user enters the space afterwards. An object id from this bundle, which may not begin with `_`; defaults to `entrypoint`. The reserved `_widgets` (sidebar dashboard) and `_graph` are accepted but are rarely right for a use case — on desktop the widgets are always in the sidebar anyway, so `_widgets` wastes the main pane; it only reads well on mobile, which shows one screen at a time. Name a page." + }, + "widgets": { + "type": "array", + "description": "Sidebar widgets, in the order they should appear. Independent of `entrypoint` — declaring one does not require a widget for it. Only give a type a widget when you want fast access to its objects (`layout: \"view\"`); the sidebar already lists every type automatically, so a link or list widget on a type just duplicates that.", + "items": { + "$ref": "#/$defs/widget" + } + }, + "auto_widget_targets": { + "type": "array", + "items": { + "$ref": "#/$defs/widgetTarget" + }, + "description": "The targets the client has already auto-added a widget for — its ledger for not re-adding one the user then deleted. Space state, not sidebar content: an entry usually names a widget that is NOT in `widgets` any more, which is the point. Entries are spelled like widget targets (an object id, or a reserved listing). Machine-written on export; an authored bundle has no reason to carry one." + }, + "auto_widget_disabled": { + "type": "boolean", + "description": "The user turned automatic widget creation off for this space entirely. Machine-written on export, like the ledger above." + }, + "manifest": { + "$ref": "#/$defs/manifest", + "description": "Where to find what a reader must resolve by key or id rather than by walking (§2c): types, the property dictionary, and file blobs. The format defines NO folder layout — objects/, types/, files/ are one exporter's convention — and an object names its type by spelling alone, so without this a reader resolves a type by scanning every document for a matching key, and a file document's bytes by guessing at a layout." + } + }, + "$defs": { + "manifest": { + "type": "object", + "additionalProperties": false, + "properties": { + "types": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1 + }, + "description": "The type's CANONICAL SPELLING → the type document's path, relative to this file. The canonical spelling, not a per-document term: the display name from the shipped table for a bundled type, the stored key verbatim for a space-minted one — a pure function of the key, the same rule the property dictionary applies to its entry keys (§2f), because this file carries no legend and its keys must resolve through the shipped chain alone: an exact stored key names itself, then the shipped display-name table, then the forgiving fold. Reader flow: object → `type` spelling → the object's own legend → stored key → this map's key resolved the same way → the type file, no scanning. The manifest locates types, the dictionary and file blobs: it does not locate options, because the dictionary states each property's whole vocabulary inline (name, colour, position, internal_key), so a reader never has an option lookup to resolve by path." + }, + "properties": { + "type": "string", + "minLength": 1, + "description": "The property dictionary's path, relative to this file — properties.json at the bundle root (§2f). A pointer rather than an inline map because properties resolve by stored KEY through each document's own legend, and the dictionary is the file that answers for stored keys." + }, + "files": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1 + }, + "description": "File object id → the blob's path, relative to this file (§2c, v0.47). The authoritative binding between a `kind: \"file_object\"` document and its bytes: the document itself carries no path — a document member is not a slot for archive bookkeeping — and every importer holding a file document must find its bytes, so the lookup lives where every other by-id lookup lives. Keys are object ids verbatim, an authored bundle writes them against its own minted ids and any layout it likes; adjacency of blob and document in files/ is one exporter's convention, never load-bearing. An entry that cannot be honoured — a key naming no document, a path escaping the bundle, a blob missing at the path — is a cross-document refusal for the tooling; a file document a present map does not bind is warned about (its bytes did not travel), and a bundle with no map is a metadata-only export." + } + } + }, + "widgetTarget": { + "type": "string", + "minLength": 1, + "anyOf": [ + { + "enum": [ + "_favorite", + "_recent", + "_recent_open", + "_set", + "_collection", + "_all_objects", + "_chat", + "_bin" + ] + }, + { + "pattern": "^[^_]" + } + ], + "description": "An object id from this bundle (a page, a type, a set, a collection), or one of the eight reserved listings — the built-in screens a live sidebar can show. The leading `_` is the platform's address space and is what keeps the two kinds of target apart: an object id from the bundle may never begin with one, so a bundle cannot shadow a listing with an object of its own. All eight survive import (widget.IsPredefinedWidgetTargetId knows their wire spellings); a `_`-prefixed name outside the inventory is refused by name, with the inventory in the message, because it can only be a typo." + }, + "widget": { + "type": "object", + "required": [ + "target" + ], + "additionalProperties": false, + "properties": { + "target": { + "$ref": "#/$defs/widgetTarget", + "description": "What the widget points at." + }, + "layout": { + "$ref": "https://schemas.anytype.io/anyblock/2/object.schema.json#/$defs/widgetLayout", + "default": "link", + "description": "How the widget renders — the widget BLOCK's own §5 vocabulary, one $ref rather than a copy. `link` is a single row; `tree` expands children; `list` and `compact_list` show contents; `view` renders a type's or set's own view." + }, + "limit": { + "$ref": "https://schemas.anytype.io/anyblock/2/object.schema.json#/$defs/widgetListingLimit", + "description": "How many entries a listing widget shows. Ignored by `link`. The cap is the object schema's $defs/widgetListingLimit — one $ref rather than a copy, like the display members beside it." + }, + "view_id": { + "type": "string", + "minLength": 1, + "description": "Which of the target's views a `view` widget shows — a view id inside the target's dataview. Omit for the target's default view, which is also all an author can name: view ids are minted on import." + }, + "auto_added": { + "type": "boolean", + "description": "The client added this widget automatically (a new type, the bin) rather than the user placing it. Machine-written on export; carried so a restored sidebar keeps behaving like the original — the client treats auto-added widgets as its own to manage." + }, + "card_style": { + "$ref": "https://schemas.anytype.io/anyblock/2/object.schema.json#/$defs/cardStyle", + "description": "How the widget's row renders — the link block's own §5 member, one $ref rather than a copy. Defaults to `text`." + }, + "icon_size": { + "$ref": "https://schemas.anytype.io/anyblock/2/object.schema.json#/$defs/iconSize", + "description": "The icon size on the widget's card — the link block's own §5 member. Defaults to `none`." + }, + "description": { + "$ref": "https://schemas.anytype.io/anyblock/2/object.schema.json#/$defs/linkDescription", + "description": "What the widget's card shows under the title — the link block's own §5 member. Defaults to `none`." + }, + "properties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + }, + "description": "The property SPELLINGS shown on the widget's card — the link block's own §5 member. They resolve through the bundle's property dictionary (properties.json), the file that answers for stored keys; there is no per-document legend here." + } + } + } + } +} diff --git a/pkg/lib/anyblockjson/schema/object.schema.json b/pkg/lib/anyblockjson/schema/object.schema.json new file mode 100644 index 0000000000..60a8906973 --- /dev/null +++ b/pkg/lib/anyblockjson/schema/object.schema.json @@ -0,0 +1,1884 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "title": "AnyBlock object", + "description": "A single Anytype object in AnyBlock JSON interchange format, version 2", + "x-app": "Anytype", + "type": "object", + "properties": { + "$schema": { + "type": "string", + "description": "Which of the format's three grammars this document follows — an object document, a bundle index, or a property dictionary. OPTIONAL and DECORATIVE for validity: `version` alone gates the format, so a stale or invented URL here never makes a document invalid (a bundle written against a later schema still reads). What it does do is DISPATCH: a reader handed a file with no filename to go by uses this to pick the grammar, matching on the trailing `object|index|properties.schema.json` and ignoring the version segment. Without it a reader falls back to the document's shape, which can only tell the grammars apart when the document happens to carry a member unique to one. Every document this format writes carries it, and a hand-written one is easier for a reader to place if it does too." + }, + "version": { + "const": 2 + }, + "kind": { + "enum": [ + "account_old", + "page", + "profile_page", + "home", + "archive", + "widget", + "file", + "template", + "bundled_template", + "bundled_property", + "sub_object", + "bundled_object_type", + "anytype_profile", + "date", + "space_settings", + "property", + "object_type", + "property_option", + "space_view", + "identity", + "participant", + "missing_object", + "file_object", + "notification", + "devices", + "chat_object", + "chat", + "account", + "discussion", + "tech_space", + "tech_space_virtual" + ] + }, + "id": { + "type": "string" + }, + "type": { + "type": "string", + "minLength": 1 + }, + "template_for": { + "type": "string", + "minLength": 1 + }, + "internal_key": { + "description": "The STORED identity key of a definition document (a type, a property, a property option) — the uniqueKey's internal part, written verbatim. This is an id the app MINTS, not something an author writes: for a custom property or type it is a bson id like 6a83296f61fab2265263ae34, which no author can produce, and for a bundled one it is the bundled camelCase key (dueDate). Export writes it for fidelity; an authored definition document may omit it, and the import wiring then mints a fresh internal key exactly as the app does when a user creates one. It is deliberately NOT a spelling: unlike every key slot in §3 it is never translated through a legend or the bundled table, so it does not match the display-name spelling documents use (that spelling is the `property` member of a property definition). Charset is a deny rule, not an allowlist — a property option's stored key is built from its NAME, so keys like completion_status_Not Started are real stored keys.", + "type": "string", + "pattern": "^[^\u0000-\u001f]+$", + "maxLength": 255 + }, + "property_settings": { + "description": "Only on property documents (kind \"property\" or \"bundled_property\"), where it is REQUIRED: the definition of the property this document IS — one propertyDefinition (§2d, §2e). Three members travel today, each mirroring stored presence EXACTLY, value included: `format` (required — a §3 format NAME, the same vocabulary type_properties[].format speaks, never a raw enum number; stands for the stored relationFormat key, which `properties` refuses), `include_time` (present exactly when the stored relationFormatIncludeTime key is; meaningful on format \"date\" only, and a stored null travels as null), and `object_types` (present exactly when the stored relationFormatObjectTypes key is; the target type keys in priority order — a type-key slot exactly like type_properties[].object_types, inverted through the type_internal_keys legend, meaningful on \"objects\"/\"files\" only). The members refused with `false` have a home already: `internal_key` is the envelope's (and `property`, the spelling, is derived from it — a property document never states its own spelling), `name` and `description` are properties', `options` are property_option documents, and max_count/readonly/default_value still travel in `properties` under their stored keys until the dictionary lifts them — admitting a second spelling of any of them here would be the §15 #14 disease this group exists to end.", + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/propertyDefinition" + } + ], + "properties": { + "property": false, + "internal_key": false, + "name": false, + "options": false, + "description": false, + "max_count": false, + "readonly": false, + "default_value": false + }, + "required": [ + "format" + ], + "unevaluatedProperties": false + }, + "icon": { + "$ref": "#/$defs/icon" + }, + "cover": { + "$ref": "#/$defs/cover" + }, + "properties": { + "$ref": "#/$defs/propertyMap" + }, + "type_settings": { + "description": "Only on type documents (kind \"object_type\" or \"bundled_object_type\"): everything that defines the TYPE, in one gated subtree (§2a). `layout` is the recommended layout of objects of this type (a layout name; a stored number outside the vocabulary passes through raw). `api_key` is the type's public API key — NOT a slug: of 1,326 corpus type documents with one, it differs from the document's own spelling in 247 (the `property` type's api key is `relation`, the word the public API kept when the format renamed the concept) — calling it a slug would imply it is the term used elsewhere in the document, which for those 247 it is not. `plural_name` is the plural display name. `default_template` is the object id of the template new objects of this type start from. `default_view` is the default view type (a §6.2 view-type name; a stored number outside the vocabulary passes through raw). `property_definitions` is the type's property list — one propertyDefinition + `section` per entry (§2e); it lived at the document root as `type_properties` until v0.32, and the word is `property_definitions` rather than `properties` because the document already uses that word for property VALUES at the root — one word per concept.", + "type": "object", + "properties": { + "layout": { + "anyOf": [ + { + "$ref": "#/$defs/objectLayout" + }, + { + "type": "number" + } + ], + "description": "The layout instances of this type get. Stated as a name; see $defs/objectLayout." + }, + "api_key": { + "type": "string" + }, + "plural_name": { + "type": "string" + }, + "default_template": { + "$ref": "#/$defs/objectRef" + }, + "default_view": { + "anyOf": [ + { + "$ref": "#/$defs/viewType" + }, + { + "type": "number" + } + ], + "description": "How this type's own view renders by default. Stated as a name; see $defs/viewType." + }, + "property_definitions": { + "type": "array", + "items": { + "$ref": "#/$defs/typeProperty" + } + } + }, + "additionalProperties": false + }, + "property_internal_keys": { + "type": "object", + "description": "Legend: the stored property key each SPELLING in this document names (§3) — a spelling is a display name, so this is the map from \"Cooking time\" to the key the space stores it under. Present only for keys the bundled table cannot invert; a reader consults it before its own vocabulary. A value is a stored key and obeys the same writable-key rule as a property name (§3).", + "propertyNames": { + "pattern": "^[^\u0000-\u001f]+$", + "maxLength": 128 + }, + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + } + }, + "type_internal_keys": { + "type": "object", + "description": "Legend: the stored type key each type SPELLING in this document names (§3). Same rule as property_internal_keys, on a legend of the type namespace's own: present only for spellings the bundled table cannot invert; a reader consults it before its own vocabulary. A value is a stored type key and obeys the writable-key rule (§3).", + "propertyNames": { + "pattern": "^[^\u0000-\u001f]+$", + "maxLength": 128 + }, + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + } + }, + "option_ids": { + "type": "object", + "description": "Legend: the id of the option each select/multi_select NAME in this document stands for (§3, §9a). Nested: an outer key is a property spelling as this document writes it, an inner key is an option name exactly as the value slot spells it, and the value is the full option id. Written unconditionally wherever export spells an option by name; absent from an id-less export. Read as a HINT, not an address: an id is honoured only where the target space still serves it as a live option of that property, and otherwise the name resolves as it would without the legend. An outer key naming a property the document never spells is reported as a warning — the entry can never be consulted.", + "propertyNames": { + "pattern": "^[^\u0000-\u001f]+$", + "maxLength": 128 + }, + "additionalProperties": { + "type": "object", + "propertyNames": { + "minLength": 1 + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + }, + "blocks": { + "type": "array", + "items": { + "$ref": "#/$defs/block" + } + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "store": { + "type": "object", + "x-output-only": true + }, + "root": { + "type": "object", + "x-output-only": true + } + }, + "required": [ + "version" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "kind": { + "enum": [ + "property", + "bundled_property" + ] + } + }, + "required": [ + "kind" + ] + }, + "then": { + "required": [ + "property_settings" + ] + }, + "else": { + "properties": { + "property_settings": false + } + } + }, + { + "if": { + "properties": { + "kind": { + "enum": [ + "object_type", + "bundled_object_type" + ] + } + }, + "required": [ + "kind" + ] + }, + "then": {}, + "else": { + "properties": { + "type_settings": false + } + } + } + ], + "$defs": { + "cardStyle": { + "enum": [ + "text", + "card", + "inline" + ], + "description": "How a link renders: a plain text row (the default), a card, or inline. One definition for every surface that shows a link — the link block and a bundle index's widgets both $ref it, so they cannot disagree." + }, + "iconSize": { + "enum": [ + "none", + "small", + "medium" + ], + "description": "The icon size on a link's card. Defaults to none." + }, + "linkDescription": { + "enum": [ + "none", + "manual", + "content" + ], + "description": "What the link's card shows under the title: nothing (the default), the manually written description, or the target's own content." + }, + "widgetLayout": { + "enum": [ + "link", + "tree", + "list", + "compact_list", + "view" + ], + "description": "How a sidebar widget renders: `link` is a single row (the default), `tree` expands children, `list` and `compact_list` show contents, `view` renders a type's or set's own view. One definition for the widget block and a bundle index's widgets." + }, + "widgetListingLimit": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "description": "How many entries a sidebar listing widget shows — the product cap for a bundle index's flat widget (the live corpus maximum is 50). One definition, referenced by the index schema, so the cap cannot drift into a second spelling. The widget BLOCK's own `limit` member deliberately takes the whole int32 range instead: the block is the fidelity fallback for exactly the widget the index refuses over this cap (the widget-object lift keeps the whole document), so the cap must not bind there." + }, + "propertyMap": { + "type": "object", + "propertyNames": { + "pattern": "^[^\u0000-\u001f]+$", + "maxLength": 128 + }, + "additionalProperties": { + "anyOf": [ + { + "type": [ + "string", + "number", + "boolean", + "null" + ] + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + }, + "objectRef": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[^/]+$", + "description": "The id of an object in this space, or a bundle-local slug. Never a URL and never a filesystem path — this format does no I/O, so it can only name something the store already holds. To supply an image by URL, use a layer that can fetch it (the API): it uploads the image, mints the file object and writes that object's id here. ONE SLOT, TWO ADDRESS SPACES, and a reader has to tell them apart: an icon's `file` may hold a raw CONTENT cid instead of an object id, because the app writes one there — a participant's avatar and a space invite's icon are stored as the cid of the image itself (core/acl/aclservice.go), not as an object. 992 of 12,378 file icons in a 77-space export are of that kind. They are told apart by CID codec: an OBJECT id is dag-cbor and begins `bafyrei`, a content cid is raw or dag-pb and begins `bafybei`. A content cid resolves against the blob store, never against the bundle, so a reader that dereferences it as an object id finds nothing — not because the object is missing but because it was never an object." + }, + "iconColor": { + "description": "One of the ten palette colours — the same palette select options use (§2a) — or, for a stored value the palette has no name for, its raw number.", + "oneOf": [ + { + "$ref": "#/$defs/optionColor" + }, + { + "type": "integer", + "minimum": 1 + } + ] + }, + "opaqueName": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[^/]+$", + "description": "A name from a vocabulary this format does not enumerate (the app's icon, cover-colour and gradient sets). Validation checks the shape only — a name outside the app's set is accepted here and shows as no icon." + }, + "iconVariants": { + "type": "object", + "description": "The icon variant machinery, shared by every icon slot. It carries the per-variant if/then rules but NOT the set of variant names: each slot states its own, so a value outside a slot's set fails exactly one enum and the reader is told one thing.", + "required": [ + "format" + ], + "properties": { + "format": { + "description": "Which kind of icon this is: an emoji, an image object in this space, a built-in named icon, or a bare colour." + } + }, + "allOf": [ + { + "if": { + "properties": { + "format": { + "const": "emoji" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "emoji" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "emoji": { + "type": "string", + "minLength": 1 + }, + "color": { + "$ref": "#/$defs/iconColor" + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "file" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "file" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "file": { + "$ref": "#/$defs/objectRef", + "description": "The image. Normally the id of a file object in this bundle; for a participant avatar or an invite icon it is the raw content cid of the image instead — see $defs/objectRef for how the two are told apart." + }, + "color": { + "$ref": "#/$defs/iconColor" + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "icon" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "name" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "name": { + "$ref": "#/$defs/opaqueName" + }, + "color": { + "$ref": "#/$defs/iconColor" + }, + "emoji": { + "type": "string", + "minLength": 1, + "x-output-only": true, + "description": "A legacy emoji superseded by this named icon, carried so export loses nothing (§2b). Export writes it; a document that supplies it is not choosing an icon — `format` already did." + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "color" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "color" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "color": { + "$ref": "#/$defs/iconColor" + } + } + } + } + ] + }, + "icon": { + "description": "The object's icon: ONE of four kinds, chosen by `format` (§2b). Replaces the flat iconEmoji/iconImage/iconName/iconOption keys, which are refused in `properties`.", + "allOf": [ + { + "$ref": "#/$defs/iconVariants" + } + ], + "properties": { + "format": { + "enum": [ + "emoji", + "file", + "icon", + "color" + ], + "description": "Which kind of icon this is: an emoji, an image object in this space, a built-in named icon, or a bare colour." + } + } + }, + "plainIcon": { + "description": "An icon restricted to the two kinds a callout block (§5.2) and a bundle index (§2c) can hold: an emoji, or an image object in this space. The same shape as the object icon, minus the two variants only an object has.", + "allOf": [ + { + "$ref": "#/$defs/iconVariants" + } + ], + "properties": { + "format": { + "enum": [ + "emoji", + "file" + ] + } + } + }, + "cover": { + "type": "object", + "description": "The object's cover: ONE of three kinds, chosen by `format` (§2b). Replaces the flat coverId/coverType/coverScale/coverX/coverY keys, which are refused in `properties`. `cover_id` alone cannot be read: the same string names a colour under one type and a gradient under another.", + "required": [ + "format" + ], + "properties": { + "format": { + "enum": [ + "image", + "color", + "gradient" + ], + "description": "Which kind of cover this is: an image object in this space, a flat colour, or a gradient." + } + }, + "allOf": [ + { + "if": { + "properties": { + "format": { + "const": "image" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "file" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "file": { + "$ref": "#/$defs/objectRef" + }, + "source": { + "enum": [ + "unsplash", + "prebuilt" + ], + "x-output-only": true, + "description": "Where this image came from (§2b). Export writes it; a generator omits it — an uploaded image has no provenance to claim." + }, + "scale": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "color" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "color" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "color": { + "$ref": "#/$defs/opaqueName" + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "gradient" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "gradient" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "gradient": { + "$ref": "#/$defs/opaqueName" + } + } + } + } + ] + }, + "blockId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{1,64}$" + }, + "tableInnerId": { + "type": "string", + "pattern": "^[A-Za-z0-9_]{1,64}$" + }, + "color": { + "type": "string" + }, + "block": { + "type": "object", + "required": [ + "type" + ], + "allOf": [ + { + "$ref": "#/$defs/blockCore" + }, + { + "if": { + "properties": { + "type": { + "const": "table" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "columns": { + "type": "array", + "items": { + "$ref": "#/$defs/tableColumn" + } + }, + "rows": { + "type": "array", + "items": { + "$ref": "#/$defs/tableRow" + } + } + } + } + } + ], + "unevaluatedProperties": false + }, + "cellBlock": { + "type": "object", + "required": [ + "type" + ], + "allOf": [ + { + "$ref": "#/$defs/blockCore" + } + ], + "properties": { + "type": { + "not": { + "const": "table" + } + } + }, + "unevaluatedProperties": false + }, + "blockCore": { + "properties": { + "indent": { + "type": "integer", + "minimum": 0, + "maximum": 32 + }, + "id": { + "$ref": "#/$defs/blockId" + }, + "type": { + "enum": [ + "paragraph", + "heading_1", + "heading_2", + "heading_3", + "heading_4", + "header_4", + "quote", + "code", + "title", + "description", + "checkbox", + "bulleted_list_item", + "numbered_list_item", + "toggle", + "callout", + "toggle_heading_1", + "toggle_heading_2", + "toggle_heading_3", + "file", + "image", + "video", + "audio", + "pdf", + "bookmark", + "link", + "divider", + "row", + "column", + "group", + "table", + "embed", + "equation", + "table_of_contents", + "property", + "dataview", + "widget", + "chat", + "featured_properties", + "icon" + ] + }, + "align": { + "$ref": "#/$defs/blockAlign" + }, + "vertical_align": { + "enum": [ + "top", + "middle", + "bottom" + ] + }, + "background_color": { + "type": "string" + }, + "fields": { + "type": "object", + "x-output-only": true + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "enum": [ + "paragraph", + "quote", + "toggle", + "bulleted_list_item", + "numbered_list_item", + "toggle_heading_1", + "toggle_heading_2", + "toggle_heading_3", + "title", + "description" + ] + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "color": { + "$ref": "#/$defs/color" + }, + "text": { + "type": "string", + "description": "The block's text. Formatting is expressed INLINE, inside this string — this format has no `marks` array and no offsets (§8). A CommonMark-inline subset plus a small tag whitelist, and nothing else:\n `**bold**`, `*italic*`, `~~strikethrough~~`, `` `code` ``\n `[text](https://example.com)` — an external link\n `[text](anytype://object?objectId=)` — a link to an object in this space\n `\">text` — a decorated object reference (icon + name)\n `underline`, `text`, `text`\nEverything else is literal text: no block syntax, no images, no autolinks, no HTML beyond those tags. `\\n` is a soft line break within the block. A literal `*`, `` ` ``, `[`, `]`, `~`, `<` or `\\` in prose is backslash-escaped." + } + } + } + }, + { + "if": { + "properties": { + "type": { + "enum": [ + "heading_1", + "heading_2", + "heading_3", + "heading_4", + "header_4" + ] + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "color": { + "$ref": "#/$defs/color" + }, + "text": { + "type": "string", + "description": "The block's text. Formatting is expressed INLINE, inside this string — this format has no `marks` array and no offsets (§8). A CommonMark-inline subset plus a small tag whitelist, and nothing else:\n `**bold**`, `*italic*`, `~~strikethrough~~`, `` `code` ``\n `[text](https://example.com)` — an external link\n `[text](anytype://object?objectId=)` — a link to an object in this space\n `\">text` — a decorated object reference (icon + name)\n `underline`, `text`, `text`\nEverything else is literal text: no block syntax, no images, no autolinks, no HTML beyond those tags. `\\n` is a soft line break within the block. A literal `*`, `` ` ``, `[`, `]`, `~`, `<` or `\\` in prose is backslash-escaped." + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "checkbox" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "checked": { + "type": "boolean" + }, + "color": { + "$ref": "#/$defs/color" + }, + "text": { + "type": "string", + "description": "The block's text. Formatting is expressed INLINE, inside this string — this format has no `marks` array and no offsets (§8). A CommonMark-inline subset plus a small tag whitelist, and nothing else:\n `**bold**`, `*italic*`, `~~strikethrough~~`, `` `code` ``\n `[text](https://example.com)` — an external link\n `[text](anytype://object?objectId=)` — a link to an object in this space\n `\">text` — a decorated object reference (icon + name)\n `underline`, `text`, `text`\nEverything else is literal text: no block syntax, no images, no autolinks, no HTML beyond those tags. `\\n` is a soft line break within the block. A literal `*`, `` ` ``, `[`, `]`, `~`, `<` or `\\` in prose is backslash-escaped." + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "code" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "language": { + "type": "string" + }, + "text": { + "type": "string", + "description": "The block's text. Formatting is expressed INLINE, inside this string — this format has no `marks` array and no offsets (§8). A CommonMark-inline subset plus a small tag whitelist, and nothing else:\n `**bold**`, `*italic*`, `~~strikethrough~~`, `` `code` ``\n `[text](https://example.com)` — an external link\n `[text](anytype://object?objectId=)` — a link to an object in this space\n `\">text` — a decorated object reference (icon + name)\n `underline`, `text`, `text`\nEverything else is literal text: no block syntax, no images, no autolinks, no HTML beyond those tags. `\\n` is a soft line break within the block. A literal `*`, `` ` ``, `[`, `]`, `~`, `<` or `\\` in prose is backslash-escaped." + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "callout" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "icon": { + "$ref": "#/$defs/plainIcon" + }, + "color": { + "$ref": "#/$defs/color" + }, + "text": { + "type": "string", + "description": "The block's text. Formatting is expressed INLINE, inside this string — this format has no `marks` array and no offsets (§8). A CommonMark-inline subset plus a small tag whitelist, and nothing else:\n `**bold**`, `*italic*`, `~~strikethrough~~`, `` `code` ``\n `[text](https://example.com)` — an external link\n `[text](anytype://object?objectId=)` — a link to an object in this space\n `\">text` — a decorated object reference (icon + name)\n `underline`, `text`, `text`\nEverything else is literal text: no block syntax, no images, no autolinks, no HTML beyond those tags. `\\n` is a soft line break within the block. A literal `*`, `` ` ``, `[`, `]`, `~`, `<` or `\\` in prose is backslash-escaped." + } + } + } + }, + { + "if": { + "properties": { + "type": { + "enum": [ + "embed", + "equation" + ] + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "processor": { + "enum": [ + "latex", + "mermaid", + "chart", + "youtube", + "vimeo", + "soundcloud", + "google_maps", + "miro", + "figma", + "twitter", + "open_street_map", + "reddit", + "facebook", + "instagram", + "telegram", + "github_gist", + "codepen", + "bilibili", + "excalidraw", + "kroki", + "graphviz", + "sketchfab", + "image", + "drawio", + "spotify" + ] + }, + "text": { + "type": "string", + "description": "The block's text. Formatting is expressed INLINE, inside this string — this format has no `marks` array and no offsets (§8). A CommonMark-inline subset plus a small tag whitelist, and nothing else:\n `**bold**`, `*italic*`, `~~strikethrough~~`, `` `code` ``\n `[text](https://example.com)` — an external link\n `[text](anytype://object?objectId=)` — a link to an object in this space\n `\">text` — a decorated object reference (icon + name)\n `underline`, `text`, `text`\nEverything else is literal text: no block syntax, no images, no autolinks, no HTML beyond those tags. `\\n` is a soft line break within the block. A literal `*`, `` ` ``, `[`, `]`, `~`, `<` or `\\` in prose is backslash-escaped." + }, + "url": { + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "enum": [ + "file", + "image", + "video", + "audio", + "pdf" + ] + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "object_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "style": { + "enum": [ + "auto", + "link", + "embed" + ] + }, + "added_at": { + "type": "string" + }, + "hash": { + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "bookmark" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "url": { + "type": "string" + }, + "object_id": { + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "link" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "object_id": { + "type": "string" + }, + "card_style": { + "$ref": "#/$defs/cardStyle" + }, + "icon_size": { + "$ref": "#/$defs/iconSize" + }, + "description": { + "$ref": "#/$defs/linkDescription" + }, + "properties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "divider" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "style": { + "enum": [ + "line", + "dots" + ] + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "property" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "property": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$", + "description": "The property this block renders inline, by its document-facing spelling (`Due date`) — the same member name, and the same spelling, every other slot that names one property uses (a dataview's `properties[]` entry, a view's `columns[]`/`sorts[]`/`filters[]`). Spelled `key` before v0.41." + } + }, + "required": [ + "property" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "dataview" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "object_id": { + "type": "string" + }, + "is_collection": { + "type": "boolean" + }, + "source": { + "type": "array", + "items": { + "type": "string" + }, + "x-output-only": true + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/dataviewProperty" + } + }, + "views": { + "type": "array", + "items": { + "$ref": "#/$defs/view" + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "widget" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "layout": { + "$ref": "#/$defs/widgetLayout" + }, + "limit": { + "type": "integer", + "minimum": 0, + "maximum": 2147483647, + "description": "How many entries the widget shows. The whole int32 range ON PURPOSE, where the bundle index's flat widget caps the same fact at $defs/widgetListingLimit: a widget above that cap is not lifted into the index and travels as a full document, and this block is that fallback — bounding it here would invalidate the very document the refusal produces." + }, + "view_id": { + "type": "string" + }, + "auto_added": { + "type": "boolean" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "icon" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "name": { + "type": "string" + } + } + } + } + ] + }, + "tableColumn": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/tableInnerId" + }, + "width": { + "type": "number" + }, + "fields": { + "type": "object", + "x-output-only": true + } + }, + "additionalProperties": false + }, + "tableRow": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/tableInnerId" + }, + "is_header": { + "type": "boolean" + }, + "cells": { + "type": "array", + "items": { + "$ref": "#/$defs/tableCell" + } + } + }, + "additionalProperties": false + }, + "tableCell": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + }, + { + "allOf": [ + { + "$ref": "#/$defs/cellBlock" + } + ], + "properties": { + "id": false, + "indent": false + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/cellBlock" + }, + "minItems": 1 + } + ] + }, + "propertyFormat": { + "enum": [ + "text", + "number", + "select", + "multi_select", + "date", + "files", + "checkbox", + "url", + "email", + "phone", + "emoji", + "objects", + "properties", + "map" + ], + "description": "A property format, by NAME (§3): the public REST API vocabulary, plus names for the internal formats a store really carries — emoji, objects, properties, and map, which 72 production property documents hold (the bundled templatePlaceholders property). One list, referenced from every slot that states a format, so the three slots cannot drift apart. Includes `map`, which only a property document's own property_settings may state: it is the format of hidden system properties (templatePlaceholders is the only carrier — 72 production documents) and no authored property may declare it, so type_properties[] and a dataview's properties[] reference authorableFormat instead." + }, + "authorableFormat": { + "enum": [ + "text", + "number", + "select", + "multi_select", + "date", + "files", + "checkbox", + "url", + "email", + "phone", + "emoji", + "objects", + "properties" + ], + "description": "A property format an author may DECLARE (§3) — propertyFormat without `map`. `map` names the shape of a hidden system property's value, never something a type or a view declares: it occurs on 0 of 19,862 type_properties entries and 0 of 28,034 dataview property entries in a 38,061-document corpus, and its only carrier is the bundled, hidden templatePlaceholders. Splitting the vocabulary keeps a property document able to state what it is (§2d) while keeping the authorable slots — the ones a model writes, and the ones a generated per-type schema is built from — unable to invent one." + }, + "propertyDefinition": { + "type": "object", + "description": "ONE property, described in one shape, wherever this format describes a property (§2e). Exactly three homes reference it — a dictionary entry, a type document's property-definition list, and a property document's property_settings — and no fourth spelling of the concept may exist: a property document and a type's property list once described the same property in two vocabularies, and the raw one was a measured live trap (§15 #14). The shape states no `required` of its own and stays open: each home adds its own requirements, narrows what it must (an authored home may not declare `map`), refuses the members another surface already owns, and closes itself with unevaluatedProperties. The property's identity splits into two members on purpose: `property` is the document-facing spelling (a label, translated like every key slot), `internal_key` the stored id the app mints (export fidelity, never required from an author) — the word `key` used to carry both meanings and no longer exists in this shape.", + "properties": { + "property": { + "description": "The property's document-facing SPELLING (§3) — `Due date`, the same display-name spelling every other key slot writes, inverted through the document's own legend and the vocabulary in force. A spelling, never a stored id: the stored id is `internal_key`.", + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + }, + "internal_key": { + "description": "The property's STORED internal key, written by export for fidelity: the bson id the app minted for a custom property (`6a83296f61fab2265263ae34`), the camelCase key of a bundled one (`dueDate`). An author does not write this — an author cannot produce a correct one, and does not need to: identity is `property`, or `internal_key`, or a `name` the spelling derives from, and when a custom property states no internal key the import wiring MINTS a fresh one, exactly as the app does when a user creates a property. Taken verbatim on the way in — a stored key is always its own address (§3).", + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + }, + "name": { + "type": "string" + }, + "format": { + "$ref": "#/$defs/propertyFormat" + }, + "options": { + "type": "array", + "items": { + "$ref": "#/$defs/vocabularyOption" + } + }, + "object_types": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string", + "minLength": 1 + } + }, + "description": { + "type": "string" + }, + "include_time": { + "type": [ + "boolean", + "null" + ] + }, + "max_count": { + "type": "integer", + "minimum": 0, + "maximum": 2147483647 + }, + "readonly": { + "type": "boolean" + }, + "default_value": {} + } + }, + "typeProperty": { + "type": "object", + "description": "One entry of a type document's property-definition list (§2a): a propertyDefinition plus `section`, the ONE field that belongs to the type rather than the property — of 1,614 properties declared by 2+ types within one space, zero differ in anything else. The shared members are referenced, never restated; this layer only narrows: `format` to the authorable vocabulary, `object_types` to a real list (a type declares targets or omits the member — only a property document's stored value can hold a null). An entry must identify the property it declares: state a `property` (the document-facing spelling), or an `internal_key` (the stored id), or a `name` the spelling is derived from. A NAME ALONE IS ENOUGH — `{\"name\": \"Cooking Time\", \"format\": \"number\"}` declares a property spelled `Cooking Time`: the name IS the spelling. Prefer `property` or `name` when authoring: `internal_key` is an id the app mints (every exported example carries one like `6a83296f61fab2265263ae34`, because export writes the keys a real space holds), an author has no space to draw one from, and inventing one is never right — when a custom property states none, the import wiring mints a fresh internal key exactly as the app does when a user creates one. A spelling runs through the same resolution however it is stated, so a property named \"Due Date\" folds onto the bundled `dueDate` instead of minting a lookalike beside it.", + "allOf": [ + { + "$ref": "#/$defs/propertyDefinition" + } + ], + "properties": { + "format": { + "$ref": "#/$defs/authorableFormat" + }, + "object_types": { + "type": "array" + }, + "section": { + "enum": [ + "featured", + "hidden", + "file" + ] + } + }, + "unevaluatedProperties": false, + "anyOf": [ + { + "required": [ + "property" + ] + }, + { + "required": [ + "internal_key" + ] + }, + { + "required": [ + "name" + ] + } + ] + }, + "vocabularyOption": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "color": { + "$ref": "#/$defs/optionColor" + }, + "internal_key": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The option's stored key — a minted id, so an AUTHOR never writes one and export writes it only where it exists. It is the one thing about an option that is derivable from nothing: the name and colour say what the option MEANS, the array position says where it sits, and the option's api key is regenerated from the name by the app's own rule (measured: every one of 514 real option api keys is reproduced by that rule, so none needs to travel). Carrying the internal key is what lets a bundle state a vocabulary completely rather than only describe it." + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + ] + }, + "optionColor": { + "enum": [ + "grey", + "yellow", + "orange", + "red", + "pink", + "purple", + "blue", + "ice", + "teal", + "lime" + ] + }, + "dataviewProperty": { + "type": "object", + "description": "One entry of a dataview's `properties[]`: a property available to the block's views, with its format (§6.2). Declares the property; the views then refer to it — a column, sort or filter naming a property this list does not carry is still legal (the §3 chain answers its format).", + "properties": { + "property": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$", + "description": "The property this entry declares, by its document-facing spelling (`Due date`) — the same member name, and the same spelling, the view's `columns[]`, `sorts[]` and `filters[]` beside it use to refer to it. Spelled `key` before v0.41, twelve lines from siblings that spelled `property`, and each was a schema error in the other's position." + }, + "format": { + "$ref": "#/$defs/authorableFormat" + } + }, + "required": [ + "property" + ], + "additionalProperties": false + }, + "view": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "$ref": "#/$defs/viewType" + }, + "name": { + "type": "string" + }, + "group_by": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + }, + "cover_property": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + }, + "end_property": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + }, + "hide_icon": { + "type": "boolean" + }, + "card_size": { + "enum": [ + "small", + "medium", + "large" + ] + }, + "cover_fit": { + "type": "boolean" + }, + "colored_groups": { + "type": "boolean" + }, + "page_size": { + "type": "integer", + "minimum": 0, + "maximum": 2147483647 + }, + "default_template_id": { + "type": "string" + }, + "default_type_id": { + "type": "string" + }, + "wrap_content": { + "type": "boolean" + }, + "list_size": { + "enum": [ + "compact", + "regular" + ] + }, + "alternate_rows": { + "type": "boolean" + }, + "sorts": { + "type": "array", + "items": { + "$ref": "#/$defs/sort" + } + }, + "filters": { + "type": "array", + "items": { + "$ref": "#/$defs/filterNode" + } + }, + "columns": { + "type": "array", + "items": { + "$ref": "#/$defs/viewColumn" + } + }, + "groups": { + "type": "array", + "items": { + "$ref": "#/$defs/viewGroup" + }, + "x-output-only": true + }, + "object_orders": { + "type": "array", + "items": { + "$ref": "#/$defs/objectOrder" + }, + "x-output-only": true + } + }, + "additionalProperties": false + }, + "sort": { + "type": "object", + "properties": { + "property": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + }, + "direction": { + "enum": [ + "asc", + "desc", + "custom" + ] + }, + "custom_order": { + "type": "array" + }, + "empty_placement": { + "enum": [ + "start", + "end" + ] + }, + "include_time": { + "type": "boolean" + }, + "no_collate": { + "type": "boolean" + }, + "id": { + "type": "string", + "x-output-only": true + } + }, + "required": [ + "property" + ], + "additionalProperties": false + }, + "filterNode": { + "oneOf": [ + { + "type": "object", + "properties": { + "operator": { + "enum": [ + "and", + "or" + ] + }, + "filters": { + "type": "array", + "items": { + "$ref": "#/$defs/filterNode" + } + } + }, + "required": [ + "operator", + "filters" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "property": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$" + }, + "condition": { + "enum": [ + "equal", + "not_equal", + "greater", + "less", + "greater_or_equal", + "less_or_equal", + "contains", + "not_contains", + "in", + "not_in", + "empty", + "not_empty", + "all_in", + "not_all_in", + "exact_in", + "not_exact_in", + "exists" + ] + }, + "value": {}, + "date_preset": { + "enum": [ + "yesterday", + "today", + "tomorrow", + "last_week", + "current_week", + "next_week", + "last_month", + "current_month", + "next_month", + "number_of_days_ago", + "number_of_days_now", + "last_year", + "current_year", + "next_year" + ] + }, + "include_time": { + "type": "boolean" + }, + "nested_property": { + "type": "string", + "x-output-only": true + }, + "id": { + "type": "string", + "x-output-only": true + } + }, + "required": [ + "property" + ], + "additionalProperties": false + } + ] + }, + "viewColumn": { + "type": "object", + "description": "One column of a view: which property it shows, plus display facts (width, alignment, an aggregation). Column order is display order.", + "properties": { + "property": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\u0000-\u001f]+$", + "description": "The property this column shows, by its document-facing spelling — the same member name, and the same spelling, the sorts, filters and the dataview's `properties[]` entries use." + }, + "hidden": { + "type": "boolean" + }, + "width": { + "type": "integer", + "minimum": 0, + "maximum": 2147483647 + }, + "aggregation": { + "enum": [ + "count", + "count_value", + "count_distinct", + "count_empty", + "count_not_empty", + "percent_empty", + "percent_not_empty", + "sum", + "average", + "median", + "min", + "max", + "range" + ] + }, + "align": { + "$ref": "#/$defs/blockAlign" + } + }, + "required": [ + "property" + ], + "additionalProperties": false + }, + "viewGroup": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "hidden": { + "type": "boolean" + }, + "background_color": { + "type": "string" + } + }, + "additionalProperties": false + }, + "objectOrder": { + "type": "object", + "properties": { + "group_id": { + "type": "string" + }, + "object_ids": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "objectLayout": { + "enum": [ + "audio", + "basic", + "bookmark", + "chat_deprecated", + "chat_derived", + "collection", + "dashboard", + "date", + "devices", + "discussion", + "file", + "image", + "missing_object", + "note", + "notification", + "object_type", + "participant", + "pdf", + "profile", + "property", + "property_option", + "property_options_list", + "set", + "space", + "space_view", + "tag", + "todo", + "video" + ], + "description": "The closed set of object layouts, by NAME (§3). Validate refuses a name outside this set: a typo would import as a raw string onto a number-format detail, where every consumer reads it with an integer getter and silently sees `basic`. A raw NUMBER is accepted beside a name, because a stored value outside the vocabulary round-trips as its number rather than being lost." + }, + "viewType": { + "enum": [ + "table", + "list", + "gallery", + "kanban", + "calendar", + "graph" + ], + "description": "How a dataview view renders. One definition, shared by a view's own `type` and a type's `default_view`, so the two cannot drift into different vocabularies for one concept." + }, + "blockAlign": { + "enum": [ + "left", + "center", + "right", + "justify" + ], + "description": "Horizontal alignment, by NAME. One definition for one concept: a block's `align`, a view column's `align`, and the `layout_align` property value (§3) all draw from it. The property slot cannot $ref this definition — a property SPELLING is not fixed to its stored key, so the vocabulary there is enforced by the semantic pass on the RESOLVED key — but the vocabulary is this one: an unknown name on `layout_align` is a validation error, and a raw NUMBER is accepted beside a name, because a stored value outside the vocabulary round-trips as its number rather than being lost." + }, + "objectOrigin": { + "enum": [ + "api", + "bookmark", + "builtin", + "clipboard", + "drag_and_drop", + "import", + "none", + "sharing_extension", + "usecase", + "webclipper" + ], + "description": "How an object entered its space, by NAME — the vocabulary of the `origin` property value (§3). No slot $refs this definition, because a property SPELLING is not fixed to its stored key (a legend may rebind it): the vocabulary is enforced by the semantic pass on the RESOLVED key, and stated here so a reader of the schema can learn it. An unknown name is a validation error; a raw NUMBER is accepted beside a name, because a stored value outside the vocabulary round-trips as its number rather than being lost." + }, + "importType": { + "enum": [ + "csv", + "external", + "html", + "markdown", + "notion", + "obsidian", + "pb", + "txt" + ], + "description": "Which importer created an import- or usecase-originated object, by NAME — the vocabulary of the `import_type` property value (§3), enforced like $defs/objectOrigin by the semantic pass on the resolved key. Named or refused, never stored as a stray string: the underlying enum's ZERO is notion, so an unchecked string here used to read back as a false claim that the object came from Notion." + }, + "imageKind": { + "enum": [ + "automatically_added", + "basic", + "cover", + "icon" + ], + "description": "What an image was uploaded FOR, by NAME — the vocabulary of the `image_kind` property value (§3), enforced like $defs/objectOrigin by the semantic pass on the resolved key. Set on file objects. `automatically_added` marks an image a pipeline brought in rather than a person, and travels beside `is_hidden_discovery`, which is what a client actually filters on." + } + } +} diff --git a/pkg/lib/anyblockjson/schema/properties.schema.json b/pkg/lib/anyblockjson/schema/properties.schema.json new file mode 100644 index 0000000000..d97e185c1c --- /dev/null +++ b/pkg/lib/anyblockjson/schema/properties.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.anytype.io/anyblock/2/properties.schema.json", + "title": "AnyBlock JSON property dictionary", + "description": "One properties.json at the root of a bundle names every property the bundle's objects use — the dictionary (§2f). It is a sibling of index.json, not a section inside it: an index says where things are, a dictionary says what they mean. It replaces the ~9,500 property documents per account (kind \"property\") that restated the bundled table field for field (98% of installed copies are identical to it), and it is what lets a third-party reader interpret a backup WITHOUT shipping the bundled table: every entry carries its format, so a date is a date and an option name is an option name even to a reader that has never seen Anytype.", + "type": "object", + "required": [ + "version" + ], + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string", + "description": "Which of the format's three grammars this document follows — an object document, a bundle index, or a property dictionary. OPTIONAL and DECORATIVE for validity: `version` alone gates the format, so a stale or invented URL here never makes a document invalid (a bundle written against a later schema still reads). What it does do is DISPATCH: a reader handed a file with no filename to go by uses this to pick the grammar, matching on the trailing `object|index|properties.schema.json` and ignoring the version segment. Without it a reader falls back to the document's shape, which can only tell the grammars apart when the document happens to carry a member unique to one. Every document this format writes carries it, and a hand-written one is easier for a reader to place if it does too." + }, + "version": { + "const": 2 + }, + "installed": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "description": "The BUNDLED properties present in the space — presence, not definition. Spelled the way every other slot in the format spells a property: the display name from the shipped table, `Due date` and `Tag`, NOT the stored `dueDate`. A bundled name names its property uniquely — the codec pins that over the whole wire-reachable table — so a reader inverts it without a legend. A stored key is still accepted and read as itself, and the retired derived slugs (`due_date`) keep resolving through the forgiving fold. 98% of installed copies are field-identical to the bundled table, so a key list is the whole of what a restore needs: the reader reinstalls each key from its own bundled table. An installed copy that DIVERGES from the table (a rename, a changed is_hidden) additionally gets a full entry in `properties`, which overrides the table member for member. A key the reader's table does not know is skipped, not refused — it is a newer app's bundled property, and rejecting the dictionary for it would make every backup unreadable one app version back." + }, + "properties": { + "type": "array", + "items": { + "$ref": "#/$defs/dictionaryEntry" + }, + "description": "One propertyDefinition per property the bundle's objects actually REFERENCE — used-only, not everything installed: a space installs a median 125 bundled properties and uses 57, and the 68 nothing touches buy a reader nothing (the restore reinstalls the bundled table anyway). Keys are CANONICAL SPELLINGS: the display name from the shipped table for a bundled key, the stored key verbatim for a space-minted one — a pure function of the key, because this file carries no legend. The reader flow is object → spelling → (1) the document's property_internal_keys legend, (2) a verbatim match against a dictionary key, (3) the shipped name table over the dictionary's own keys, with the forgiving fold behind it for near-misses and pre-v0.48 derived-slug spellings. Step 3 carries most value slots, because §3 writes a legend line only for a spelling the bundled table does not bind. Look up, never transform: the name and the key say different words (`Creation date` / `createdDate`). No folder convention, no scanning (§2f)." + } + }, + "$defs": { + "dictionaryEntry": { + "description": "One dictionary entry: a propertyDefinition (§2e), referenced rather than restated. Its `property` is the entry's spelling: the display name for a bundled property (`Due date`), the stored key verbatim for a space-minted one — a bson id like `6a32d4856761631534b22f85`, from which nothing must ever be derived. Its `internal_key` is the stored key itself, written by export for fidelity and never required from an author. This is the third of the shape's three homes, beside a type document's property-definition entry and a property document's property_settings. The layer narrows `object_types` to a real array (only a property document's STORED value can hold a null; a dictionary describes, it does not mirror a store slot) and requires `format`, because self-sufficiency is the constraint that shapes the dictionary: an entry without a format is readable only by a reader shipping bundle/relations.json, which is exactly the dependence the dictionary exists to end. An entry must identify the property it declares: state a `property` (the spelling), or an `internal_key` (the stored id), or a `name` the spelling is derived from. A NAME ALONE IS ENOUGH — `{\"name\": \"Cooking Time\", \"format\": \"number\"}` declares a property spelled `Cooking Time`. Prefer `property` or `name` when authoring: `internal_key` is an id the app mints, an author has no space to draw one from, and inventing one is never right — when a custom property states none, the import wiring mints a fresh internal key exactly as the app does when a user creates one. A derived spelling runs through the same resolution as a written one, so a property named \"Due Date\" folds onto the bundled `dueDate` instead of minting a lookalike beside it.", + "allOf": [ + { + "$ref": "https://schemas.anytype.io/anyblock/2/object.schema.json#/$defs/propertyDefinition" + } + ], + "properties": { + "object_types": { + "type": "array" + } + }, + "required": [ + "format" + ], + "unevaluatedProperties": false, + "anyOf": [ + { + "required": [ + "property" + ] + }, + { + "required": [ + "internal_key" + ] + }, + { + "required": [ + "name" + ] + } + ] + } + } +} diff --git a/pkg/lib/anyblockjson/schemavocab_test.go b/pkg/lib/anyblockjson/schemavocab_test.go new file mode 100644 index 0000000000..3cf4a6c44d --- /dev/null +++ b/pkg/lib/anyblockjson/schemavocab_test.go @@ -0,0 +1,132 @@ +package anyblockjson + +// schemavocab_test.go — the published schema and the codec must state the +// SAME closed vocabularies (§11, §13). + +import ( + "encoding/json" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A JSON Schema that is looser than the validator is the worst of both +// worlds: an author or generator reading the schema produces documents the +// codec refuses, and a consumer trusting the schema accepts documents the +// codec would refuse. `type_settings.layout` and `default_view` were +// `{"type": ["string","number"]}` while Validate refused any name outside a +// closed set — a gap only reachable by reading the Go source, which §1 +// promises an author never has to do. +// +// This pins the two vocabularies against the tables the codec actually +// enforces, in BOTH directions, so neither can grow a member the other has +// not heard of. +// +// How this can fail: add a layout to the Go table for a new object kind and +// leave the schema alone — every document using it is refused by the +// published schema while the codec accepts it, and nothing else notices. +func TestSchemaVocabularies_MatchTheCodec(t *testing.T) { + var schema struct { + Defs map[string]struct { + Enum []string `json:"enum"` + } `json:"$defs"` + } + require.NoError(t, json.Unmarshal(schemaJSON, &schema)) + + for _, tc := range []struct { + def string + names map[string]bool + what string + }{ + {"objectLayout", keysOfLayouts(), "object layout"}, + {"viewType", keysOfViewTypes(), "dataview view type"}, + {"blockAlign", keysOfEnumNames(alignNames), "alignment"}, + {"objectOrigin", keysOfEnumNames(originNames), "object origin"}, + {"importType", keysOfEnumNames(importTypeNames), "import type"}, + {"imageKind", keysOfEnumNames(imageKindNames), "image kind"}, + } { + t.Run(tc.def, func(t *testing.T) { + got := append([]string(nil), schema.Defs[tc.def].Enum...) + require.NotEmpty(t, got, "$defs/%s must state the vocabulary", tc.def) + + want := make([]string, 0, len(tc.names)) + for n := range tc.names { + want = append(want, n) + } + sort.Strings(want) + sorted := append([]string(nil), got...) + sort.Strings(sorted) + + assert.Equal(t, want, sorted, + "the schema's %s vocabulary and the codec's disagree", tc.what) + }) + } +} + +// The two slots that name a view type share one definition rather than each +// restating six names — one concept, one spelling. +func TestSchemaVocabularies_OneViewTypeDefinition(t *testing.T) { + var schema struct { + Defs map[string]json.RawMessage `json:"$defs"` + } + require.NoError(t, json.Unmarshal(schemaJSON, &schema)) + + var view struct { + Properties struct { + Type struct { + Ref string `json:"$ref"` + Enum []string `json:"enum"` + } `json:"type"` + } `json:"properties"` + } + require.NoError(t, json.Unmarshal(schema.Defs["view"], &view)) + assert.Equal(t, "#/$defs/viewType", view.Properties.Type.Ref) + assert.Empty(t, view.Properties.Type.Enum, "a second copy of the vocabulary is a place to drift") +} + +func keysOfLayouts() map[string]bool { + return keysOfEnumNames(layoutNames) +} + +func keysOfViewTypes() map[string]bool { + return keysOfEnumNames(viewTypeNames) +} + +func keysOfEnumNames[T comparable](e enumNames[T]) map[string]bool { + out := map[string]bool{} + for n := range e.toVal { + out[n] = true + } + return out +} + +// The three slots that spell an alignment share one definition rather than +// each restating four names — one concept, one spelling: a block's `align`, +// a view column's `align`, and (by the semantic pass, which is the only +// place a property slot's vocabulary CAN bind — a property spelling is not +// fixed to its stored key) the `layout_align` property value. +func TestSchemaVocabularies_OneAlignDefinition(t *testing.T) { + var schema struct { + Defs map[string]json.RawMessage `json:"$defs"` + } + require.NoError(t, json.Unmarshal(schemaJSON, &schema)) + + for _, tc := range []struct{ def, member string }{ + {"blockCore", "align"}, + {"viewColumn", "align"}, + } { + var node struct { + Properties map[string]struct { + Ref string `json:"$ref"` + Enum []string `json:"enum"` + } `json:"properties"` + } + require.NoError(t, json.Unmarshal(schema.Defs[tc.def], &node)) + assert.Equal(t, "#/$defs/blockAlign", node.Properties[tc.member].Ref, + "%s.%s must share the one alignment definition", tc.def, tc.member) + assert.Empty(t, node.Properties[tc.member].Enum, + "a second copy of the vocabulary is a place to drift") + } +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/attribution_test.go b/pkg/lib/anyblockjson/snapshotdiff/attribution_test.go new file mode 100644 index 0000000000..bc3cdf7b4f --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/attribution_test.go @@ -0,0 +1,79 @@ +package snapshotdiff + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +const participantId = "_participant_bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq_30afw2fe3tvff_AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA" + +// Every object in a real account carries a `creator`, and after a round trip +// through this format none of them does: export writes the member's NAME and +// import drops the key (§3). A comparator that did not know would report data +// loss on all 36,966 of them — the same shape as the 1,344 false failures the +// recommended-list normalization produced. +// +// This is inherited rather than restated: Compare skips +// anyblockjson.InternalPropertyKeys(), which the attribution keys join. The +// point of pinning it here is that the inheritance is not obvious — the keys +// are exempt from the deny rule, so "internal" and "refused" have come apart +// — and a future edit that took them off the internal list would make the +// sweep unusable with nothing else failing. +// +// How this can fail: make the comparator compare the attribution keys — +// through InternalPropertyKeys(), which is what it reads — and both cases +// report a changed/lost detail. +// +// NOT via `derivedAttributionProperties` in strippedDetailKeys(): an audit +// deleted that loop and all four packages stayed green, because both keys are +// already in bundle.LocalAndDerivedRelationKeys and neither is in +// propertiesKeptOnExport, so the loop is inert today. It is defence for the +// day one of them moves onto the keep-list — not the mechanism this test +// pins, and naming it here sent a reader looking in the wrong place. +func TestCompare_AttributionKeysAreNotDataLoss(t *testing.T) { + t.Run("creator lost to the round trip is not reported", func(t *testing.T) { + // given + orig := snapshot(map[string]*types.Value{ + "name": str("Doc"), + "creator": str(participantId), + }) + got := snapshot(map[string]*types.Value{"name": str("Doc")}) + + // when + diff := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + assert.Empty(t, diff) + }) + + t.Run("lastModifiedBy likewise, in both directions", func(t *testing.T) { + // given + orig := snapshot(map[string]*types.Value{"lastModifiedBy": str(participantId)}) + got := snapshot(map[string]*types.Value{"lastModifiedBy": str("someone-else")}) + + // when + diff := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + assert.Empty(t, diff) + }) + + t.Run("a user-chosen participant reference is still compared", func(t *testing.T) { + // given the control: assignee is source: details, and losing it IS loss + orig := snapshot(map[string]*types.Value{"assignee": str(participantId)}) + got := snapshot(map[string]*types.Value{}) + + // when + diff := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + require.Len(t, diff, 1) + assert.Contains(t, diff[0], `detail "assignee" changed`) + }) +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/iconcover_test.go b/pkg/lib/anyblockjson/snapshotdiff/iconcover_test.go new file mode 100644 index 0000000000..0d65bf4881 --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/iconcover_test.go @@ -0,0 +1,95 @@ +package snapshotdiff + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func text(s string) *types.Value { return &types.Value{Kind: &types.Value_StringValue{StringValue: s}} } +func number(n float64) *types.Value { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: n}} +} + +// §2b lifted nine hidden keys into the typed `icon` and `cover` envelope +// fields, and a source whose stored value is EMPTY is not a source — so a key +// present and empty comes back absent. Measured over a 36 966-object account: +// 2 307 objects produce 3 038 such findings, nearly double the +// recommended-list noise that buried a previous sweep. +// +// The suppression is scoped to that step and nothing else. The rows below pin +// both halves, because a suppression that grew to the whole KEY would go blind +// to the 33 objects in that same account whose cover really is lost — an +// absolute filesystem path a Notion import wrote into coverId, which the typed +// field cannot carry. +// +// How this can fail: delete the isDroppedEmptyIconCover call in Compare and +// the first three rows report a finding each; widen it to ignore the key +// outright and the last three stop reporting. +func TestCompare_DroppedEmptyIconCoverIsNormalization(t *testing.T) { + for name, tc := range map[string]struct { + orig, got map[string]*types.Value + report []string + }{ + "an empty emoji beside a real image": { + orig: map[string]*types.Value{"iconEmoji": text(""), "iconImage": list("bafyimage")}, + got: map[string]*types.Value{"iconImage": list("bafyimage")}, + }, + "an empty image list beside a real emoji": { + orig: map[string]*types.Value{"iconImage": list(), "iconEmoji": text("📕")}, + got: map[string]*types.Value{"iconEmoji": text("📕")}, + }, + "framing present and zero on a gradient cover": { + orig: map[string]*types.Value{ + "coverId": text("pinkOrange"), "coverType": number(3), + "coverScale": number(0), "coverX": number(0), "coverY": number(0)}, + got: map[string]*types.Value{"coverId": text("pinkOrange"), "coverType": number(3)}, + }, + + "a cover that really was lost": { + orig: map[string]*types.Value{"coverId": text("/var/folders/j0/T/leaked.png"), "coverType": number(1)}, + got: map[string]*types.Value{}, + // both keys report: the pair is one cover, and losing it loses + // both halves. 33 objects in the corpus produce these 66 findings + report: []string{"coverId", "coverType"}, + }, + "an emoji that really was lost": { + orig: map[string]*types.Value{"iconEmoji": text("📕")}, + got: map[string]*types.Value{}, + report: []string{"iconEmoji"}, + }, + "framing that really was lost": { + orig: map[string]*types.Value{"coverY": number(-0.25)}, + got: map[string]*types.Value{}, + report: []string{"coverY"}, + }, + } { + t.Run(name, func(t *testing.T) { + found := Compare(snapWith(tc.orig), snapWith(tc.got), model.SmartBlockType_Page, anyblockjson.Options{}) + if len(tc.report) == 0 { + assert.Empty(t, found) + return + } + require.Len(t, found, len(tc.report)) + for i, want := range tc.report { + assert.Contains(t, found[i], want) + } + }) + } +} + +// The suppression asks the format which values are sources, rather than +// deciding for itself. Two lists would drift, which is exactly how this +// comparator once reported 10 378 false data-loss issues. +func TestCompare_TheLiftedSetIsTheFormatsOwn(t *testing.T) { + assert.Len(t, anyblockjson.LiftedPropertyKeys(), 9) + assert.True(t, anyblockjson.DroppedEmptyIconCover("iconEmoji", text(""))) + assert.False(t, anyblockjson.DroppedEmptyIconCover("iconEmoji", text("📕"))) + assert.False(t, anyblockjson.DroppedEmptyIconCover("name", text("")), + "a key outside the lift list is nothing to do with this rule") +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/missingref_test.go b/pkg/lib/anyblockjson/snapshotdiff/missingref_test.go new file mode 100644 index 0000000000..0bcf6699a2 --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/missingref_test.go @@ -0,0 +1,157 @@ +package snapshotdiff + +// missingref_test.go — the missing-reference rule (§9) reaches this +// comparator in the SAME change that taught export: an objects/files value +// entry or an `object_types` entry naming an object the space does not hold +// is dropped by design, and the comparison applies the format's own +// predicate (DroppedMissingObjectRef) to both sides. Without that, every +// document the rule touches would report its dropped entries as data loss — +// the drift class that once produced 1,344 false failures in one sweep. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// testCid mints a REAL content id — the §9 shape gate only lets the +// existence question reach CID-shaped entries. +func testCid(seed string) string { + sum, err := mh.Sum([]byte(seed), mh.SHA2_256, -1) + if err != nil { + panic(err) + } + return cid.NewCidV1(cid.DagCBOR, sum).String() +} + +var ( + liveCid = testCid("live") + deadCid = testCid("dead") +) + +// existenceStore is the storeresolver shape reduced to the object-namespace +// pair: ids in the set exist, everything else does not. +type existenceStore map[string]bool + +func (m existenceStore) ObjectName(string) (string, bool) { return "", false } + +func (m existenceStore) ObjectExists(id string) (exists, known bool) { + return m[id], true +} + +func missingRefOpts() anyblockjson.Options { + return anyblockjson.Options{ + ResolveFormat: func(key domain.RelationKey) (model.RelationFormat, bool) { + if key == "related" { + return model.RelationFormat_object, true + } + return 0, false + }, + ResolveObjectNames: existenceStore{liveCid: true}, + } +} + +func pageSnap(related *types.Value) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": text("obj1"), + "name": text("Host"), + "related": related, + }}, + Blocks: []*model.Block{{ + Id: "obj1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + } +} + +// The real shape of the sweep: an actual round trip through the codec, with +// the SAME options handed to export, import and Compare — exactly how +// cmd/anyblockroundtrip wires it. The dropped entries must not report. +// +// How this can fail: teach export the drop without this file's normalization +// and the objects-format row reports `detail "related" changed` on every +// object carrying a dangling reference — ~990 documents in the last corpus. +func TestCompare_MissingReferenceDropIsNotLoss(t *testing.T) { + t.Run("objects-format value through a real round trip", func(t *testing.T) { + // given + opts := missingRefOpts() + orig := pageSnap(list(liveCid, deadCid, missingObjectSentinel)) + data, err := anyblockjson.Marshal(model.SmartBlockType_Page, orig, opts) + require.NoError(t, err) + sbType, got, err := anyblockjson.Unmarshal(data, opts) + require.NoError(t, err) + + // when + diffs := Compare(orig, got, sbType, opts) + + // then + assert.Empty(t, diffs, "a dropped-by-design entry is a normalization, not loss") + }) + + t.Run("object_types on a property document through a real round trip", func(t *testing.T) { + // given — the corpus shape: an object id naming nothing beside a + // live type id and a legacy bare key + opts := missingRefOpts() + opts.ResolveProperties = relTypeResolver{} + orig := relationSnap(map[string]*types.Value{ + "relationFormat": number(float64(model.RelationFormat_object)), + "relationFormatObjectTypes": list("typeid-page", deadCid, "wine"), + }) + data, err := anyblockjson.Marshal(model.SmartBlockType_STRelation, orig, opts) + require.NoError(t, err) + sbType, got, err := anyblockjson.Unmarshal(data, opts) + require.NoError(t, err) + + // when + diffs := Compare(orig, got, sbType, opts) + + // then + assert.Empty(t, diffs) + }) +} + +// The suppression is scoped exactly to what export drops: a LIVE entry that +// vanishes still reports, and with no existence capability in the options +// nothing is suppressed — export dropped nothing, so a shorter list really +// is loss. +func TestCompare_MissingReferenceScopeStaysTight(t *testing.T) { + t.Run("a live entry that vanishes still reports", func(t *testing.T) { + // given + opts := missingRefOpts() + orig := pageSnap(list(liveCid, deadCid)) + got := pageSnap(list()) + + // when + diffs := Compare(orig, got, model.SmartBlockType_Page, opts) + + // then + require.Len(t, diffs, 1) + assert.Contains(t, diffs[0], `detail "related" changed`) + }) + + t.Run("no capability in the options: a dropped entry is loss", func(t *testing.T) { + // given — the gate the export side has, mirrored: absence of an + // answer is not evidence of absence, so a comparator wired without + // the store must not excuse a missing entry + opts := missingRefOpts() + opts.ResolveObjectNames = nil + orig := pageSnap(list(liveCid, deadCid)) + got := pageSnap(list(liveCid)) + + // when + diffs := Compare(orig, got, model.SmartBlockType_Page, opts) + + // then + require.Len(t, diffs, 1) + assert.Contains(t, diffs[0], `detail "related" changed`) + }) +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/objecttypes_test.go b/pkg/lib/anyblockjson/snapshotdiff/objecttypes_test.go new file mode 100644 index 0000000000..267f6ff0a9 --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/objecttypes_test.go @@ -0,0 +1,234 @@ +package snapshotdiff + +// Compare used to read only details and text, so the TYPE namespace was +// invisible to it: a 36 808-object production sweep could not have caught a +// type substitution, and every claim about type-key correctness rested on +// synthetic tests. These tests pin both directions of the new axis — a +// rebinding must be reported, the documented truncation must not — because a +// comparator that cries wolf on normal exports gets ignored, and one that +// misses a rebinding is why the axis was added. + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// customTypeKey is a space-minted (bson) type key — the stored key a +// spelling binds to when a vocabulary is in play. +const customTypeKey = "69bbfc78877a91b1d12d1a7c" + +func typed(objectTypes ...string) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ObjectTypes: objectTypes} +} + +func TestCompareObjectTypes(t *testing.T) { + // --- must be reported: the type list means something different now --- + + t.Run("a rebound type is reported", func(t *testing.T) { + // given: the round trip landed on a different type entirely + got := Compare(typed("ot-task"), typed("ot-page"), model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + require.Len(t, got, 1) + assert.Equal(t, `object type [0] changed: "task" -> "page"`, got[0]) + }) + + t.Run("a rebound template target is reported", func(t *testing.T) { + // given: position 1 is modelled for a template, so a change there is + // the template silently moving to another type + got := Compare(typed("ot-template", "ot-task"), typed("ot-template", "ot-page"), model.SmartBlockType_Template, anyblockjson.Options{}) + + // then + require.Len(t, got, 1) + assert.Equal(t, `object type [1] changed: "task" -> "page"`, got[0]) + }) + + t.Run("a lost template target is reported", func(t *testing.T) { + got := Compare(typed("ot-template", "ot-task"), typed("ot-template"), + model.SmartBlockType_Template, anyblockjson.Options{}) + + require.Len(t, got, 1) + assert.Equal(t, `object type [1] lost: "task"`, got[0]) + }) + + // The second slot belongs to a TEMPLATE, not to a list that happens to + // start with the template key (§2, v0.22). Keyed off the data, this diff + // called the same two losses opposite things: the pair above was drift, + // and the identical pair below — a template whose types do not begin with + // the template key, which export really did truncate — was "by design". + t.Run("a lost target is reported whatever the template's first type is", func(t *testing.T) { + got := Compare(typed("ot-task", "ot-page"), typed("ot-task"), + model.SmartBlockType_Template, anyblockjson.Options{}) + + require.Len(t, got, 1) + assert.Equal(t, `object type [1] lost: "page"`, got[0]) + }) + + t.Run("and a non-template's second type is still truncation", func(t *testing.T) { + got := Compare(typed("ot-template", "ot-task"), typed("ot-template"), + model.SmartBlockType_Page, anyblockjson.Options{}) + + assert.Empty(t, got, "a page has one slot, whatever its first type is called") + }) + + t.Run("a lost type is reported", func(t *testing.T) { + got := Compare(typed("ot-task"), typed(), model.SmartBlockType_Page, anyblockjson.Options{}) + + require.Len(t, got, 1) + assert.Equal(t, `object type [0] lost: "task"`, got[0]) + }) + + t.Run("a type the round trip invented is reported", func(t *testing.T) { + // given: export models one slot here, so a second entry did not come + // from the original + got := Compare(typed("ot-page"), typed("ot-page", "ot-task"), model.SmartBlockType_Page, anyblockjson.Options{}) + + require.Len(t, got, 1) + assert.Equal(t, `object type [1] added: "task"`, got[0]) + }) + + // --- must NOT be reported: documented normalization --- + + t.Run("truncation past the modelled positions is not drift", func(t *testing.T) { + // given: a non-template has one modelled slot; ot-task has nowhere to + // be written and is dropped by design + got := Compare(typed("ot-page", "ot-task"), typed("ot-page"), model.SmartBlockType_Page, anyblockjson.Options{}) + + assert.Empty(t, got) + }) + + t.Run("a template's third type is truncated, not lost", func(t *testing.T) { + got := Compare(typed("ot-template", "ot-task", "ot-page"), typed("ot-template", "ot-task"), model.SmartBlockType_Template, anyblockjson.Options{}) + + assert.Empty(t, got) + }) + + t.Run("a keyless entry drops and the survivors close ranks", func(t *testing.T) { + // given: "ot-" names no type, so export drops it with a warning and + // ot-task moves up into the `type` slot + got := Compare(typed("ot-", "ot-task"), typed("ot-task"), model.SmartBlockType_Page, anyblockjson.Options{}) + + assert.Empty(t, got) + }) + + t.Run("a keyless entry between good ones drops", func(t *testing.T) { + got := Compare(typed("ot-template", "ot-", "ot-task"), typed("ot-template", "ot-task"), model.SmartBlockType_Template, anyblockjson.Options{}) + + assert.Empty(t, got) + }) + + t.Run("an empty template target drops", func(t *testing.T) { + got := Compare(typed("ot-template", "ot-"), typed("ot-template"), model.SmartBlockType_Template, anyblockjson.Options{}) + + assert.Empty(t, got) + }) + + t.Run("a keyless entry taking its sibling with it IS reported", func(t *testing.T) { + // given: the collateral-damage bug envelopeTypeTerms used to have — a + // keyless entry omitted the `type` slot, which made template_for + // inexpressible, so the good sibling died beside its bad neighbour. + // The comparator has to see that, or the sweep cannot catch a + // regression of it. + got := Compare(typed("ot-", "ot-task"), typed(), model.SmartBlockType_Page, anyblockjson.Options{}) + + require.Len(t, got, 1) + assert.Equal(t, `object type [0] lost: "task"`, got[0]) + }) + + t.Run("the ot- prefix is normalized on both sides", func(t *testing.T) { + // given: legacy rows hold a bare key; import always writes it prefixed + got := Compare(typed("task"), typed("ot-task"), model.SmartBlockType_Page, anyblockjson.Options{}) + + assert.Empty(t, got) + }) + + t.Run("duplicates round-trip and are not drift", func(t *testing.T) { + got := Compare(typed("ot-template", "ot-template"), typed("ot-template", "ot-template"), + model.SmartBlockType_Template, anyblockjson.Options{}) + + assert.Empty(t, got) + }) + + t.Run("no types on either side is not drift", func(t *testing.T) { + got := Compare(typed(), typed(), model.SmartBlockType_Page, anyblockjson.Options{}) + + assert.Empty(t, got) + }) +} + +// divergentVocabulary is the production defect in miniature: one reader binds +// the spelling "Task" to a space-minted type, another to the bundled one. +// That is the disagreement the `type_internal_keys` legend exists to close +// (§3), and the only way a real export can come back on a different type — +// so a sweep that can see it can see the class. +type divergentVocabulary struct { + anyblockjson.BundledKeyVocabulary +} + +func (divergentVocabulary) TypeKey(spelling string) (string, bool) { + if spelling == "Task" { + return customTypeKey, true + } + return anyblockjson.BundledKeyVocabulary{}.TypeKey(spelling) +} + +// The unit cases above hand-build the two sides. This one drives the real +// codec, so the comparator is pinned against what Marshal/Unmarshal actually +// do rather than against my model of them. +func TestCompareObjectTypes_ThroughTheCodec(t *testing.T) { + t.Run("a reader that binds the spelling elsewhere is caught", func(t *testing.T) { + // given: exported by a package-only reader, read back by one whose + // vocabulary binds `task` to a space-minted type + orig := typed("ot-task") + data, err := anyblockjson.Marshal(model.SmartBlockType_Page, orig, anyblockjson.Options{}) + require.NoError(t, err) + require.NotContains(t, string(data), "type_internal_keys", + "the fixture only bites while the document carries no legend to invert the spelling") + + // when + _, back, err := anyblockjson.Unmarshal(data, anyblockjson.Options{Keys: divergentVocabulary{}}) + require.NoError(t, err) + require.Equal(t, []string{"ot-" + customTypeKey}, back.ObjectTypes, + "the fixture must actually rebind, or the assertion below is vacuous") + + // then + got := Compare(orig, back, model.SmartBlockType_Page, anyblockjson.Options{}) + require.Len(t, got, 1) + assert.Equal(t, `object type [0] changed: "task" -> "`+customTypeKey+`"`, got[0]) + }) + + t.Run("an honest round trip of every shape reports nothing", func(t *testing.T) { + for _, c := range []struct { + sbType model.SmartBlockType + types []string + }{ + {model.SmartBlockType_Page, []string{"ot-page", "ot-task"}}, + {model.SmartBlockType_Page, []string{"ot-", "ot-task"}}, + {model.SmartBlockType_Page, []string{"ot-"}}, + {model.SmartBlockType_Page, []string{"ot-" + customTypeKey}}, + {model.SmartBlockType_Template, []string{"ot-template", "ot-task", "ot-page"}}, + {model.SmartBlockType_Template, []string{"ot-template", "ot-"}}, + {model.SmartBlockType_Template, []string{"ot-template", "ot-", "ot-task"}}, + {model.SmartBlockType_Template, []string{"ot-", "ot-template", "ot-task"}}, + // a template whose types do NOT begin with the template key. + // Until v0.22 export dropped ot-page here and this diff agreed + // that the drop was by design, so the loss was invisible on both + // sides at once. + {model.SmartBlockType_Template, []string{"ot-task", "ot-page"}}, + {model.SmartBlockType_STType, []string{"ot-objectType"}}, + } { + orig := typed(c.types...) + data, err := anyblockjson.Marshal(c.sbType, orig, anyblockjson.Options{}) + require.NoError(t, err) + _, back, err := anyblockjson.Unmarshal(data, anyblockjson.Options{}) + require.NoError(t, err) + + assert.Empty(t, Compare(orig, back, c.sbType, anyblockjson.Options{}), c.types) + } + }) +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/omittedrelation_test.go b/pkg/lib/anyblockjson/snapshotdiff/omittedrelation_test.go new file mode 100644 index 0000000000..0808c2df19 --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/omittedrelation_test.go @@ -0,0 +1,171 @@ +package snapshotdiff + +// omittedrelation_test.go pins the comparator's side of the §2f omission: a +// bundled-identical relation document travels as an `installed` key, and +// what comes back is the reader's reconstruction from the bundled table. +// The two skips that trip needs — install artifacts absent, definition +// defaults stamped — are scoped to snapshots the omission predicate itself +// admits, so the ordinary document round trip keeps its full sensitivity. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// omittableCopy is a field-identical installed copy of the bundled dueDate, +// carrying the install provenance a real copy does. +func omittableCopy(t *testing.T) *model.SmartBlockSnapshotBase { + t.Helper() + det, ok := anyblockjson.InstalledRelationDetails("dueDate", anyblockjson.Options{}) + require.True(t, ok) + det.Fields["createdDate"] = &types.Value{Kind: &types.Value_NumberValue{NumberValue: 1700000000}} + det.Fields["origin"] = &types.Value{Kind: &types.Value_NumberValue{NumberValue: 2}} + det.Fields["apiObjectKey"] = &types.Value{Kind: &types.Value_StringValue{StringValue: "due_date"}} + return &model.SmartBlockSnapshotBase{Details: det} +} + +// reconstruction is what the reader builds from the `installed` key: the +// bundled table's facts and nothing of the install. +func reconstruction(t *testing.T) *model.SmartBlockSnapshotBase { + t.Helper() + det, ok := anyblockjson.InstalledRelationDetails("dueDate", anyblockjson.Options{}) + require.True(t, ok) + return &model.SmartBlockSnapshotBase{Details: det} +} + +// Across the omission trip the install artifacts come back absent — +// re-stamped by the next install — and that is normalization, not loss. +// +// How this can fail: remove the RelationInstallArtifactKey skip from +// Compare's orig-key loop, and createdDate/origin/apiObjectKey all report +// as changed-to-absent. +func TestCompare_OmittedRelationArtifactsComeBackAbsent(t *testing.T) { + diffs := Compare(omittableCopy(t), reconstruction(t), model.SmartBlockType_STRelation, anyblockjson.Options{}) + assert.Empty(t, diffs) +} + +// The reconstruction states the WHOLE definition, so a member the copy +// never stored arrives as its explicit empty default. Absent and empty say +// the same thing for a definition member with a defined default; a +// NON-empty invented member still reports. +// +// How this can fail: remove the InstallStampedDefault skip from the +// added-details loop (the stamped empty default reports as added), or widen +// it past empty values (the invented-name case goes green and a +// reconstruction bug ships as normalization). +func TestCompare_OmittedRelationStampedDefaults(t *testing.T) { + t.Run("a stamped empty default is not an addition", func(t *testing.T) { + orig := omittableCopy(t) + delete(orig.Details.Fields, "isHidden") // the copy never stored it + delete(orig.Details.Fields, "relationFormatObjectTypes") + diffs := Compare(orig, reconstruction(t), model.SmartBlockType_STRelation, anyblockjson.Options{}) + assert.Empty(t, diffs) + }) + t.Run("an invented non-empty member still reports", func(t *testing.T) { + orig := omittableCopy(t) + delete(orig.Details.Fields, "description") + got := reconstruction(t) + got.Details.Fields["description"] = &types.Value{Kind: &types.Value_StringValue{StringValue: "invented"}} + diffs := Compare(orig, got, model.SmartBlockType_STRelation, anyblockjson.Options{}) + assert.NotEmpty(t, diffs) + }) +} + +// Both skips are SCOPED to snapshots the omission predicate admits: on a +// divergent copy — one whose document is kept, so every key must survive — +// a missing install artifact is still loss. +// +// How this can fail: drop the `omittable` guard from either skip, and the +// comparator stops seeing real artifact-key loss on every kept relation +// document in the corpus. +func TestCompare_KeptRelationDocumentKeepsFullSensitivity(t *testing.T) { + orig := omittableCopy(t) + orig.Details.Fields["name"] = &types.Value{Kind: &types.Value_StringValue{StringValue: "End Date"}} // divergent: kept + got := reconstruction(t) + got.Details.Fields["name"] = &types.Value{Kind: &types.Value_StringValue{StringValue: "End Date"}} + // createdDate/origin/apiObjectKey are in orig and not in got + diffs := Compare(orig, got, model.SmartBlockType_STRelation, anyblockjson.Options{}) + assert.NotEmpty(t, diffs, "on a kept document a missing artifact key is loss, not normalization") +} + +// The artifact skip must never swallow a DEFINITION member: an omittable +// original whose name goes missing on the way back is loss, whatever else +// the trip may drop. +// +// How this can fail: add a definition key (name, relationFormat, …) to +// relationInstallArtifactKeys — this is the admission test running in +// reverse, the §2a discipline. +func TestCompare_OmittedRelationDefinitionLossStillReports(t *testing.T) { + got := reconstruction(t) + delete(got.Details.Fields, "name") + diffs := Compare(omittableCopy(t), got, model.SmartBlockType_STRelation, anyblockjson.Options{}) + assert.NotEmpty(t, diffs) +} + +// property_settings' object_types round-trips by type KEY, and legacy data +// mixes spellings: 27 corpus relations store a bare type key where the +// store speaks object ids, and import writes the id back — the SAME type, +// respelled. The comparator normalizes both sides to keys through the +// TypeResolver, exactly as it does for the recommended lists one namespace +// over, so a respelling is silent and a REBINDING still reports. +// +// How this can fail: drop the relationTargetsDetailKey arm from detailEqual +// (the respelled case reports as loss — the 27 false findings come back), +// or normalize one side only (the rebound case goes green and a genuine +// target substitution ships as normalization). +func TestCompare_RelationTargetsCompareByTypeKey(t *testing.T) { + tr := &targetsResolver{idToKey: map[string]string{"bafyderivedgoal": "goal", "bafyderivedtask": "task"}} + opts := anyblockjson.Options{ResolveProperties: tr} + targets := func(entries ...string) *model.SmartBlockSnapshotBase { + vals := make([]*types.Value, 0, len(entries)) + for _, e := range entries { + vals = append(vals, &types.Value{Kind: &types.Value_StringValue{StringValue: e}}) + } + return &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "relationFormatObjectTypes": {Kind: &types.Value_ListValue{ + ListValue: &types.ListValue{Values: vals}}}, + }}} + } + + t.Run("a respelled target is the same type", func(t *testing.T) { + diffs := Compare(targets("goal"), targets("bafyderivedgoal"), model.SmartBlockType_STRelation, opts) + assert.Empty(t, diffs) + }) + t.Run("a rebound target still reports", func(t *testing.T) { + diffs := Compare(targets("goal"), targets("bafyderivedtask"), model.SmartBlockType_STRelation, opts) + assert.NotEmpty(t, diffs) + }) + t.Run("without the capability the raw comparison stands", func(t *testing.T) { + // §2d: verbatim both ways without a resolver, so equal stays equal + diffs := Compare(targets("goal"), targets("goal"), model.SmartBlockType_STRelation, anyblockjson.Options{}) + assert.Empty(t, diffs) + }) +} + +// targetsResolver is the TypeResolver capability over a fixed id table. +type targetsResolver struct{ idToKey map[string]string } + +func (r *targetsResolver) PropertyById(id string) (anyblockjson.PropertyDefinition, bool) { + return anyblockjson.PropertyDefinition{}, false +} +func (r *targetsResolver) PropertyId(def anyblockjson.PropertyDefinition) (string, bool) { + return "", false +} +func (r *targetsResolver) TypeKeyById(id string) (string, bool) { + k, ok := r.idToKey[id] + return k, ok +} +func (r *targetsResolver) TypeIdByKey(key string) (string, bool) { + for id, k := range r.idToKey { + if k == key { + return id, true + } + } + return "", false +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/participantprovenance_test.go b/pkg/lib/anyblockjson/snapshotdiff/participantprovenance_test.go new file mode 100644 index 0000000000..cc16072451 --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/participantprovenance_test.go @@ -0,0 +1,58 @@ +package snapshotdiff + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// A participant document does not carry `created_date` (§3): the stored +// value is time.Now() stamped on every cold build — measured across a +// 1,164-document double-export, the ONLY kind that drifted (22/22) and the +// ONLY field (created_date) — so export omits it whatever it holds, and the +// comparator learns the rule in the same commit through the format's OWN +// predicate. This is the standing rule for every drop: the last comparator +// that learned about a drop late reported 1,344 false failures in one +// sweep, and this one would report one per corpus participant — 2,492. +// +// How this can fail: teach export the drop without wiring +// DroppedParticipantProvenanceKey here (the first case reports created_date +// lost on every participant); widen the suppression past the predicate's +// kind scope (the page case goes quiet on real loss); or suppress a got +// side that still CARRIES the key (the absent-only scoping is what keeps a +// wrong value reportable). +func TestParticipantProvenance_DropIsNormalizationOnParticipants(t *testing.T) { + orig := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": str2("AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA"), + "name": str2("Roman"), + "createdDate": num2(1756180000), + "lastModifiedDate": num2(1700000000), + }}, + } + + t.Run("the documented drop is silent, through the real round trip", func(t *testing.T) { + data, err := anyblockjson.Marshal(model.SmartBlockType_Participant, orig, anyblockjson.Options{}) + require.NoError(t, err) + _, got, err := anyblockjson.Unmarshal(data, anyblockjson.Options{}) + require.NoError(t, err) + + found := Compare(orig, got, model.SmartBlockType_Participant, anyblockjson.Options{}) + assert.Empty(t, found, "the drop is the format's decision, not loss") + }) + + t.Run("the same key vanishing from a page still reports", func(t *testing.T) { + got := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": str2("bafypage"), "name": str2("Roman"), "lastModifiedDate": num2(1700000000), + }}, + } + found := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + assert.NotEmpty(t, found, "off a participant, createdDate is real provenance") + }) +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/recommendedlists_test.go b/pkg/lib/anyblockjson/snapshotdiff/recommendedlists_test.go new file mode 100644 index 0000000000..2417faba02 --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/recommendedlists_test.go @@ -0,0 +1,77 @@ +package snapshotdiff + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// A type's four role lists are usually ABSENT in the store when nothing +// occupies the role, and import rebuilds all four, so the round trip adds an +// empty one. That step is normalization (see recommendedListKeys) — but only +// that step: a list arriving with members, or an empty list on a key that is +// not one of the four, is still a real difference. These pin both directions, +// so the suppression cannot quietly grow. +func snapWith(details map[string]*types.Value) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: details}, + ObjectTypes: []string{"ot-objectType"}, + } +} + +func list(vals ...string) *types.Value { + out := make([]*types.Value, 0, len(vals)) + for _, v := range vals { + out = append(out, &types.Value{Kind: &types.Value_StringValue{StringValue: v}}) + } + return &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: out}}} +} + +func TestCompare_EmptyRecommendedListIsNormalization(t *testing.T) { + fileKey := bundle.RelationKeyRecommendedFileRelations.String() + featuredKey := bundle.RelationKeyRecommendedFeaturedRelations.String() + + t.Run("an added empty role list is not reported", func(t *testing.T) { + orig := snapWith(map[string]*types.Value{"name": {Kind: &types.Value_StringValue{StringValue: "Task"}}}) + got := snapWith(map[string]*types.Value{ + "name": {Kind: &types.Value_StringValue{StringValue: "Task"}}, + fileKey: list(), + }) + assert.Empty(t, Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{})) + }) + + t.Run("an added role list WITH members is still reported", func(t *testing.T) { + orig := snapWith(map[string]*types.Value{"name": {Kind: &types.Value_StringValue{StringValue: "Task"}}}) + got := snapWith(map[string]*types.Value{ + "name": {Kind: &types.Value_StringValue{StringValue: "Task"}}, + fileKey: list("rel-cover"), + }) + found := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + require.Len(t, found, 1, "a role list that gained content is real drift, not normalization") + assert.Contains(t, found[0], fileKey) + }) + + t.Run("a role list that LOST its members is still reported", func(t *testing.T) { + orig := snapWith(map[string]*types.Value{featuredKey: list("rel-name")}) + got := snapWith(map[string]*types.Value{featuredKey: list()}) + found := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + require.Len(t, found, 1, "emptying an existing list is loss; only absent->empty is normalization") + }) + + t.Run("an empty list on an unrelated key is still reported", func(t *testing.T) { + orig := snapWith(map[string]*types.Value{"name": {Kind: &types.Value_StringValue{StringValue: "Task"}}}) + got := snapWith(map[string]*types.Value{ + "name": {Kind: &types.Value_StringValue{StringValue: "Task"}}, + "tag": list(), + }) + found := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + require.Len(t, found, 1, "the rule is scoped to the four role lists") + assert.Contains(t, found[0], "tag") + }) +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/relationformat_test.go b/pkg/lib/anyblockjson/snapshotdiff/relationformat_test.go new file mode 100644 index 0000000000..f9030f0e9f --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/relationformat_test.go @@ -0,0 +1,137 @@ +package snapshotdiff + +// relationformat_test.go — the §2d lift moved relationFormat, +// relationFormatIncludeTime and relationFormatObjectTypes onto a relation +// document's envelope, and this comparator needed NO new rule for it. That +// was a design obligation, not luck, and it is pinned here: the envelope +// fields mirror stored presence exactly (false, [] and null all travel), and +// the target-type id↔key translation is an inverse (TypeResolver), so the +// details that go in are the details that come out — unlike §2a's +// recommended lists and §2b's icon/cover, which both needed suppression +// rules above. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// relTypeResolver is the storeresolver shape reduced to what §2d consults: a +// PropertyResolver carrying the TypeResolver capability. +type relTypeResolver struct{} + +func (relTypeResolver) PropertyById(string) (anyblockjson.PropertyDefinition, bool) { + return anyblockjson.PropertyDefinition{}, false +} + +func (relTypeResolver) PropertyId(def anyblockjson.PropertyDefinition) (string, bool) { + return "", false +} + +func (relTypeResolver) TypeKeyById(id string) (string, bool) { + if id == "typeid-page" { + return "page", true + } + return "", false +} + +func (relTypeResolver) TypeIdByKey(key string) (string, bool) { + if key == "page" { + return "typeid-page", true + } + return "", false +} + +func relationSnap(details map[string]*types.Value) *model.SmartBlockSnapshotBase { + details["id"] = text("relObjectId") + details["name"] = text("Budget") + return &model.SmartBlockSnapshotBase{ + Key: "budget", + Details: &types.Struct{Fields: details}, + Blocks: []*model.Block{{ + Id: "relObjectId", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + } +} + +// A relation document round-trips with zero findings, resolver or no +// resolver — including the shapes real data carries that a lossy lift would +// have normalized away: a false include_time (8,412 production relations), a +// null one (80), an empty target list (8,903), and target ids beside a +// verbatim survivor. +// +// How this can fail: make the §2d export omit a present-and-empty value, or +// import write keys where ids were (drop the TypeIdByKey arm), and the +// matching row reports a changed or added detail — the noise class every +// suppression rule above exists to record, which §2d was designed not to +// produce. +func TestCompare_RelationDocumentRoundTripsClean(t *testing.T) { + for name, tc := range map[string]struct { + details map[string]*types.Value + opts anyblockjson.Options + }{ + "false and empty, no resolver": { + details: map[string]*types.Value{ + "relationFormat": number(6), + "relationFormatIncludeTime": {Kind: &types.Value_BoolValue{BoolValue: false}}, + "relationFormatObjectTypes": list(), + }, + }, + "null include_time, no resolver": { + details: map[string]*types.Value{ + "relationFormat": number(4), + "relationFormatIncludeTime": {Kind: &types.Value_NullValue{}}, + }, + }, + "target ids under the TypeResolver capability": { + details: map[string]*types.Value{ + "relationFormat": number(100), + "relationFormatObjectTypes": list("typeid-page", "bafyreidangling"), + }, + opts: anyblockjson.Options{ResolveProperties: relTypeResolver{}}, + }, + "map format": { + details: map[string]*types.Value{"relationFormat": number(102)}, + }, + } { + t.Run(name, func(t *testing.T) { + // given + orig := relationSnap(tc.details) + + // when + data, err := anyblockjson.Marshal(model.SmartBlockType_STRelation, orig, tc.opts) + require.NoError(t, err) + sbType, got, err := anyblockjson.Unmarshal(data, tc.opts) + require.NoError(t, err) + + // then + assert.Empty(t, Compare(orig, got, sbType, tc.opts), + "the §2d lift is a spelling change: the same details go in and out") + }) + } +} + +// The comparator still SEES a §2d detail that really changes — the rows +// above are clean because the codec is faithful, not because the keys are +// ignored. +// +// How this can fail: add the three keys to a suppression list (the §2b +// shape) and this loss goes dark. +func TestCompare_ARelationFormatChangeStillReports(t *testing.T) { + // given a format rewrite — the exact loss the §2d lift exists to prevent + orig := relationSnap(map[string]*types.Value{"relationFormat": number(2)}) + got := relationSnap(map[string]*types.Value{"relationFormat": number(0)}) + + // when + found := Compare(orig, got, model.SmartBlockType_STRelation, anyblockjson.Options{}) + + // then + require.Len(t, found, 1) + assert.Contains(t, found[0], "relationFormat") +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/snapshotdiff.go b/pkg/lib/anyblockjson/snapshotdiff/snapshotdiff.go new file mode 100644 index 0000000000..d8825504ac --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/snapshotdiff.go @@ -0,0 +1,632 @@ +// Package snapshotdiff compares two smartblock snapshots on the axes the +// AnyBlock JSON format promises to preserve: the object's TYPES, detail +// values (up to the documented normalizations) and the text content of +// non-structural text blocks (as a multiset). It is the state-diff / +// text-multiset comparator behind cmd/anyblockroundtrip and the API v2 eval +// metric (DELEGATE-52 backtranslation). Findings are triage input, not +// proof. +package snapshotdiff + +import ( + "fmt" + "sort" + "strings" + + "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// strippedKeys is the format's own internal-property set, not a copy of it. +// The copy that used to stand here fell out of date the moment the package +// added the importer's provenance keys, and every object carrying one was +// reported as data loss (§3, §11). +var strippedKeys = anyblockjson.InternalPropertyKeys() + +// recommendedListKeys are the four role lists a type keeps its recommended +// properties in. typeProperties (§2a) collapses them into one labelled array, +// and import rebuilds all four from it — writing an empty list for a role +// nothing occupies. Most types have no file-role property, so the store +// usually has no recommendedFileRelations key at all and the round trip adds +// an empty one. +// +// That is a difference, and it is normalization rather than drift: an absent +// list and an empty list say the same thing, and the empty list is the only +// way the format can express a role being cleared, since typeProperties +// cannot name a section that exists with no members. Left unrecorded it +// buried the sweep — 1 344 of 1 351 differing objects in a 34 339-object +// account differed by nothing else. Whether the object state itself should +// carry all four consistently is GO-7451, not this comparator's call. +// isDroppedEmptyIconCover is the icon/cover analogue, and it is bigger: §2b +// lifted nine hidden keys into the typed `icon` and `cover` envelope fields, +// and a source whose stored value is EMPTY is not a source — so a key present +// and empty comes back absent. Roughly 2 300 objects in a 36 966-object +// account carry at least one, nearly double the recommended-list noise above, +// and left unrecorded it would bury the sweep the same way. +// +// Scoped to absent-vs-dropped-empty and nothing else: a cover that really was +// lost (33 objects hold an absolute filesystem path a Notion import left in +// coverId, which the typed field cannot write) still reports, because its +// value is not empty. The predicate is the format's own, not a copy. +func isDroppedEmptyIconCover(key string, orig, got *types.Value) bool { + return got == nil && anyblockjson.DroppedEmptyIconCover(key, orig) +} + +// isDroppedEmptySystemProperty is the third normalization of this shape, and +// the narrowest: §15 #12 admits seven system-stamped keys whose EMPTY value +// says nothing a reader could act on (`isHidden` false, `revision` 0, +// `relationMaxCount` 0, …), so export omits them and they come back absent. +// The whitelist is deliberately explicit rather than a rule over +// bundle.SystemRelations — see systemtrim.go for the admission test each key +// had to pass, and for the keys that failed it. +// +// Scoped to absent-vs-dropped-empty like its neighbours: a non-empty value +// on one of those keys still reports if it goes missing, and so does an +// empty one that came back SET. The predicate is the format's own. +func isDroppedEmptySystemProperty(key string, orig, got *types.Value) bool { + return got == nil && anyblockjson.DroppedEmptySystemProperty(key, orig) +} + +var recommendedListKeys = map[string]bool{ + bundle.RelationKeyRecommendedFeaturedRelations.String(): true, + bundle.RelationKeyRecommendedRelations.String(): true, + bundle.RelationKeyRecommendedFileRelations.String(): true, + bundle.RelationKeyRecommendedHiddenRelations.String(): true, +} + +// isEmptyRecommendedList reports whether an ADDED detail is one of those four +// arriving empty. A recommended list that arrives with members is a real +// difference and is still reported: this suppresses the absent-to-empty step +// only, never a list that gained content. +// firstStringValue reads the first string a detail holds, whether it is a +// bare string or a one-element list. `iconImage` is stored both ways, and +// export reads images[0] — the comparator must ask about the SAME id. +func firstStringValue(v *types.Value) string { + if v == nil { + return "" + } + if s := v.GetStringValue(); s != "" { + return s + } + for _, el := range v.GetListValue().GetValues() { + if s := el.GetStringValue(); s != "" { + return s + } + } + return "" +} + +func isEmptyRecommendedList(key string, v *types.Value) bool { + if !recommendedListKeys[key] { + return false + } + list := v.GetListValue() + return list != nil && len(list.Values) == 0 +} + +// Compare reports every place where got diverges from orig on a +// format-preserved axis, as human-readable findings. An empty result means +// no detectable drift. +// +// sbType is the snapshot's smartblock type, and it is a parameter rather than +// something read off the pair because it is the only thing that +// says how many type slots the envelope had: `template_for` exists exactly on +// a Template (§2). A snapshot cannot answer that question about itself — a +// template's ObjectTypes need not begin with the template key — so a caller +// that has the type must hand it over, or the diff reports a faithfully +// preserved target type as an invented one. +func Compare(orig, got *model.SmartBlockSnapshotBase, sbType model.SmartBlockType, opts anyblockjson.Options) []string { + var out []string + + out = append(out, compareObjectTypes(orig, got, sbType)...) + + // the §2f omission: a bundled-identical relation document is not written + // at all — its key travels in the dictionary's `installed` list and a + // reader reconstructs it from the bundled table. Across that trip the + // install artifacts (createdDate, origin, apiObjectKey, …) come back + // absent, re-stamped by the next install, and a definition member the + // copy never stored comes back as its explicit empty default. Both skips + // below are scoped to snapshots the omission predicate itself admits — + // the predicate is the format's own, not a copy — so on the ordinary + // document round trip, where every key survives, neither ever fires. + _, omittable := anyblockjson.OmittedBundledRelation(sbType, orig, opts) + + // the §2c widget-object omission: a widget document a bundle does not + // write, because index.json states everything it holds and the object is + // rebuilt from it (WidgetsSnapshot). Across that trip the object's own + // timestamps and its empty name come back absent — a restored sidebar is + // created when it is restored, the space-document rule. Scoped to + // snapshots the omission predicate itself admits, and the predicate is + // the format's own, not a copy — taught in the same commit that taught + // the export wiring, the drift that once produced 1,344 false failures + // in one sweep. + widgetOmitted := anyblockjson.OmittedWidgetObject(sbType, orig) + + if orig.Details != nil { + gotFields := map[string]*types.Value{} + if got.Details != nil { + gotFields = got.Details.Fields + } + keys := make([]string, 0, len(orig.Details.Fields)) + for k := range orig.Details.Fields { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if strippedKeys[k] { + continue + } + if isDroppedEmptyIconCover(k, orig.Details.Fields[k], gotFields[k]) { + continue + } + if isDroppedEmptySystemProperty(k, orig.Details.Fields[k], gotFields[k]) { + continue + } + // an icon whose image object the space DELETED is dropped + // rather than carried as a reference that resolves to nothing + // (§2b). `iconImage` is a detail, so without this the + // comparator reads all 134 corpus cases as data loss — the same + // drift the type-provenance rule above was taught to avoid. + // Scoped to absent-on-the-way-back, and the predicate is the + // format's own rather than a copy. + if gotFields[k] == nil && k == bundle.RelationKeyIconImage.String() && + anyblockjson.DroppedDeletedIconRef(opts, firstStringValue(orig.Details.Fields[k])) { + continue + } + // a TYPE document does not carry its own install provenance + // (§2a): eight keys export omits there whatever their + // value — each admitted against the corpus individually — so the + // comparator learns the rule in the same commit that taught + // export (the miss that produced 1,344 false failures in one + // sweep). Scoped to absent-on-the-way-back: a got side that + // somehow carries the key still reports. The predicate is the + // format's own, not a copy. + if gotFields[k] == nil && anyblockjson.DroppedTypeProvenanceKey(sbType, k) { + continue + } + // a PARTICIPANT document does not carry createdDate (§3): the + // stored value is a load timestamp re-stamped on every cold + // build, dropped by export whatever it holds — taught here in + // the same commit, the standing rule for every drop, or the pb + // sweep reports false loss on all 2,492 corpus participants. + // Same scoping, same ownership of the predicate. + if gotFields[k] == nil && anyblockjson.DroppedParticipantProvenanceKey(sbType, k) { + continue + } + // the five type_settings members follow the §4 omit-empty canon + // (§2a): a pluralName of "" or a defaultTemplateId of [] comes + // back absent. Same scoping, same ownership of the predicate. + if gotFields[k] == nil && anyblockjson.DroppedEmptyTypeSetting(sbType, k, orig.Details.Fields[k]) { + continue + } + // an omitted relation document's install artifacts (§2f): absent + // on the way back, re-stamped by the next install. Scoped to + // absent-and-artifact on an omittable snapshot — a definition + // member that goes missing still reports. + if gotFields[k] == nil && omittable && anyblockjson.RelationInstallArtifactKey(k) { + continue + } + // an omitted widget document's residual keys (§2c): the two + // object timestamps, and a name that was EMPTY — a non-empty + // name keeps the whole document, so within the omitted scope it + // cannot be here. Scoped to absent-on-the-way-back like its + // neighbours; the lifted state itself (the widgets, the + // auto-widget ledger) is rebuilt by WidgetsSnapshot and compares + // as ordinary detail and block state. + if gotFields[k] == nil && widgetOmitted && anyblockjson.WidgetObjectResidualKey(k, orig.Details.Fields[k]) { + continue + } + if !detailEqual(k, orig.Details.Fields[k], gotFields[k], opts) { + out = append(out, fmt.Sprintf("detail %q changed: %s -> %s", + k, valuePreview(orig.Details.Fields[k]), valuePreview(gotFields[k]))) + } + } + } + + // added details: keys present in got but not orig. The orig-key loop + // above already flags changed/removed (detailEqual against a nil got + // value), but never sees keys the round trip introduced. + if got.Details != nil { + gotOnly := make([]string, 0) + for k := range got.Details.Fields { + if strippedKeys[k] { + continue + } + if orig.Details != nil { + if _, inOrig := orig.Details.Fields[k]; inOrig { + continue + } + } + gotOnly = append(gotOnly, k) + } + sort.Strings(gotOnly) + for _, k := range gotOnly { + if isEmptyRecommendedList(k, got.Details.Fields[k]) { + continue + } + // the reconstruction of an omitted relation document (§2f) + // states the WHOLE definition, so a member the original copy + // never stored arrives as its explicit empty default — + // `isHidden: false`, `object_types: []`. Absent and empty say + // the same thing for a definition member with a defined + // default; a NON-empty invented member still reports. + if omittable && anyblockjson.InstallStampedDefault(k, got.Details.Fields[k]) { + continue + } + out = append(out, fmt.Sprintf("detail %q added: %s", k, valuePreview(got.Details.Fields[k]))) + } + } + + // Compare is intentionally order-insensitive on text (a multiset): the + // round-trip verifier tolerates legitimate normalization reordering. + // Order-sensitive scoring lives in the eval corruption metric via + // TextSequence, where a backtranslation must restore exact order. + origTexts := TextInventory(orig) + gotTexts := TextInventory(got) + for text, n := range origTexts { + if gotTexts[text] < n { + out = append(out, fmt.Sprintf("text block lost (%dx): %q", n-gotTexts[text], preview(text))) + } + } + return out +} + +// typeKeyIdPrefix is the "ot-" prefix an ObjectTypes entry carries. +var typeKeyIdPrefix = domain.TypeKey("").URL() + +// compareObjectTypes reports divergence in the TYPE namespace — the axis +// Compare used to be structurally blind to. It read only details and text, so +// a 36 808-object production sweep could never have caught a type +// substitution: every claim about type-key correctness rested on synthetic +// tests alone. A rebinding is exactly the loss the `type_internal_keys` legend (§3) +// exists to prevent, and exactly what a sweep must be able to see. +// +// Equality is the wrong predicate here, because export normalizes the list +// before it writes it (§2, export.envelopeTypeTerms) and every step of that is +// by design. Measured, not assumed: +// +// ["ot-page","ot-task"] -> ["ot-page"] (truncated) +// ["ot-template","ot-task","ot-page"] -> ["ot-template","ot-task"] (truncated) +// ["ot-","ot-task"] -> ["ot-task"] (closed ranks) +// ["ot-template","ot-","ot-task"] -> ["ot-template","ot-task"] (both) +// +// So the comparison applies the same two normalizations to orig — drop the +// keyless entries, then keep the modelled positions — before demanding +// position-for-position identity. That is detailEqual's shape: normalize both +// sides to what the format preserves, then compare exactly, rather than +// reporting a documented normalization as loss. Identity has to be exact +// because order and duplicates carry meaning (`[0]` is the type, `[1]` the +// template target) and both round-trip today. Anything got carries beyond the +// modelled positions is drift the other way: the round trip invented a type. +func compareObjectTypes(orig, got *model.SmartBlockSnapshotBase, sbType model.SmartBlockType) []string { + origKeys := typeKeysOf(orig) + gotKeys := typeKeysOf(got) + modelled := modelledTypeSlots(origKeys, sbType) + + var out []string + for i := 0; i < modelled; i++ { + switch { + case i >= len(gotKeys): + out = append(out, fmt.Sprintf("object type [%d] lost: %q", i, origKeys[i])) + case gotKeys[i] != origKeys[i]: + out = append(out, fmt.Sprintf("object type [%d] changed: %q -> %q", i, origKeys[i], gotKeys[i])) + } + } + for i := modelled; i < len(gotKeys); i++ { + out = append(out, fmt.Sprintf("object type [%d] added: %q", i, gotKeys[i])) + } + return out +} + +// typeKeysOf is export's first normalization: the stored key of each entry, +// with the keyless ones dropped and the survivors closing ranks. Trimming the +// prefix is itself a normalization — a legacy row may hold a bare key, and +// import always writes the prefixed form back. A keyless entry (`ot-`, or "") +// names no type and export drops it with a warning rather than letting it take +// its siblings with it, so it is not loss. +func typeKeysOf(s *model.SmartBlockSnapshotBase) []string { + if s == nil { + return nil + } + out := make([]string, 0, len(s.ObjectTypes)) + for _, t := range s.ObjectTypes { + if key := strings.TrimPrefix(t, typeKeyIdPrefix); key != "" { + out = append(out, key) + } + } + return out +} + +// modelledTypeSlots is how many of the surviving keys the format has a slot +// for — export.modelledTypeKeys' own two conditions (§2): +// +// - `type` takes the first surviving key, whatever it is; +// - `template_for` exists only on a TEMPLATE, and takes the second. +// +// The second condition used to be "only when the first key is the template +// key", mirroring what export did. Both were wrong the same way: a template +// whose object types do not begin with the template key — a shape nothing in +// the model forbids — lost its target type on export, and this diff called +// that loss correct. `kind` carries template-ness now, so the question is +// answered by the smartblock type and not by the data. +// +// There is no third slot, so anything further is dropped by design. +func modelledTypeSlots(keys []string, sbType model.SmartBlockType) int { + if len(keys) == 0 { + return 0 + } + if sbType == model.SmartBlockType_Template && len(keys) > 1 { + return 2 + } + return 1 +} + +// detailEqual compares one detail value up to the documented normalizations: +// scalars of list-shaped formats become single-element lists, dates truncate +// to whole seconds. +func detailEqual(key string, a, b *types.Value, opts anyblockjson.Options) bool { + if b == nil { + return false + } + if recommendedDetailKeys[key] && opts.ResolveProperties != nil { + return equalStrings( + normalizeRecommended(stringsOf(a), opts.ResolveProperties), + normalizeRecommended(stringsOf(b), opts.ResolveProperties)) + } + // relationFormatObjectTypes round-trips by type KEY (§2d), and legacy + // data mixes spellings the same way the recommended lists do: 27 corpus + // relations store a bare type key where the store speaks object ids, and + // import writes the id back — the same TYPE, in the store's own + // spelling. So the comparison normalizes both sides to keys through the + // TypeResolver capability and demands position-for-position identity, + // exactly as normalizeRecommended does one namespace over: a rebound + // TARGET still reports, a respelled one does not. Without the + // capability the translation is verbatim both ways (§2d), so the raw + // comparison below is already exact. + if key == relationTargetsDetailKey { + if tr, ok := opts.ResolveProperties.(anyblockjson.TypeResolver); ok { + return equalStrings(normalizeTypeRefs(stringsOf(a), tr, opts), normalizeTypeRefs(stringsOf(b), tr, opts)) + } + } + format, _ := resolveFormat(key, opts) + switch format { + case model.RelationFormat_object, model.RelationFormat_file: + // mirror the format's list extraction (scalars wrap, empty strings + // drop) AND its missing-reference rule (§9): an entry naming an + // object the space does not hold is dropped by design on export, so + // the comparison applies the format's own predicate + // (DroppedMissingObjectRef) to BOTH sides — taught in the same + // commit that taught export, the drift that once produced 1,344 + // false failures in one sweep. A live entry that vanishes still + // reports, and with no existence capability in opts the predicate + // drops nothing, exactly as export drops nothing. + return equalStrings(keptObjectRefs(stringsOf(a), opts), keptObjectRefs(stringsOf(b), opts)) + case model.RelationFormat_status, model.RelationFormat_tag: + // mirror the format's list extraction (scalars wrap, empty strings + // drop) AND the sentinel half of the missing-reference rule (§9): a + // select value of `_missing_object` names an option that is gone, so + // export writes a shorter list. Taught here in the same commit that + // taught export — the drift that once produced 1,344 false failures. + return equalStrings(keptOptionRefs(stringsOf(a)), keptOptionRefs(stringsOf(b))) + case model.RelationFormat_date: + return int64(a.GetNumberValue()) == int64(b.GetNumberValue()) + } + return proto.Equal(a, b) +} + +// keptOptionRefs drops the stored dangling-option sentinel, which export +// drops from a select value. Unlike the object arm this needs no existence +// capability: the sentinel is self-describing, so both sides of a comparison +// agree whether or not a resolver is wired. +func keptOptionRefs(in []string) []string { + out := make([]string, 0, len(in)) + for _, s := range in { + if s == anyblockjson.MissingObjectId { + continue + } + out = append(out, s) + } + return out +} + +// recommendedDetailKeys are the four lists SPEC §2a lifts into +// typeProperties. They round-trip by property KEY, and legacy data mixes ids +// and bare keys, so comparison normalizes both sides to keys and skips +// entries neither side can resolve (dropped-by-design, like missing-object +// sentinels). +var recommendedDetailKeys = map[string]bool{ + "recommendedFeaturedRelations": true, + "recommendedRelations": true, + "recommendedFileRelations": true, + "recommendedHiddenRelations": true, +} + +// relationTargetsDetailKey is the stored key behind property_settings' +// object_types (§2d) — the type-namespace twin of the four lists above. +// Named off the bundle rather than spelled, the §2b rule: a rename there +// is a compile error here instead of a comparator that silently stops +// normalizing the key it was written for. +var relationTargetsDetailKey = bundle.RelationKeyRelationFormatObjectTypes.String() + +// normalizeTypeRefs reduces each target entry to the type KEY it names: a +// bundled url through the table, an object id through the resolver, and +// anything else — a legacy bare key included — verbatim, its own address. +// Applied to BOTH sides, so only a change of the type named survives it. +// +// An entry that resolves to no key AND names no object the space holds is +// skipped — export's own missing-reference drop (§9), asked through the +// format's own predicate so the two cannot drift. The order matters and +// mirrors export's: resolution first, so a live type's id can never reach +// the drop, and the predicate's CID-shape gate keeps it off bare keys, +// which are vocabulary, not references. +func normalizeTypeRefs(entries []string, tr anyblockjson.TypeResolver, opts anyblockjson.Options) []string { + out := make([]string, 0, len(entries)) + for _, entry := range entries { + if key, err := bundle.TypeKeyFromUrl(entry); err == nil && key != "" { + out = append(out, string(key)) + continue + } + if key, ok := tr.TypeKeyById(entry); ok && key != "" { + out = append(out, key) + continue + } + if anyblockjson.DroppedMissingObjectRef(opts, entry) { + continue + } + out = append(out, entry) + } + return out +} + +// keptObjectRefs filters an objects/files value down to the entries export +// keeps (§9): the missing-reference predicate is the format's own, applied +// identically to the original and the round-tripped side, so a +// dropped-by-design entry is not loss and everything else still compares +// position for position. +func keptObjectRefs(entries []string, opts anyblockjson.Options) []string { + out := make([]string, 0, len(entries)) + for _, entry := range entries { + if anyblockjson.DroppedMissingObjectRef(opts, entry) { + continue + } + out = append(out, entry) + } + return out +} + +func normalizeRecommended(entries []string, r anyblockjson.PropertyResolver) []string { + var out []string + for _, id := range entries { + if def, ok := r.PropertyById(id); ok { + out = append(out, string(def.Key)) + continue + } + if _, ok := r.PropertyId(anyblockjson.PropertyDefinition{Key: domain.RelationKey(id)}); ok { + out = append(out, id) // already a key + continue + } + if _, err := bundle.GetRelation(domain.RelationKey(id)); err == nil { + out = append(out, id) // bundle key without a space object + } + // otherwise unresolvable: dropped by design on export, skip + } + return out +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// missingObjectSentinel marks a dangling object reference in stored details +// (pkg/lib/localstore/addr). Export legitimately drops these unresolvable +// refs, so the comparison must not count them as loss. +const missingObjectSentinel = "_missing_object" + +// stringsOf reads a value as the format's string list: single strings wrap, +// empty strings drop (the export-side valueStringList semantics), and +// pre-broken missing-object sentinels are ignored. +func stringsOf(v *types.Value) []string { + if s := v.GetStringValue(); s != "" && s != missingObjectSentinel { + return []string{s} + } + var out []string + for _, el := range v.GetListValue().GetValues() { + if s := el.GetStringValue(); s != "" && s != missingObjectSentinel { + out = append(out, s) + } + } + return out +} + +func resolveFormat(key string, opts anyblockjson.Options) (model.RelationFormat, bool) { + if f, err := bundle.GetRelationFormat(domain.RelationKey(key)); err == nil { + return f, true + } + if opts.ResolveFormat != nil { + return opts.ResolveFormat(domain.RelationKey(key)) + } + return 0, false +} + +// TextInventory counts the plain text of text blocks the format preserves — +// the text multiset. Structural styles (title, description) are dropped by +// design; blocks with emoji marks are skipped because emoji materialization +// changes the text lossily by design (SPEC §8). +func TextInventory(s *model.SmartBlockSnapshotBase) map[string]int { + out := map[string]int{} + for _, b := range s.Blocks { + t := b.GetText() + if t == nil || t.Text == "" { + continue + } + switch t.Style { + case model.BlockContentText_Title, model.BlockContentText_Description: + continue + } + skip := false + for _, m := range t.Marks.GetMarks() { + if m != nil && m.Type == model.BlockContentTextMark_Emoji { + skip = true + break + } + } + if !skip { + out[t.Text]++ + } + } + return out +} + +// TextSequence is the ordered analog of TextInventory: the preserved text of +// text blocks in snapshot block order, with the same structural/emoji +// filtering. Used to detect pure reordering, which the multiset cannot. +func TextSequence(s *model.SmartBlockSnapshotBase) []string { + var out []string + for _, b := range s.Blocks { + t := b.GetText() + if t == nil || t.Text == "" { + continue + } + switch t.Style { + case model.BlockContentText_Title, model.BlockContentText_Description: + continue + } + skip := false + for _, m := range t.Marks.GetMarks() { + if m != nil && m.Type == model.BlockContentTextMark_Emoji { + skip = true + break + } + } + if !skip { + out = append(out, t.Text) + } + } + return out +} + +func preview(s string) string { + if len(s) > 80 { + return s[:80] + "…" + } + return s +} + +func valuePreview(v *types.Value) string { + if v == nil { + return "" + } + return preview(v.String()) +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/snapshotdiff_test.go b/pkg/lib/anyblockjson/snapshotdiff/snapshotdiff_test.go new file mode 100644 index 0000000000..986eb3ec74 --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/snapshotdiff_test.go @@ -0,0 +1,216 @@ +package snapshotdiff + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func str(s string) *types.Value { + return &types.Value{Kind: &types.Value_StringValue{StringValue: s}} +} + +func num(n float64) *types.Value { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: n}} +} + +func fields(kv map[string]*types.Value) *types.Struct { + return &types.Struct{Fields: kv} +} + +func textBlock(id, text string) *model.Block { + return &model.Block{Id: id, Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: text}, + }} +} + +func snapshot(details map[string]*types.Value, blocks ...*model.Block) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{Details: fields(details), Blocks: blocks} +} + +func TestCompare(t *testing.T) { + t.Run("identical snapshots have no drift", func(t *testing.T) { + // given + a := snapshot(map[string]*types.Value{"name": str("Doc")}, textBlock("b1", "hello")) + b := snapshot(map[string]*types.Value{"name": str("Doc")}, textBlock("b1", "hello")) + + // when + got := Compare(a, b, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + assert.Empty(t, got) + }) + + t.Run("lost text block is reported", func(t *testing.T) { + // given + a := snapshot(map[string]*types.Value{}, textBlock("b1", "hello"), textBlock("b2", "world")) + b := snapshot(map[string]*types.Value{}, textBlock("b1", "hello")) + + // when + got := Compare(a, b, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + require.Len(t, got, 1) + assert.Contains(t, got[0], `text block lost (1x): "world"`) + }) + + t.Run("changed detail is reported", func(t *testing.T) { + // given + a := snapshot(map[string]*types.Value{"name": str("Doc")}) + b := snapshot(map[string]*types.Value{"name": str("Renamed")}) + + // when + got := Compare(a, b, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + require.Len(t, got, 1) + assert.Contains(t, got[0], `detail "name" changed`) + }) + + t.Run("added detail is reported", func(t *testing.T) { + // given: the round trip introduced a detail absent from the original + a := snapshot(map[string]*types.Value{"name": str("Doc")}) + b := snapshot(map[string]*types.Value{"name": str("Doc"), "description": str("new")}) + + // when + got := Compare(a, b, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + require.Len(t, got, 1) + assert.Contains(t, got[0], `detail "description" added`) + }) + + t.Run("date sub-second truncation is not drift", func(t *testing.T) { + // given: dueDate is a bundled date relation + a := snapshot(map[string]*types.Value{"dueDate": num(1700000000.7)}) + b := snapshot(map[string]*types.Value{"dueDate": num(1700000000)}) + + // when + got := Compare(a, b, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + assert.Empty(t, got) + }) + + t.Run("stripped local keys are ignored", func(t *testing.T) { + // given: lastOpenedDate is local-only, stripped on export by design + a := snapshot(map[string]*types.Value{"lastOpenedDate": num(1)}) + b := snapshot(map[string]*types.Value{}) + + // when + got := Compare(a, b, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + assert.Empty(t, got) + }) + + t.Run("moved text is not drift, the inventory is a multiset", func(t *testing.T) { + // given + a := snapshot(nil, textBlock("b1", "one"), textBlock("b2", "two")) + b := snapshot(nil, textBlock("x9", "two"), textBlock("x8", "one")) + + // when + got := Compare(a, b, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + assert.Empty(t, got) + }) +} + +func TestTextInventory(t *testing.T) { + // given: title/description styles and emoji-marked blocks are excluded + title := &model.Block{Id: "t", Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "Title", Style: model.BlockContentText_Title}, + }} + emoji := &model.Block{Id: "e", Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: "hi :)", Marks: &model.BlockContentTextMarks{ + Marks: []*model.BlockContentTextMark{{Type: model.BlockContentTextMark_Emoji}}, + }}, + }} + s := snapshot(nil, title, emoji, textBlock("b1", "kept"), textBlock("b2", "kept")) + + // when + inv := TextInventory(s) + + // then + assert.Equal(t, map[string]int{"kept": 2}, inv) +} + +// The comparator knows the §15 #12 trim, and knows only that: a dropped +// EMPTY value on an admitted key is normalization, a dropped non-empty value +// on the same key is loss, and an unlisted key is neither. +// +// The two halves have to move together. When the exporter learned to omit a +// key the comparator did not know about, every object carrying one reported +// as data loss — 1,344 of 1,351 differing objects in a 34,339-object sweep, +// which buried the seven real findings underneath. +// +// How this can fail: remove the isDroppedEmptySystemProperty arm and the +// first case reports; widen it past `got == nil` and the second stops. +func TestCompare_KnowsTheEmptySystemPropertyTrim(t *testing.T) { + boolValue := func(b bool) *types.Value { + return &types.Value{Kind: &types.Value_BoolValue{BoolValue: b}} + } + root := &model.Block{Id: "root", Content: &model.BlockContentOfSmartblock{ + Smartblock: &model.BlockContentSmartblock{}}} + snap := func(kv map[string]*types.Value) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{Blocks: []*model.Block{root}, Details: fields(kv)} + } + + t.Run("an admitted key dropped while empty is normalization", func(t *testing.T) { + // given + orig := snap(map[string]*types.Value{"isHidden": boolValue(false), "revision": num(0)}) + got := snap(map[string]*types.Value{}) + + // when + findings := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + assert.Empty(t, findings) + }) + + t.Run("the same key dropped while SET is loss", func(t *testing.T) { + // given + orig := snap(map[string]*types.Value{"isHidden": boolValue(true)}) + got := snap(map[string]*types.Value{}) + + // when + findings := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + require.Len(t, findings, 1) + assert.Contains(t, findings[0], `detail "isHidden" changed`) + }) + + t.Run("an admitted key that GAINED a value is still loss", func(t *testing.T) { + // given the suppression is scoped to a key that went absent — a round + // trip that turned an empty flag into a set one changed the object + orig := snap(map[string]*types.Value{"isHidden": boolValue(false)}) + got := snap(map[string]*types.Value{"isHidden": boolValue(true)}) + + // when + findings := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + require.Len(t, findings, 1) + assert.Contains(t, findings[0], `detail "isHidden" changed`) + }) + + t.Run("an unlisted system relation dropped while empty is still loss", func(t *testing.T) { + // given `origin` is a system relation the whitelist does not admit + orig := snap(map[string]*types.Value{"origin": num(0)}) + got := snap(map[string]*types.Value{}) + + // when + findings := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + + // then + require.Len(t, findings, 1) + assert.Contains(t, findings[0], `detail "origin" changed`) + }) +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/typesettings_test.go b/pkg/lib/anyblockjson/snapshotdiff/typesettings_test.go new file mode 100644 index 0000000000..2d0e976a51 --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/typesettings_test.go @@ -0,0 +1,70 @@ +package snapshotdiff + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// A type document does not carry its own install provenance (§2a, v0.32): +// eight stored keys come back absent from the round trip, and the five +// lifted settings come back absent when their stored value was empty. Both +// are the format's documented normalizations, so the comparator must accept +// exactly them — through the format's OWN predicates, never a copy — and +// still report everything else. When the exporter and this comparator last +// drifted on a rule of this shape, 1,344 of 1,351 differing objects in one +// sweep were false failures. +// +// How this can fail: teach export a drop without wiring the predicate here +// (the first case reports every provenance key as a change), or suppress +// more than the rule says (the second and third cases go quiet on real +// loss). +func TestTypeProvenance_DropIsNormalizationOnTypeDocuments(t *testing.T) { + orig := snapWith(map[string]*types.Value{ + "name": str2("Task"), + "origin": num2(7), + "revision": num2(3), + "setOf": list("bafyreinothing"), + "pluralName": str2(""), // present-and-empty: the omit-empty canon + }) + + t.Run("the documented drops are silent", func(t *testing.T) { + // given the real round trip, not a hand-built got + data, err := anyblockjson.Marshal(model.SmartBlockType_STType, orig, anyblockjson.Options{}) + require.NoError(t, err) + _, got, err := anyblockjson.Unmarshal(data, anyblockjson.Options{}) + require.NoError(t, err) + + // when + found := Compare(orig, got, model.SmartBlockType_STType, anyblockjson.Options{}) + + // then + assert.Empty(t, found, "the drops are the format's decision, not loss") + }) + + t.Run("the same keys still report on a page", func(t *testing.T) { + got := snapWith(map[string]*types.Value{"name": str2("Task")}) + found := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + assert.NotEmpty(t, found, "off a type document, origin and setOf are real data") + }) + + t.Run("a non-empty lifted setting that vanishes still reports", func(t *testing.T) { + withName := snapWith(map[string]*types.Value{"pluralName": str2("Tasks")}) + got := snapWith(map[string]*types.Value{}) + found := Compare(withName, got, model.SmartBlockType_STType, anyblockjson.Options{}) + assert.NotEmpty(t, found, "only the EMPTY setting's omission is documented") + }) +} + +func str2(s string) *types.Value { + return &types.Value{Kind: &types.Value_StringValue{StringValue: s}} +} + +func num2(n float64) *types.Value { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: n}} +} diff --git a/pkg/lib/anyblockjson/snapshotdiff/widgetobject_test.go b/pkg/lib/anyblockjson/snapshotdiff/widgetobject_test.go new file mode 100644 index 0000000000..e725fea01c --- /dev/null +++ b/pkg/lib/anyblockjson/snapshotdiff/widgetobject_test.go @@ -0,0 +1,112 @@ +package snapshotdiff + +// widgetobject_test.go pins the comparator's side of the §2c widget-object +// omission: a widget document travels as index.widgets plus the auto-widget +// ledger, and what a bundle carries instead is WidgetsSnapshot's rebuild. +// The one skip that trip needs — the object's own timestamps and its empty +// name absent on the way back — is scoped to snapshots the omission +// predicate itself admits, so the ordinary document round trip keeps its +// full sensitivity. Taught in the same commit that taught the export +// wiring, because a comparator that learns a normalization late reports it +// as loss across a whole corpus (the drift that once produced 1,344 false +// failures in one sweep). + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +const widgetTestTarget = "bafyreidft7aqr2fgy6g57hme4rmcdkynf24cd2jfhlyr3duxjevj6vewsu" + +// storedWidgetObject is a widget object the way a real export holds one: the +// wrapper-and-link pair, the constant hidden-dashboard details, the ledger, +// the timestamps, and the empty name 15 of 77 corpus documents carry. +func storedWidgetObject() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "root", ChildrenIds: []string{"w1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "w1", ChildrenIds: []string{"l1"}, + Content: &model.BlockContentOfWidget{Widget: &model.BlockContentWidget{ + Layout: model.BlockContentWidget_Tree, Limit: 6}}}, + {Id: "l1", Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: widgetTestTarget}}}, + }, + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "root"}}, + "isHidden": {Kind: &types.Value_BoolValue{BoolValue: true}}, + "layout": {Kind: &types.Value_NumberValue{NumberValue: float64(model.ObjectType_dashboard)}}, + "resolvedLayout": {Kind: &types.Value_NumberValue{NumberValue: float64(model.ObjectType_dashboard)}}, + "createdDate": {Kind: &types.Value_NumberValue{NumberValue: 0}}, + "lastModifiedDate": {Kind: &types.Value_NumberValue{NumberValue: 1700000000}}, + "name": {Kind: &types.Value_StringValue{StringValue: ""}}, + "autoWidgetTargets": {Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: []*types.Value{{Kind: &types.Value_StringValue{StringValue: "bin"}}}}}}, + "autoWidgetDisabled": {Kind: &types.Value_BoolValue{BoolValue: true}}, + }}, + ObjectTypes: []string{"ot-dashboard"}, + } +} + +// Across the omission trip the rebuild carries everything the index states — +// the pair, the constants, the ledger — and only the object's own timestamps +// and its empty name come back absent. That is normalization, not loss: +// a restored sidebar is created when it is restored. +// +// How this can fail: remove the WidgetObjectResidualKey skip from Compare's +// orig-key loop and createdDate/lastModifiedDate/name all report as +// changed-to-absent — on 66 of 77 corpus spaces at once. +func TestCompare_OmittedWidgetObjectAgainstItsRebuild(t *testing.T) { + orig := storedWidgetObject() + require.True(t, anyblockjson.OmittedWidgetObject(model.SmartBlockType_Widget, orig), + "the fixture must be one the omission admits, or this test pins nothing") + + var idx anyblockjson.Index + anyblockjson.IndexFromWidgetObject(&idx, orig) + rebuilt, err := anyblockjson.WidgetsSnapshot(&idx) + require.NoError(t, err) + require.NotNil(t, rebuilt) + + assert.Empty(t, Compare(orig, rebuilt, model.SmartBlockType_Widget, anyblockjson.Options{})) +} + +// The skip is scoped by the omission predicate and by the residual predicate +// both, so the comparator's sensitivity survives everywhere else. +func TestCompare_WidgetSkipStaysScoped(t *testing.T) { + t.Run("a lost timestamp on an ordinary document still reports", func(t *testing.T) { + orig := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "createdDate": {Kind: &types.Value_NumberValue{NumberValue: 1700000000}}, + }}} + got := &model.SmartBlockSnapshotBase{} + diffs := Compare(orig, got, model.SmartBlockType_Page, anyblockjson.Options{}) + require.Len(t, diffs, 1) + assert.Contains(t, diffs[0], "createdDate") + }) + + t.Run("a widget object the omission refuses keeps full sensitivity", func(t *testing.T) { + // a NON-empty name makes the object one the bundle keeps as a + // document, so nothing about it is normalization + orig := storedWidgetObject() + orig.Details.Fields["name"] = &types.Value{Kind: &types.Value_StringValue{StringValue: "My sidebar"}} + require.False(t, anyblockjson.OmittedWidgetObject(model.SmartBlockType_Widget, orig)) + diffs := Compare(orig, &model.SmartBlockSnapshotBase{ObjectTypes: orig.ObjectTypes}, + model.SmartBlockType_Widget, anyblockjson.Options{}) + assert.NotEmpty(t, diffs, "every detail it loses must report, timestamps included") + }) + + t.Run("a lost ledger still reports even on an omitted object", func(t *testing.T) { + // the ledger is LIFTED state, not residue: the rebuild writes it + // back, so a rebuild that loses it has drifted from the lift + orig := storedWidgetObject() + rebuilt := storedWidgetObject() + delete(rebuilt.Details.Fields, "autoWidgetTargets") + diffs := Compare(orig, rebuilt, model.SmartBlockType_Widget, anyblockjson.Options{}) + require.Len(t, diffs, 1) + assert.Contains(t, diffs[0], "autoWidgetTargets") + }) +} diff --git a/pkg/lib/anyblockjson/spacesettings.go b/pkg/lib/anyblockjson/spacesettings.go new file mode 100644 index 0000000000..2ec7c8522c --- /dev/null +++ b/pkg/lib/anyblockjson/spacesettings.go @@ -0,0 +1,207 @@ +package anyblockjson + +// spacesettings.go — the space's own object, and why a bundle does not carry +// one (§2c). +// +// `kind: "space_settings"` holds the space's name, description and homepage. +// A bundle already says all three, in `index.json`, which exists to "describe +// the bundle as a whole" and of which there is exactly one — an export is a +// single space. So the document restates the index and nothing else. +// +// That is not an assumption. Measured over a 77-space export, after every +// rule already in this package has run — attribution stripped as values, the +// one-distinct-value constants dropped, the source space's invite +// credentials and analytics identity refused (§3), and the deprecated +// `spaceDashboardId`/`spaceUxType`/`hasChat` with them — a space document +// reduces to exactly four members: +// +// homepage 77 of 77 → index.homepage +// createdDate 77 of 77 → dropped: when the space OBJECT was minted, +// which a restored space is not +// lastModifiedDate 77 of 77 → dropped, for the same reason +// iconOption 74 of 77 → index.icon (a colour) +// name 75 of 77 → index.name +// iconImage 56 of 77 → index.icon (an image object in the bundle) +// description 12 of 77 → index.description +// featuredRelations 12 of 77 → what the space OBJECT features, which is +// nothing once the object is not a document +// iconEmoji 1 of 77 → index.icon +// +// The icon was the reason this could not simply be dropped: a first reading +// of the corpus counted only the members a rendered DOCUMENT still showed and +// concluded four remained. Read from the stored details instead — which is +// what the omission actually sees — three more channels appear, and almost +// every space has one. +// +// So export omits it, and `IndexFromSpaceSettings` is the one place that says +// which detail becomes which index field — so a composer cannot quietly carry +// fewer of them than the omission assumes were carried. + +import ( + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// spaceSettingsIndexKeys maps the stored detail to the index field it becomes. +// The map is the contract: every member a space document would have carried +// is either here or provably absent by the strips above, and the omission +// predicate checks exactly that rather than trusting the list. +var spaceSettingsIndexKeys = map[string]string{ + bundle.RelationKeyName.String(): "name", + bundle.RelationKeyDescription.String(): "description", + "homepage": "homepage", + detailKeyIconEmoji: "icon", + detailKeyIconImage: "icon", + detailKeyIconName: "icon", + detailKeyIconOption: "icon", +} + +// pageIsEmpty reports an object whose page holds nothing a reader +// would miss — only the header scaffolding every object carries: the root +// block, the header layout, the featured-properties row, and an EMPTY title. +// +// Counting blocks does not answer this: all 77 corpus spaces have an empty +// page, and 17 of them carry four blocks of scaffolding to say so, so a +// `len(blocks) > 1` test keeps 17 documents that hold nothing at all. +// +// Fail-closed, and deliberately narrow: a text block with any text, marks or +// a style other than the two structural ones, and any other block type at +// all, keeps the document. +func pageIsEmpty(base *model.SmartBlockSnapshotBase) bool { + for _, b := range base.GetBlocks() { + switch c := b.Content.(type) { + case *model.BlockContentOfSmartblock, *model.BlockContentOfLayout, + *model.BlockContentOfFeaturedRelations: + // the scaffolding the editor puts on every object + case *model.BlockContentOfText: + t := c.Text + if t.GetText() != "" || len(t.GetMarks().GetMarks()) > 0 { + return false + } + if t.GetStyle() != model.BlockContentText_Title && + t.GetStyle() != model.BlockContentText_Description { + return false + } + default: + return false + } + } + return true +} + +// spaceIcon chooses the space's icon through the ONE precedence this format +// has (§2b), and reports whether it can be carried WHOLE. +// +// The second return is what makes the omission safe: `iconOf` warns exactly +// where a stored channel cannot be written — an icon name this format cannot +// spell, an image that is not an object id, a list holding more than one. On +// an ordinary object that warning travels with the document. Here there is no +// document to carry it, so anything less than a lossless icon keeps the +// document instead. +func spaceIcon(base *model.SmartBlockSnapshotBase) (icon *Icon, whole bool) { + det := base.GetDetails().GetFields() + lossless := true + ic := iconOf( + func(k string) *types.Value { return det[k] }, + func(string, string, ...any) { lossless = false }, + nil, // no options here; a space icon is read from the snapshot alone + ) + return ic, lossless +} + +// IndexFromSpaceSettings reads the space's own object into the index fields +// it is the source of (§2c). It is the composer's half of the omission: a +// bundle that drops the document MUST write these, or the space loses its +// name. +// +// It fills only what the object states; an absent detail leaves the index +// field alone, so a composer may set its own name and have the object's not +// overwrite it. +func IndexFromSpaceSettings(idx *Index, base *model.SmartBlockSnapshotBase) { + if idx == nil || base == nil { + return + } + det := base.GetDetails().GetFields() + if v := stringDetail(det, bundle.RelationKeyName.String()); v != "" { + idx.Name = v + } + if v := stringDetail(det, bundle.RelationKeyDescription.String()); v != "" { + idx.Description = v + } + if v := stringDetail(det, "homepage"); v != "" { + // the STORE spells a reserved screen the way core/domain/homepage.go + // does — a bare `widgets` — while the format spells it `_widgets`, + // inside the `_` namespace no bundle object may claim (§1). Lifting + // the stored value verbatim put the wire spelling in the index, and + // the batch checker then read it as an object id naming nothing: + // 8 of 77 exported indexes said `"homepage": "widgets"`. + idx.Homepage = FormatHomepage(v) + } + if ic, whole := spaceIcon(base); whole && ic != nil { + idx.Icon = ic + } +} + +// OmittedSpaceSettings reports a space document a bundle does not write, +// because `index.json` states everything it holds (§2c). +// +// Fail-closed, like the relation omission beside it: a member this package +// cannot account for keeps the document, so a space carrying something +// unforeseen travels rather than vanishing. The accounted-for set is +// everything the strips already remove plus the three index fields — and +// `featuredRelations`, which describes the object rather than the space and +// has nowhere to go once there is no object. +func OmittedSpaceSettings(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase) bool { + if sbType != model.SmartBlockType_Workspace || base == nil { + return false + } + if !pageIsEmpty(base) { + // a space object with real content on its page is not a restatement + // of anything + return false + } + if _, whole := spaceIcon(base); !whole { + // the index would carry a lesser icon than the object holds + return false + } + stripped := strippedDetailKeys() + for k := range base.GetDetails().GetFields() { + switch { + case spaceSettingsIndexKeys[k] != "": + // index.json carries it — IndexFromSpaceSettings is the proof + case isTransientProperty(k), stripped[k]: + // already refused or dropped by a rule of its own + case k == bundle.RelationKeyFeaturedRelations.String(): + // what the space OBJECT features; there is no object to feature + // anything once the document is gone + case spaceSettingsConstantKeys[k]: + // one distinct value across all 77 corpus documents + case k == bundle.RelationKeyCreatedDate.String(), + k == bundle.RelationKeyLastModifiedDate.String(): + // when the space OBJECT was minted and last touched. A bundle is + // not that object: a space restored from one is created when it + // is restored, so carrying the original timestamps would date the + // new space to the old one + default: + return false // unaccounted: keep the document + } + } + return true +} + +// spaceSettingsConstantKeys are the details every space document carries with +// the same value, so a reader learns nothing from them that the kind does not +// already say. Counted across all 77 documents of a 77-space export. +var spaceSettingsConstantKeys = map[string]bool{ + bundle.RelationKeyLayout.String(): true, // "space", 1 distinct + bundle.RelationKeyResolvedLayout.String(): true, // "dashboard", 1 distinct + bundle.RelationKeyIsHidden.String(): true, // true, 1 distinct + bundle.RelationKeyMigrationObjectContext.String(): true, // 10, 1 distinct + "migrationBackRelations": true, // 1 distinct + bundle.RelationKeyId.String(): true, // the envelope carries it + bundle.RelationKeySpaceId.String(): true, // the bundle IS the space +} + +var _ = types.Struct{} diff --git a/pkg/lib/anyblockjson/spacesettings_test.go b/pkg/lib/anyblockjson/spacesettings_test.go new file mode 100644 index 0000000000..0aabee2ee0 --- /dev/null +++ b/pkg/lib/anyblockjson/spacesettings_test.go @@ -0,0 +1,236 @@ +package anyblockjson + +// spacesettings_test.go — the space's own object, and why a bundle carries +// index.json instead of one (§2c). + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func spaceSnapshot(extra map[string]*types.Value) *model.SmartBlockSnapshotBase { + det := map[string]*types.Value{ + "id": str("bafyreispace"), "name": str("My space"), + "homepage": str("bafyreihome"), "layout": num(9), "resolvedLayout": num(10), + "isHidden": {Kind: &types.Value_BoolValue{BoolValue: true}}, + } + for k, v := range extra { + det[k] = v + } + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "bafyreispace", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(det), + } +} + +// After every rule already in this package has run, a space document reduces +// to a restatement of index.json — measured over 77 corpus space documents: +// homepage 77, name 75, description 12, featuredRelations 12, and nothing +// else. index.json says the first three and exists exactly once per bundle, +// because an export is a single space. +// +// The predicate is FAIL-CLOSED: a member this package cannot account for +// keeps the document, so a space carrying something unforeseen travels +// rather than vanishing. +// +// How this can fail: make the default arm return true and an unaccounted +// member disappears with the document; drop the kind gate and an ordinary +// page stops being exported. +func TestSpaceSettings_OmittedOnlyWhenTheIndexSaysItAll(t *testing.T) { + t.Run("a plain space document is omitted", func(t *testing.T) { + assert.True(t, OmittedSpaceSettings(model.SmartBlockType_Workspace, spaceSnapshot(nil))) + }) + + t.Run("the secrets it used to carry do not stop the omission", func(t *testing.T) { + // they are refused by their own rule (§3), so they are accounted for + assert.True(t, OmittedSpaceSettings(model.SmartBlockType_Workspace, + spaceSnapshot(map[string]*types.Value{ + "spaceInviteFileKey": str("SECRET"), "analyticsSpaceId": str("abc")}))) + }) + + t.Run("an unforeseen member keeps the document", func(t *testing.T) { + assert.False(t, OmittedSpaceSettings(model.SmartBlockType_Workspace, + spaceSnapshot(map[string]*types.Value{"somethingNobodyPlannedFor": str("x")})), + "fail closed: a space carrying something unaccounted must travel") + }) + + t.Run("real content on its page keeps the document", func(t *testing.T) { + snap := spaceSnapshot(nil) + snap.Blocks = append(snap.Blocks, &model.Block{Id: "p", + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "hello"}}}) + assert.False(t, OmittedSpaceSettings(model.SmartBlockType_Workspace, snap)) + }) + + t.Run("no other kind is ever omitted here", func(t *testing.T) { + assert.False(t, OmittedSpaceSettings(model.SmartBlockType_Page, spaceSnapshot(nil))) + }) + + // The space object's own timestamps: when it was minted, not when the + // bundle's content was written. A space restored from a bundle is created + // when it is restored. + t.Run("the object's own timestamps do not stop the omission", func(t *testing.T) { + assert.True(t, OmittedSpaceSettings(model.SmartBlockType_Workspace, + spaceSnapshot(map[string]*types.Value{ + "createdDate": num(1700000000), "lastModifiedDate": num(1700000001)}))) + }) + + // 17 of 77 corpus spaces carry the editor's header scaffolding and no + // content at all. Counting blocks kept every one of them. + t.Run("header scaffolding is not content", func(t *testing.T) { + snap := spaceSnapshot(nil) + snap.Blocks = append(snap.Blocks, + &model.Block{Id: "header", Content: &model.BlockContentOfLayout{ + Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Header}}}, + &model.Block{Id: "title", Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Style: model.BlockContentText_Title}}}, + &model.Block{Id: "featured", Content: &model.BlockContentOfFeaturedRelations{ + FeaturedRelations: &model.BlockContentFeaturedRelations{}}}) + assert.True(t, OmittedSpaceSettings(model.SmartBlockType_Workspace, snap)) + }) + + t.Run("an empty block that is not scaffolding keeps the document", func(t *testing.T) { + snap := spaceSnapshot(nil) + snap.Blocks = append(snap.Blocks, &model.Block{Id: "p", + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Style: model.BlockContentText_Paragraph}}}) + assert.False(t, OmittedSpaceSettings(model.SmartBlockType_Workspace, snap), + "fail closed: an empty paragraph is still the author's block") + }) + + // The icon is the one member whose carrying can FAIL: an image that is + // not an object id cannot be written at all. On an ordinary object that + // warning travels with the document; here there would be no document. + t.Run("an icon the index cannot carry keeps the document", func(t *testing.T) { + assert.False(t, OmittedSpaceSettings(model.SmartBlockType_Workspace, + spaceSnapshot(map[string]*types.Value{ + detailKeyIconImage: str("https://example.com/logo.png")})), + "fail closed: the index would carry a lesser icon than the object holds") + }) +} + +// The space icon travels in index.json, in the one shape every icon in this +// format has (§2b). Measured over 77 corpus spaces: 55 an image, 20 a bare +// colour — the letter avatar — and 2 no icon at all. +// +// How this can fail: give the index a narrower icon than the object surface, +// and the 20 letter avatars are deleted by an export that reports success. +func TestSpaceSettings_TheIconTravelsInTheIndex(t *testing.T) { + t.Run("an image icon, with the colour it is tinted with", func(t *testing.T) { + var idx Index + IndexFromSpaceSettings(&idx, spaceSnapshot(map[string]*types.Value{ + detailKeyIconImage: str("bafyreiimage"), detailKeyIconOption: num(3)})) + + require.NotNil(t, idx.Icon) + assert.Equal(t, "file", idx.Icon.Format) + assert.Equal(t, "bafyreiimage", idx.Icon.File) + assert.NotNil(t, idx.Icon.Color, "the colour rides along with the icon it tints") + assert.Equal(t, "bafyreiimage", idx.IconImageId()) + }) + + t.Run("a letter avatar is a colour and nothing else", func(t *testing.T) { + var idx Index + IndexFromSpaceSettings(&idx, spaceSnapshot(map[string]*types.Value{ + detailKeyIconOption: num(3)})) + + require.NotNil(t, idx.Icon, "20 of 77 real spaces have exactly this icon") + assert.Equal(t, "color", idx.Icon.Format) + assert.NotNil(t, idx.Icon.Color) + }) + + t.Run("an icon that cannot be carried whole is not carried at all", func(t *testing.T) { + var idx Index + IndexFromSpaceSettings(&idx, spaceSnapshot(map[string]*types.Value{ + detailKeyIconImage: str("https://example.com/logo.png")})) + + assert.Nil(t, idx.Icon, "and the document is kept instead, so nothing is lost") + }) + + t.Run("no icon at all", func(t *testing.T) { + var idx Index + IndexFromSpaceSettings(&idx, spaceSnapshot(nil)) + assert.Nil(t, idx.Icon) + }) +} + +// The lift is the composer's half of the omission: a bundle that drops the +// document MUST write what it held, or the space loses its name. +// +// How this can fail: drop a field from IndexFromSpaceSettings and the +// omission starts losing it silently — the predicate would still say yes, +// because spaceSettingsIndexKeys claims the index carries it. +func TestSpaceSettings_TheIndexTakesWhatTheDocumentHeld(t *testing.T) { + // given + var idx Index + + // when + IndexFromSpaceSettings(&idx, spaceSnapshot(map[string]*types.Value{ + "description": str("What it is for")})) + + // then + assert.Equal(t, "My space", idx.Name) + assert.Equal(t, "What it is for", idx.Description) + assert.Equal(t, "bafyreihome", idx.Homepage) + + // and every key the predicate treats as index-carried is actually lifted. + // Compared against the index the SAME snapshot without that key produces: + // asserting merely that the result is non-empty proves nothing, because + // the base snapshot already carries a name and a homepage. + samples := map[string]*types.Value{ + "name": str("Another name"), "description": str("What it is for"), + "homepage": str("bafyreielsewhere"), + detailKeyIconEmoji: str("📚"), detailKeyIconImage: str("bafyreiimage"), + detailKeyIconName: str("folder"), detailKeyIconOption: num(3), + } + for stored := range spaceSettingsIndexKeys { + v, ok := samples[stored] + require.Truef(t, ok, "no sample value for index-carried key %q", stored) + + var without, with Index + IndexFromSpaceSettings(&without, spaceSnapshot(nil)) + IndexFromSpaceSettings(&with, spaceSnapshot(map[string]*types.Value{stored: v})) + require.NotEqualf(t, without, with, + "%q is listed as index-carried but the lift writes nothing for it", stored) + } +} + +// The STORE spells a reserved homepage the way core/domain/homepage.go does — +// a bare `widgets` — while the format spells it `_widgets`, inside the `_` +// namespace no bundle object may claim (§1). The lift has to translate, the +// way the profile writer translates in the other direction. +// +// Measured before the fix: 8 of 77 exported indexes carried +// `"homepage": "widgets"`, which the batch checker read as an object id +// naming nothing in the bundle. +// +// How this can fail: lift the stored value verbatim and a reserved screen +// becomes a dangling object reference; translate an ordinary object id and a +// real homepage stops resolving. +func TestSpaceSettings_AReservedHomepageIsTranslatedOnTheWayIn(t *testing.T) { + t.Run("the wire spelling becomes the format spelling", func(t *testing.T) { + var idx Index + IndexFromSpaceSettings(&idx, spaceSnapshot(map[string]*types.Value{ + "homepage": str("widgets")})) + assert.Equal(t, HomepageWidgets, idx.Homepage) + assert.True(t, IsReservedBundleId(idx.Homepage) || idx.Homepage[0] == '_', + "it must land in the reserved namespace, not look like an object id") + }) + + t.Run("an ordinary object id is untouched", func(t *testing.T) { + var idx Index + IndexFromSpaceSettings(&idx, spaceSnapshot(map[string]*types.Value{ + "homepage": str("bafyreihome")})) + assert.Equal(t, "bafyreihome", idx.Homepage) + }) + + t.Run("and the pair round-trips", func(t *testing.T) { + for _, wire := range []string{"widgets", "graph"} { + assert.Equal(t, wire, WireHomepage(FormatHomepage(wire))) + } + }) +} diff --git a/pkg/lib/anyblockjson/specclaims_test.go b/pkg/lib/anyblockjson/specclaims_test.go new file mode 100644 index 0000000000..fde0b7b99a --- /dev/null +++ b/pkg/lib/anyblockjson/specclaims_test.go @@ -0,0 +1,144 @@ +package anyblockjson + +// §15 records the designs this format rejected, and — since v0.22 — the +// evidence that rejects them. A rejected design comes back; a rejected design +// whose evidence has quietly stopped being true comes back and WINS. +// +// So the evidence is pinned here rather than left as prose. Each assertion is +// one fact §15 cites by name. A change that falsifies one is not necessarily +// wrong — it means a closed question is open again, and §15 has to be rewritten +// before the change lands. + +import ( + "encoding/json" + "reflect" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// §15.3, the separator design (`#`, deleted at v0.20): a joined key needs a +// byte that appears in neither half, and there is none — the sigil appears +// inside property slugs AND inside option names. +func TestSpecClaim_NoSeparatorSurvivesRealNames(t *testing.T) { + assert.Equal(t, "c#", bundle.ApiSlug("C#"), + "a property named C# slugs with the separator INSIDE it") + assert.Equal(t, "#1_priority", bundle.ApiSlug("#1 priority"), + "and one can begin with it") + assert.Equal(t, "c/c++", bundle.ApiSlug("C/C++"), + "`/` is no better: it survives slugging too") +} + +// §15.3, the sigil design (`"@opt-high"` marking a handle in the value). +// Falsified twice over. +func TestSpecClaim_TheSigilIsNotDistinguishableFromData(t *testing.T) { + // (a) a legal property slug can BEGIN with the sigil, so a marker made of + // it marks nothing + assert.Equal(t, "@home", bundle.ApiSlug("@home")) + + // (b) Validate takes bytes and returns an error — no resolver of any kind + // (§13), so it cannot know whether a /properties value is a select value + // or an object reference, and must either accept the sigil everywhere + // (breaking I2) or refuse it where Marshal emits it (breaking I1) + var _ func(data []byte) error = Validate + + // (c) and export's own deep links are not the counter-example they look + // like: the id is percent-encoded, so a leading `@` never reaches the wire + assert.Contains(t, objectLinkDest("@miovm"), "%40", + "objectLinkDest percent-encodes, so its output never starts a value with the sigil") +} + +// §15.3, the `{name, id}` value-pair design. It was believed to be a +// format-only change whose byte cost could be paid by the store already +// knowing each option's key. The store does not know it. +func TestSpecClaim_RelationOptionHasNoKeyField(t *testing.T) { + var names []string + rt := reflect.TypeOf(model.RelationOption{}) + for i := 0; i < rt.NumField(); i++ { + names = append(names, rt.Field(i).Name) + } + assert.Equal(t, []string{"Id", "Text", "Color", "RelationKey", "OrderId"}, names, + "a key field here would reopen §15.3's value-pair design") +} + +// §15.11, both directions on the §3 chain's store step (3c). +func TestSpecClaim_TheStoreStepIsNeitherRemovableNorPromotable(t *testing.T) { + // deleting it: the bundled fold knows nothing about a space's custom + // keys, so a reader without a store resolves a spelling it was just + // handed to nothing — and mints a duplicate relation beside it + assert.Empty(t, bundle.RelationKeysByApiFold("Severity"), + "a custom key's spelling folds to nothing in the bundled table") + // while the fold DOES answer for a bundled key, which is why step 3b is + // worth having at all + assert.Equal(t, []domain.RelationKey{bundle.RelationKeyDueDate}, bundle.RelationKeysByApiFold("due-date")) + + // promoting it: a space may hold a live stored type key `Task` — this + // format creates one, `{"kind": "object_type", "internal_key": "Task"}` is legal — + // and a mandatory fold would overrule verbatim-first (§3 step 2) and + // retype every reference to it onto the bundled Task type + assert.Equal(t, []domain.TypeKey{bundle.TypeKeyTask}, bundle.TypeKeysByApiFold("Task")) +} + +// §15.3's closing note: the sigil designs were largely defended as protecting +// `object_ids` against a dropped legend. There is no such member — object-ref +// compaction was deleted at v0.20 and object references print in full. The +// only `object_ids` in the format is the dataview's own field (§6.2). +func TestSpecClaim_TheOnlyObjectIdsIsTheDataviewField(t *testing.T) { + var schema map[string]any + require.NoError(t, json.Unmarshal(schemaJSON, &schema)) + + // every `properties` map in the schema that declares an object_ids + // member, reported by its sibling members — which is what identifies the + // object it belongs to without depending on how $refs are reached + var siblings [][]string + var walk func(node any) + walk = func(node any) { + switch n := node.(type) { + case map[string]any: + for k, v := range n { + if k == "properties" { + if props, ok := v.(map[string]any); ok { + if _, has := props["object_ids"]; has { + var names []string + for name := range props { + names = append(names, name) + } + sort.Strings(names) + siblings = append(siblings, names) + } + for _, sub := range props { + walk(sub) + } + continue + } + } + walk(v) + } + case []any: + for _, v := range n { + walk(v) + } + } + } + walk(schema) + + require.Len(t, siblings, 1, "object_ids appears in more than one place: %v", siblings) + assert.Equal(t, []string{"group_id", "object_ids"}, siblings[0], + "the only object_ids is the dataview's manual object order (§6.2)") +} + +// §15.3's "additive within a version" half: the `{…}` container design argued +// that a legend member could be added later without a version bump. It could +// not — the envelope is closed, and §10 has no additive rule. +func TestSpecClaim_TheEnvelopeIsClosed(t *testing.T) { + var schema map[string]any + require.NoError(t, json.Unmarshal(schemaJSON, &schema)) + assert.Equal(t, false, schema["additionalProperties"], + "an unknown envelope member is refused, so nothing can be added within a version") +} diff --git a/pkg/lib/anyblockjson/storeresolver/keyvocab.go b/pkg/lib/anyblockjson/storeresolver/keyvocab.go new file mode 100644 index 0000000000..81abbb8802 --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/keyvocab.go @@ -0,0 +1,655 @@ +package storeresolver + +// keyvocab.go — the space-backed key vocabulary. +// +// The package default (anyblockjson.BundledKeyVocabulary) knows only the +// bundled name table, which is all an offline reader can know. Inside a +// node the space itself is the second authority: every non-bundled type and +// property has a display NAME, and §3 says the document spells THAT — NFC, +// verbatim — not the opaque BSON the store binds and not the api slug the +// API surface mints (`apiObjectKey` is never read by the format; it stays +// the API's affair). +// +// Shape, exactly as §3 prescribes: **one bounded query per kind per +// resolver instance** (i.e. per export/import operation), never one point +// query per reference. The listing that primes it is a DETAILS query rather +// than ListAllRelations, because the vocabulary needs only names, keys and +// flags. Precedence follows the §3 chain, and the ordering is load-bearing: +// an exact live STORED key always wins over the name layer (step 2 before +// any table), so a document naming a relation whose stored key happens to +// be spelled like someone else's name still binds to the relation it named. +// The vocabulary is lazy: a document with no key slots pays nothing. +// +// **Names are not unique, and the vocabulary does not pretend they are.** +// Two live properties may share one name; both spell it, because collisions +// are resolved per DOCUMENT (the exporter's term ledger disambiguates the +// 0.21% of documents where two claimants actually co-occur), not per space. +// The accept side therefore answers a shared spelling with NO single key — +// PropertyKey refuses to guess — and exposes the full candidate list +// through the ScopedKeyVocabulary capability, where the importer resolves +// within the declared type or raises a loud error asking for the legend. +// The one spelling an entity can NEVER take, in any document, is a string +// that is some other live entity's stored key: stored keys resolve +// verbatim-first at every reader, so that claimant degrades through the +// same ladder the document ledger uses — its stored key when that is +// readable, else ` ()` with the stored key's last six hex, +// else the stored key regardless. + +import ( + "sort" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/database" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// keyMaps is one namespace's spelling tables plus the stored-key set that +// gives chain step 2 its precedence. +type keyMaps struct { + // labelByKey is the granted document spelling per live visible entity — + // the plain NFC name for nearly everyone, the disambiguated form where + // the name was unusable (see grant). Absent means the entity spells its + // stored key verbatim, which is always its own address. + labelByKey map[string]string + // keysByLabel is the reverse, MULTI-VALUED: a shared name lists every + // claimant, in sorted-key order, and the accept side refuses to pick. + keysByLabel map[string][]string + // nameByKey is the raw NFC display name of every visible non-bundled + // entity — the diagnostics surface for the glued-annotation warning, + // kept separate from labelByKey because a degraded label is not the + // name a writer would have copied. + nameByKey map[string]string + storedKey map[string]bool + // keysByFold is chain step 4 — see fold. + keysByFold map[string][]string + // keyById is the stored key of each live entity by OBJECT ID — not part + // of the resolution chain, which never sees an id, but the inverse the + // store speaks in: a relation's `relationFormatObjectTypes` holds the + // target types' object ids, and PropertyDefinition.ObjectTypes is + // defined in stored type KEYS (§2a). Filled from the same one bounded + // listing, so the mapping costs nothing extra. Hidden entities are + // included: identity is not the name namespace. + // + // idByKey is its inverse, for the import half of the same translation + // (anyblockjson.TypeResolver, §2d). First-wins on a duplicated key, + // like keyById on a duplicated id. + keyById map[string]string + idByKey map[string]string + // propertyIdsByKey exists on the TYPE namespace only: the four + // recommended property lists of each live type, as the object ids the + // store holds, read from the same one bounded listing. It is the + // type-scoped resolution surface (TypePropertyKeys): the declared type + // is what disambiguates a shared property name for a reader with no + // legend. + propertyIdsByKey map[string][]string + + bundledKey func(spelling string) (string, bool) + bundledFold func(input string) []string + // bundled reports whether the bundled table speaks for a stored key. + // Such a key takes its spelling from the code table in every space and + // offline (§3), so the space's own row never derives one for it. + bundled func(key string) bool + // label is the namespace's half of the §3 label rule + // (anyblockjson.PropertyLabel / TypeLabel): NFC(name), else nothing. + label func(key, name string) string +} + +// namespace is one key namespace's configuration: the listing it loads +// from, the two bundled tables it consults, and its label rule. +type namespace struct { + layout model.ObjectTypeLayout + keyOf func(*domain.Details) string + label func(key, name string) string + bundled func(key string) bool + bundledKey func(spelling string) (string, bool) + bundledFold func(input string) []string +} + +// entity is one row of the one bounded listing: everything the space stores +// that a spelling can be built from. +type entity struct { + key string + name string + hidden bool +} + +func newKeyMaps(ns namespace) *keyMaps { + return &keyMaps{ + labelByKey: map[string]string{}, + keysByLabel: map[string][]string{}, + nameByKey: map[string]string{}, + storedKey: map[string]bool{}, + keysByFold: map[string][]string{}, + keyById: map[string]string{}, + idByKey: map[string]string{}, + propertyIdsByKey: map[string][]string{}, + bundledKey: ns.bundledKey, + bundledFold: ns.bundledFold, + bundled: ns.bundled, + label: ns.label, + } +} + +// add records one live entity. +// +// A HIDDEN entity keeps its stored key (chain step 2 — the stored key is +// always an address, and the emit side must still refuse to grant a +// spelling it owns) but does NOT enter the name namespace, exactly as v2's +// request namespace has it (core/api/v2/service/keys.go, +// propertyEntry.Hidden). A BUNDLED entity keeps its stored key too and +// contributes nothing else: its spelling is the code table's in every space +// and offline, and its folds are the bundled fold table's — letting a space +// row's name speak for it would let a renamed local copy move a spelling +// that ships with every reader. +func (m *keyMaps) add(row entity) { + if row.key == "" { + return + } + m.storedKey[row.key] = true +} + +// grant runs the label pass for one visible non-bundled entity, after every +// stored key is known — the ordering is the whole reason granting is a +// second pass: the one hard refusal below needs the complete stored-key +// set, and the order rows arrive in must not decide anyone's spelling. +// +// A shared name is NOT refused: collisions are per-document (the exporter's +// ledger), so every claimant is granted the plain name and keysByLabel +// holds them all. The one spelling no entity may take is a string that is +// some other live entity's STORED KEY — verbatim-first outranks every +// table, so such a label could never resolve to its owner anywhere. That +// claimant degrades through the same ladder the document ledger uses: +// +// (a) its stored key, by granting NO label, when the key is readable +// (not a minted 24-hex bson id); +// (b) else ` ()`, tail6 = the stored key's last six hex — +// deterministic, immutable, visibly synthetic; +// (c) else no label, and the stored key is written regardless. +func (m *keyMaps) grant(row entity) { + if row.key == "" || row.hidden || m.bundled(row.key) { + return + } + name := m.label(row.key, row.name) + if name != "" { + m.nameByKey[row.key] = name + } + m.addFold(anyblockjson.FoldKeyTerm(row.key), row.key) + if name == "" { + return + } + m.addFold(anyblockjson.FoldKeyTerm(name), row.key) + label := name + if m.storedKey[label] { // necessarily someone else's: label==own key yields "" above + label = anyblockjson.DisambiguatedKeySpelling(name, row.key) + if label == "" || m.storedKey[label] { + return // rung (a) or (c): the stored key is the spelling + } + m.addFold(anyblockjson.FoldKeyTerm(label), row.key) + } + m.labelByKey[row.key] = label + m.addClaimant(label, row.key) +} + +// addClaimant records one claimant of a spelling, first-wins on a repeat — +// the same guard addFold carries just below, for a sharper reason. keysByLabel +// is the ambiguity signal the accept side reads: TWO entries mean two live +// entities, so keyMaps.key refuses to pick and the importer stops to ask for a +// legend. One entity listed twice therefore refuses a document the exporter +// had just written, and nothing about it is recoverable downstream — the +// reader sees a candidate count, not a row count. +// +// Nothing in the listing promises a stored key reaches grant once. The +// relation namespace reads its key off the `relationKey` DETAIL, not off the +// row identity, so a legacy row and its derived twin both carrying one key are +// two rows and one entity; the type namespace has the same shape through +// uniqueKey. The neighbouring maps have taken first-wins against exactly that +// for as long as they have existed (keyById, idByKey, propertyIdsByKey, and +// relKeyToId one file over), and GetRelationByKey answers a duplicated +// relationKey with records[0] — the guard belongs here rather than in an +// argument that the duplicate cannot happen. +func (m *keyMaps) addClaimant(label, key string) { + for _, existing := range m.keysByLabel[label] { + if existing == key { + return + } + } + m.keysByLabel[label] = append(m.keysByLabel[label], key) +} + +func (m *keyMaps) addFold(fold, key string) { + for _, existing := range m.keysByFold[fold] { + if existing == key { + return + } + } + m.keysByFold[fold] = append(m.keysByFold[fold], key) +} + +// candidates is the exact name layer's full answer for a term: every live +// visible claimant of the spelling, plus the bundled table's binding when +// it has one — sorted, deduplicated. It deliberately says nothing about +// stored keys: verbatim-first is the caller's step, asked before this one. +// +// This is the method that makes the published contract true — the candidate +// list is a SET, and its LENGTH is what every caller reads as "how many live +// entities answer to this spelling". addClaimant keeps keysByLabel a set at +// build time; the dedup here is the second belt, so that the answer stays a +// set whatever a future population pass puts in the map. Belt and braces on +// one number is cheap: the map is allocated only for a term someone actually +// asks about, and getting the number wrong costs a refused document. +func (m *keyMaps) candidates(term string) []string { + claimants := m.keysByLabel[term] + out := make([]string, 0, len(claimants)+1) + seen := make(map[string]bool, len(claimants)+1) + add := func(key string) { + if key == "" || seen[key] { + return + } + seen[key] = true + out = append(out, key) + } + for _, key := range claimants { + add(key) + } + if m.bundledKey != nil { + if key, ok := m.bundledKey(term); ok { + add(key) + } + } + if len(out) == 0 { + return nil + } + sort.Strings(out) + return out +} + +// key is the accept side's exact chain for one term: verbatim-first, then +// the name layer where EXACTLY ONE candidate holds the spelling. Several +// candidates are an ambiguity this method refuses to resolve — the caller +// with type context (the importer, through ScopedKeyVocabulary) may; a +// caller without one degrades to the verbatim term, never to a guess. +func (m *keyMaps) key(term string) (string, bool) { + if m.storedKey[term] { + return "", false // chain step 2: an exact stored key wins + } + if cands := m.candidates(term); len(cands) == 1 { + return cands[0], true + } + return "", false +} + +// fold is chain step 4, the forgiving layer: every exact lookup has already +// failed, so a SINGLE key whose stored key, name or granted label folds to +// the input's class is the intended forgiveness, and several are an +// ambiguity that degrades to the verbatim term — never a guess. Hidden +// holders do not participate, as at every other step. +func (m *keyMaps) fold(input string) (string, bool) { + stored := m.keysByFold[anyblockjson.FoldKeyTerm(input)] + candidates := append(make([]string, 0, len(stored)+1), stored...) + if m.bundledFold != nil { + seen := make(map[string]bool, len(candidates)) + for _, c := range candidates { + seen[c] = true + } + for _, key := range m.bundledFold(input) { + if !seen[key] { + seen[key] = true + candidates = append(candidates, key) + } + } + } + if len(candidates) == 1 { + return candidates[0], true + } + return "", false +} + +// extendsLiveName reports the live visible entity NAME that term extends +// with trailing text past a word boundary — the glued-annotation +// diagnostic. Longest name wins; equal lengths break lexicographically. +func (m *keyMaps) extendsLiveName(term string) string { + var best string + for _, name := range m.nameByKey { + if !anyblockjson.KeyTermExtendsName(term, name) { + continue + } + if len(name) > len(best) || (len(name) == len(best) && name < best) { + best = name + } + } + return best +} + +func (r *Resolvers) relationKeyMaps() *keyMaps { + if r.relVocab == nil { + r.relVocab = r.loadKeyMaps(namespace{ + layout: model.ObjectType_relation, + keyOf: func(d *domain.Details) string { + return d.GetString(bundle.RelationKeyRelationKey) + }, + label: anyblockjson.PropertyLabel, + bundled: func(key string) bool { + return bundle.HasRelation(domain.RelationKey(key)) + }, + // the bundled arms run through anyblockjson's name tables, not + // pkg/lib/bundle's slug tables: bundled keys answer to their + // display names (and to their key/name fold classes) — the same + // chain the package-only reader runs, or the two disagree on + // one spelling + bundledKey: anyblockjson.BundledPropertyKeyByName, + bundledFold: anyblockjson.BundledPropertyKeysByFold, + }) + } + return r.relVocab +} + +func (r *Resolvers) typeKeyMaps() *keyMaps { + if r.typeVocab == nil { + r.typeVocab = r.loadKeyMaps(namespace{ + layout: model.ObjectType_objectType, + keyOf: func(d *domain.Details) string { + key, err := domain.GetTypeKeyFromRawUniqueKey(d.GetString(bundle.RelationKeyUniqueKey)) + if err != nil { + return "" + } + return string(key) + }, + label: anyblockjson.TypeLabel, + bundled: func(key string) bool { + return bundle.HasObjectTypeByKey(domain.TypeKey(key)) + }, + bundledKey: anyblockjson.BundledTypeKeyByName, + bundledFold: anyblockjson.BundledTypeKeysByFold, + }) + } + return r.typeVocab +} + +// recommendedListDetailKeys are the four stored lists a type declares its +// properties through — the same lists the exporter's §2a machinery reads, +// restated here because this package reads them off raw details rows. +var recommendedListDetailKeys = []domain.RelationKey{ + bundle.RelationKeyRecommendedFeaturedRelations, + bundle.RelationKeyRecommendedRelations, + bundle.RelationKeyRecommendedFileRelations, + bundle.RelationKeyRecommendedHiddenRelations, +} + +// loadKeyMaps runs the one bounded listing. A store error yields an EMPTY +// vocabulary, not a partial one: the caller then falls back to the bundled +// table, which is the offline-safe answer — never a stale or half-built +// map, which would resolve a write against the wrong property. +func (r *Resolvers) loadKeyMaps(ns namespace) *keyMaps { + maps := newKeyMaps(ns) + // ONE listing, TWO populations. The uninstalled entity is excluded from + // the NAME namespace and included in the id→key naming, and the two + // questions are genuinely different: + // + // - "which entity answers to the spelling `Project`?" — a UI-deleted + // type must vacate it, or a new type minted under the freed name is + // shadowed by a corpse. That policy is why this listing filtered + // uninstalled entities out in the first place. + // + // - "what does the id I am HOLDING point at?" — naming a corpse + // claims nothing. The store never removes the type, so the answer + // exists; refusing to look was what put raw object ids into + // `object_types`, where the slot's vocabulary is type KEYS. + // + // The filter therefore lives on the name half, not on the query. + records, err := r.index.Query(database.Query{Filters: []database.FilterRequest{ + { + RelationKey: bundle.RelationKeyResolvedLayout, + Condition: model.BlockContentDataviewFilter_Equal, + Value: domain.Int64(int64(ns.layout)), + }, + }}) + if err != nil { + return maps + } + rows := make([]entity, 0, len(records)) + // live entities first, so that where a freed spelling HAS been retaken + // the living owner claims id↔key before any corpse sharing its key + for _, pass := range []bool{false, true} { + for _, record := range records { + uninstalled := record.Details.GetBool(bundle.RelationKeyIsUninstalled) + if uninstalled != pass { + continue + } + key := ns.keyOf(record.Details) + if !uninstalled { + rows = append(rows, entity{ + key: key, + name: record.Details.GetString(bundle.RelationKeyName), + hidden: record.Details.GetBool(bundle.RelationKeyIsHidden), + }) + if ns.layout == model.ObjectType_objectType && key != "" { + if _, taken := maps.propertyIdsByKey[key]; !taken { + var ids []string + for _, l := range recommendedListDetailKeys { + ids = append(ids, record.Details.GetStringList(l)...) + } + maps.propertyIdsByKey[key] = ids + } + } + } + if id := record.Details.GetString(bundle.RelationKeyId); id != "" && key != "" { + if _, taken := maps.keyById[id]; !taken { + maps.keyById[id] = key + } + if _, taken := maps.idByKey[key]; !taken { + maps.idByKey[key] = id + } + } + } + } + // two passes, in this order: every stored key is registered first, and + // only then are labels granted — the grant's one hard refusal (a name + // that is someone else's stored key) needs the complete set, and rows + // are sorted so nothing depends on store order. + for _, row := range rows { + maps.add(row) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].key < rows[j].key }) + for _, row := range rows { + maps.grant(row) + } + return maps +} + +// PropertySlug implements anyblockjson.KeyVocabulary: bundled keys spell +// their display name from the code table (the authority in every space and +// offline — §3), the rest their granted label. Either arm still yields to a +// live stored key that owns the very string (verbatim-first): the granted +// labels were vetted against the stored-key set at build time, so only the +// bundled arm re-checks here. +func (r *Resolvers) PropertySlug(key string) string { + maps := r.relationKeyMaps() + if maps.bundled(key) { + candidate := (anyblockjson.BundledKeyVocabulary{}).PropertySlug(key) + if candidate != key && !maps.storedKey[candidate] { + return candidate + } + return key + } + if label := maps.labelByKey[key]; label != "" { + return label + } + return key +} + +func (r *Resolvers) PropertyKey(spelling string) (string, bool) { + maps := r.relationKeyMaps() + if maps.storedKey[spelling] { + return spelling, false // chain step 2 — verbatim, no table consulted + } + if key, ok := maps.key(spelling); ok { + return key, true + } + if len(maps.candidates(spelling)) > 1 { + return spelling, false // ambiguous: never resolved by guess here + } + if key, ok := maps.fold(spelling); ok { + return key, true + } + return spelling, false +} + +// TypeSlug is PropertySlug for the type namespace. +func (r *Resolvers) TypeSlug(key string) string { + maps := r.typeKeyMaps() + if maps.bundled(key) { + candidate := (anyblockjson.BundledKeyVocabulary{}).TypeSlug(key) + if candidate != key && !maps.storedKey[candidate] { + return candidate + } + return key + } + if label := maps.labelByKey[key]; label != "" { + return label + } + return key +} + +func (r *Resolvers) TypeKey(spelling string) (string, bool) { + maps := r.typeKeyMaps() + if maps.storedKey[spelling] { + return spelling, false + } + if key, ok := maps.key(spelling); ok { + return key, true + } + if len(maps.candidates(spelling)) > 1 { + return spelling, false + } + if key, ok := maps.fold(spelling); ok { + return key, true + } + return spelling, false +} + +// PropertyKeyCandidates, TypeKeyCandidates, TypePropertyKeys, +// PropertyTermFacts and TypeTermFacts implement +// anyblockjson.ScopedKeyVocabulary — the capability the importer discovers +// by assertion to resolve a shared name within the declared type, and to +// diagnose a term it is about to store verbatim. + +func (r *Resolvers) PropertyKeyCandidates(spelling string) []string { + return r.relationKeyMaps().candidates(spelling) +} + +func (r *Resolvers) TypeKeyCandidates(spelling string) []string { + return r.typeKeyMaps().candidates(spelling) +} + +// TypePropertyKeys returns the stored property keys the type declares +// through its four recommended lists — the space's own row where the type +// is installed, the bundled table's links otherwise. The store speaks in +// relation object ids, so the answer is translated through the relation +// namespace's id→key map, dropping ids the space cannot name (a dropped id +// only narrows the scope, which degrades toward the loud error rather than +// toward a wrong resolution). +// +// The answer is a SET, like the candidate lists, and for the same reason: the +// importer INTERSECTS it with the candidates and counts what survives, so a +// property the type names twice reads as two claimants and the type stops +// being able to single out its own property — the exact opposite of what the +// scope exists for. Nothing declares the four lists disjoint (a featured +// property sitting in `recommendedRelations` as well is a single misordered +// write away, and the bundled arm concatenates RelationLinks the same way), so +// the deduplication is done here, once, rather than assumed at the four +// sites that read it. +func (r *Resolvers) TypePropertyKeys(typeKey string) []string { + if typeKey == "" { + return nil + } + tm := r.typeKeyMaps() + if ids, ok := tm.propertyIdsByKey[typeKey]; ok { + rm := r.relationKeyMaps() + keys := newKeySet(len(ids)) + for _, id := range ids { + if key, ok := rm.keyById[id]; ok && key != "" { + keys.add(key) + } else if key, err := bundle.RelationKeyFromID(id); err == nil { + keys.add(string(key)) + } + } + return keys.keys + } + if t, err := bundle.GetType(domain.TypeKey(typeKey)); err == nil { + keys := newKeySet(len(t.RelationLinks)) + for _, l := range t.RelationLinks { + if l != nil { + keys.add(l.Key) + } + } + return keys.keys + } + return nil +} + +// keySet accumulates stored keys in first-seen order, dropping repeats and +// the empty key. Order is kept rather than sorted: the recommended lists are +// the type's own ordering and a caller reading them for anything but the +// count would lose it. +type keySet struct { + keys []string + seen map[string]bool +} + +func newKeySet(size int) *keySet { + return &keySet{keys: make([]string, 0, size), seen: make(map[string]bool, size)} +} + +func (s *keySet) add(key string) { + if key == "" || s.seen[key] { + return + } + s.seen[key] = true + s.keys = append(s.keys, key) +} + +func (r *Resolvers) PropertyTermFacts(term string) anyblockjson.KeyTermFacts { + maps := r.relationKeyMaps() + return anyblockjson.KeyTermFacts{ + LiveStoredKey: maps.storedKey[term], + ExtendsName: maps.extendsLiveName(term), + } +} + +func (r *Resolvers) TypeTermFacts(term string) anyblockjson.KeyTermFacts { + maps := r.typeKeyMaps() + return anyblockjson.KeyTermFacts{ + LiveStoredKey: maps.storedKey[term], + ExtendsName: maps.extendsLiveName(term), + } +} + +// TypeKeyById and TypeIdByKey implement anyblockjson.TypeResolver (§2d): the +// translation between the type object ids `relationFormatObjectTypes` stores +// and the stored type keys the format spells. It is targetTypeKeys' own +// mapping (keyById, filled from the one bounded type listing) surfaced as +// the capability the codec discovers by assertion, plus the bundled-url arm +// for legacy entries that were never rewritten to derived ids. +// +// Both answer false on a miss, deliberately: the codec's degradation for an +// unanswered entry is verbatim pass-through — its own address (§3) — and an +// invented answer here would translate one direction with nothing to invert +// it on the other. +func (r *Resolvers) TypeKeyById(id string) (string, bool) { + if key, err := bundle.TypeKeyFromUrl(id); err == nil && key != "" { + return string(key), true + } + if key := r.typeKeyMaps().keyById[id]; key != "" { + return key, true + } + return "", false +} + +func (r *Resolvers) TypeIdByKey(key string) (string, bool) { + if id := r.typeKeyMaps().idByKey[key]; id != "" { + return id, true + } + return "", false +} diff --git a/pkg/lib/anyblockjson/storeresolver/keyvocab_test.go b/pkg/lib/anyblockjson/storeresolver/keyvocab_test.go new file mode 100644 index 0000000000..70dbb9d2c3 --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/keyvocab_test.go @@ -0,0 +1,829 @@ +package storeresolver + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The space-backed key vocabulary (§3): a key's document spelling is its +// display NAME, NFC and otherwise verbatim, and the accept side inverts +// exactly what the emit side writes. Names are not unique, so the accept +// side REFUSES a shared spelling rather than picking a holder, and exposes +// the candidates through ScopedKeyVocabulary for the importer's type-scoped +// resolution. `apiObjectKey` is never read: the last test pins that a +// stored slug moves nothing here any more. + +const ( + bsonPropKey = "6a7663db61fab21cd4b9e101" + bsonTwinKey = "6a7663db61fab21cd4b9e102" + bsonTypeKey = "6a7663db61fab21cd4b9e103" +) + +// named gives a row the display name the §3 label rule reads. +func named(row spaceindex.TestObject, name string) spaceindex.TestObject { + row[bundle.RelationKeyName] = domain.String(name) + return row +} + +// nameless strips a row's display name: such an entity has no label but its +// stored key. +func nameless(row spaceindex.TestObject) spaceindex.TestObject { + delete(row, bundle.RelationKeyName) + return row +} + +// vocabFixture builds a resolver over exactly the rows given. +func vocabFixture(t *testing.T, objects ...spaceindex.TestObject) *Resolvers { + index := spaceindex.NewStoreFixture(t) + if len(objects) > 0 { + index.AddObjects(t, objects) + } + return New(index) +} + +func relationRow(id, key, name string) spaceindex.TestObject { + row := spaceindex.TestObject{ + bundle.RelationKeyId: domain.String(id), + bundle.RelationKeyRelationKey: domain.String(key), + bundle.RelationKeyName: domain.String(name), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_relation)), + } + return row +} + +func typeRow(id, key, name string) spaceindex.TestObject { + row := spaceindex.TestObject{ + bundle.RelationKeyId: domain.String(id), + bundle.RelationKeyUniqueKey: domain.String("ot-" + key), + bundle.RelationKeyName: domain.String(name), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_objectType)), + } + return row +} + +func TestPropertyKeyVocabulary(t *testing.T) { + t.Run("a name spells the BSON key, both directions", func(t *testing.T) { + // given + r := vocabFixture(t, relationRow("rel-manual", bsonPropKey, "Manual property")) + + // when / then + assert.Equal(t, "Manual property", r.PropertySlug(bsonPropKey)) + key, ok := r.PropertyKey("Manual property") + require.True(t, ok) + assert.Equal(t, bsonPropKey, key) + }) + + t.Run("a bundled key spells its display name from the code table", func(t *testing.T) { + r := vocabFixture(t, relationRow("rel-due", "dueDate", "Due date")) + + assert.Equal(t, "Due date", r.PropertySlug("dueDate")) + key, ok := r.PropertyKey("Due date") + require.True(t, ok) + assert.Equal(t, "dueDate", key) + }) + + // chain step 2: revert the storedKey branch in PropertyKey and this + // resolves to the BUNDLED relation instead — a write addressed at the + // legacy relation landing on a different property. + t.Run("an exact live stored key wins over every table", func(t *testing.T) { + // given: a legacy relation STORED under the very string that is + // bundled dueDate's display name + r := vocabFixture(t, + relationRow("rel-legacy", "Due date", "Something else"), + relationRow("rel-due", "dueDate", "Due date"), + ) + + // when + key, ok := r.PropertyKey("Due date") + + // then + assert.False(t, ok, "not a name — a stored key, verbatim") + assert.Equal(t, "Due date", key, "the bundled table is never consulted") + // and the emit side agrees: the bundled key cannot spell a term a + // live stored key owns, so it degrades to its own stored key + assert.Equal(t, "dueDate", r.PropertySlug("dueDate")) + }) + + t.Run("a key the vocabulary does not know passes through both ways", func(t *testing.T) { + r := vocabFixture(t) + + assert.Equal(t, "whatever", r.PropertySlug("whatever")) + key, ok := r.PropertyKey("whatever") + assert.False(t, ok) + assert.Equal(t, "whatever", key) + }) + + // Names are NOT unique, and the vocabulary does not pretend they are: + // collisions are per DOCUMENT (the exporter's term ledger), so both + // holders spell the plain name — and the accept side refuses to pick. + t.Run("a shared name is spelled by every holder and resolved by none", func(t *testing.T) { + // given — the corpus shape: two live properties named "Projects" + r := vocabFixture(t, + relationRow("rel-a", bsonPropKey, "Projects"), + relationRow("rel-b", bsonTwinKey, "Projects"), + ) + + // when / then + assert.Equal(t, "Projects", r.PropertySlug(bsonPropKey)) + assert.Equal(t, "Projects", r.PropertySlug(bsonTwinKey), + "both spell it: a name ambiguous space-wide is still unambiguous in nearly every document") + _, ok := r.PropertyKey("Projects") + assert.False(t, ok, "an ambiguous address must never resolve by store order") + assert.Equal(t, []string{bsonPropKey, bsonTwinKey}, r.PropertyKeyCandidates("Projects"), + "the candidates are exposed for the importer's type-scoped resolution") + }) + + t.Run("a nameless relation has no label but its stored key", func(t *testing.T) { + r := vocabFixture(t, nameless(relationRow("rel-x", bsonPropKey, ""))) + + assert.Equal(t, bsonPropKey, r.PropertySlug(bsonPropKey), "the stored key is always its own address") + }) + + // The ladder: the one spelling no entity may take is another live + // entity's STORED KEY — verbatim-first outranks every table, so such a + // label could never resolve to its owner anywhere. + t.Run("a name that is someone's stored key degrades through the ladder", func(t *testing.T) { + // given: a bson-keyed relation NAMED with the exact string another + // relation is stored under + r := vocabFixture(t, + named(relationRow("rel-named", bsonPropKey, ""), "manual_property"), + named(relationRow("rel-stored", "manual_property", ""), "Something else"), + ) + + // when / then — rung (b): the claimant's own key is a minted bson + // id, so it takes ` ()`, deterministic and invertible + assert.Equal(t, "manual_property (b9e101)", r.PropertySlug(bsonPropKey)) + key, ok := r.PropertyKey("manual_property") + assert.False(t, ok, "an exact stored key wins over any label") + assert.Equal(t, "manual_property", key) + }) + + t.Run("a readable stored key is its own disambiguation", func(t *testing.T) { + // rung (a): the claimant's own key is readable, so it spells itself + r := vocabFixture(t, + named(relationRow("rel-named", "producer_region", ""), "wine_region"), + named(relationRow("rel-stored", "wine_region", ""), "Region"), + ) + + assert.Equal(t, "producer_region", r.PropertySlug("producer_region"), + "a readable stored key needs no suffix — it is the honest spelling") + }) + + // A UI-deleted holder vacates the name namespace. Drop the + // isUninstalled filter and the freed name is shadowed by a corpse no + // listing shows. + t.Run("a UI-deleted holder vacates the name namespace", func(t *testing.T) { + // given + corpse := relationRow("rel-corpse", bsonTwinKey, "Warranty until") + corpse[bundle.RelationKeyIsUninstalled] = domain.Bool(true) + r := vocabFixture(t, relationRow("rel-live", bsonPropKey, "Warranty until"), corpse) + + // when / then + assert.Equal(t, "Warranty until", r.PropertySlug(bsonPropKey), "the live holder keeps the name") + key, ok := r.PropertyKey("Warranty until") + require.True(t, ok) + assert.Equal(t, bsonPropKey, key) + assert.Equal(t, bsonTwinKey, r.PropertySlug(bsonTwinKey), "the corpse spells nothing") + }) + + t.Run("the emitted spelling always inverts back to the key it labels", func(t *testing.T) { + // the whole contract in one loop, over a space holding every shape + // at once: whatever PropertySlug emits either inverts to the same + // key or IS the key (its own address) + r := vocabFixture(t, + relationRow("rel-manual", bsonPropKey, "Manual property"), + relationRow("rel-twinA", bsonTwinKey, "Shared name"), + relationRow("rel-twinB", "6a7663db61fab21cd4b9e104", "Shared name"), + relationRow("rel-due", "dueDate", "Due date"), + relationRow("rel-shadow", "6a7663db61fab21cd4b9e105", "Due date"), + relationRow("rel-legacy", "manual_alias", "Legacy"), + named(relationRow("rel-squat", "6a7663db61fab21cd4b9e106", ""), "manual_alias"), + ) + + for _, key := range []string{ + bsonPropKey, bsonTwinKey, "6a7663db61fab21cd4b9e104", "dueDate", + "6a7663db61fab21cd4b9e105", "manual_alias", "6a7663db61fab21cd4b9e106", + } { + spelling := r.PropertySlug(key) + back, ok := r.PropertyKey(spelling) + if !ok { + // not a unique name: the emitted spelling is then either the + // key itself (always an address) or a spelling the DOCUMENT + // ledger will disambiguate — never one that inverts elsewhere + if spelling == key { + assert.Equal(t, key, back, "emitted %q for %q", spelling, key) + continue + } + assert.Contains(t, r.PropertyKeyCandidates(spelling), key, + "emitted the shared %q for %q — the holder must be among its candidates", spelling, key) + continue + } + assert.Equal(t, key, back, "emitted %q for %q, which inverts elsewhere", spelling, key) + } + }) +} + +func TestTypeKeyVocabulary(t *testing.T) { + t.Run("a name spells the BSON type key, both directions", func(t *testing.T) { + r := vocabFixture(t, typeRow("type-meeting", bsonTypeKey, "Meeting note")) + + assert.Equal(t, "Meeting note", r.TypeSlug(bsonTypeKey)) + key, ok := r.TypeKey("Meeting note") + require.True(t, ok) + assert.Equal(t, bsonTypeKey, key) + }) + + t.Run("a bundled type spells its display name", func(t *testing.T) { + r := vocabFixture(t, typeRow("type-objectType", "objectType", "Type")) + + assert.Equal(t, "Type", r.TypeSlug("objectType")) + key, ok := r.TypeKey("Type") + require.True(t, ok) + assert.Equal(t, "objectType", key) + }) + + t.Run("an exact live stored type key wins over every table", func(t *testing.T) { + r := vocabFixture(t, + typeRow("type-legacy", "Task", "Legacy task"), + typeRow("type-task", "task", "Task"), + ) + + key, ok := r.TypeKey("Task") + assert.False(t, ok) + assert.Equal(t, "Task", key) + assert.Equal(t, "task", r.TypeSlug("task"), + "and the bundled type degrades to its own stored key") + }) + + t.Run("a shared type name is spelled by every holder and resolved by none", func(t *testing.T) { + r := vocabFixture(t, + typeRow("type-a", bsonTypeKey, "Meeting note"), + typeRow("type-b", "6a7663db61fab21cd4b9e107", "Meeting note"), + ) + + _, ok := r.TypeKey("Meeting note") + assert.False(t, ok) + assert.Equal(t, "Meeting note", r.TypeSlug(bsonTypeKey)) + assert.Equal(t, "Meeting note", r.TypeSlug("6a7663db61fab21cd4b9e107")) + assert.Len(t, r.TypeKeyCandidates("Meeting note"), 2) + }) +} + +// TestKeyVocabularyWiring pins that the resolvers ARE the vocabulary the +// Options carry — the read half and the write half must hand the codec the +// same table. +func TestKeyVocabularyWiring(t *testing.T) { + r := vocabFixture(t) + + opts := r.Options() + + assert.Equal(t, r, opts.Keys) +} + +// hiddenRelationRow is a relation the app hides from the user: invisible in +// every listing, and undeletable through the API — whatever spelling it +// occupied would be occupied forever with no visible cause, which is why it +// occupies none. +func hiddenRelationRow(id, key, name string) spaceindex.TestObject { + row := relationRow(id, key, name) + row[bundle.RelationKeyIsHidden] = domain.Bool(true) + return row +} + +// TestHiddenHoldersDoNotOwnNames is the one-place-for-one-rule test: v2's +// request namespace excludes hidden holders (core/api/v2/service/keys.go), +// and this vocabulary — which decides what a DOCUMENT's keys bind to — must +// agree, or the listing serves an address the write half resolves elsewhere. +func TestHiddenHoldersDoNotOwnNames(t *testing.T) { + t.Run("a hidden twin does not make a visible holder's name ambiguous", func(t *testing.T) { + // given — a visible relation and a hidden one both named "Severity" + r := vocabFixture(t, + relationRow("rel-visible", bsonPropKey, "Severity"), + hiddenRelationRow("rel-hidden", bsonTwinKey, "Severity"), + ) + + // when + key, ok := r.PropertyKey("Severity") + + // then + require.True(t, ok, "the visible holder owns the name alone") + assert.Equal(t, bsonPropKey, key) + assert.Equal(t, "Severity", r.PropertySlug(bsonPropKey)) + }) + + t.Run("a hidden entity keeps its stored key as an address", func(t *testing.T) { + // chain step 2 is not a namespace question: the stored key is always + // an address, and the emit side must still refuse to spell someone + // else's entity with it + r := vocabFixture(t, + hiddenRelationRow("rel-hidden", "hidden_key", "Hidden"), + named(relationRow("rel-other", bsonPropKey, ""), "hidden_key"), + ) + + key, ok := r.PropertyKey("hidden_key") + assert.False(t, ok, "a stored key, verbatim — never the name layer") + assert.Equal(t, "hidden_key", key) + assert.NotEqual(t, "hidden_key", r.PropertySlug(bsonPropKey), + "and the other holder does not emit a spelling the hidden stored key answers to") + }) + + t.Run("a custom relation sharing a bundled NAME does not capture it", func(t *testing.T) { + // the space holds only the custom relation named "Priority" — a real + // corpus population, 25 custom names equal a bundled relation name. + // Both spell it; the accept side refuses to pick, and the importer's + // type scope or the document's legend is what decides. + r := vocabFixture(t, relationRow("rel-custom", bsonPropKey, "Priority")) + + _, ok := r.PropertyKey("Priority") + assert.False(t, ok, "shared between the space and the bundled table: never a guess") + assert.Equal(t, []string{bsonPropKey, "priority"}, r.PropertyKeyCandidates("Priority"), + "the bundled binding is among the candidates") + assert.Equal(t, "Priority", r.PropertySlug(bsonPropKey), + "the custom holder still spells its name — collisions are per document") + assert.Equal(t, "Priority", r.PropertySlug("priority"), + "and so does the bundled key: the code table is its authority") + }) + + t.Run("the type namespace follows the same hidden rule", func(t *testing.T) { + // given + hidden := typeRow("type-hidden", "6a7663db61fab21cd4b9e107", "Invoice") + hidden[bundle.RelationKeyIsHidden] = domain.Bool(true) + r := vocabFixture(t, typeRow("type-visible", bsonTypeKey, "Invoice"), hidden) + + // when + key, ok := r.TypeKey("Invoice") + + // then + require.True(t, ok) + assert.Equal(t, bsonTypeKey, key) + assert.Equal(t, "Invoice", r.TypeSlug(bsonTypeKey)) + }) +} + +// TestAcceptHalfFolds is chain step 4 on the accept side: the forgiving +// layer, answering only when exactly one candidate remains. The extended +// fold drops case, `_`, `-`, spaces and invisible code points, so the +// legacy name-derived labels (`publish_date` for "Publish Date") land in +// their name's own class and documents already written keep resolving. +func TestAcceptHalfFolds(t *testing.T) { + sev := func(t *testing.T) *Resolvers { + return vocabFixture(t, relationRow("rel-sev", bsonPropKey, "Severity")) + } + + t.Run("case, separator and legacy-label variants fold to the name", func(t *testing.T) { + for _, input := range []string{"severity", "SEVERITY", "sever_ity", "sever-ity", " Severity "} { + key, ok := sev(t).PropertyKey(input) + require.True(t, ok, input) + assert.Equal(t, bsonPropKey, key, input) + } + }) + + t.Run("an exact stored key still wins the fold", func(t *testing.T) { + // given — `severity` is one relation's stored KEY and folds onto + // another's name + r := vocabFixture(t, + relationRow("rel-sev", bsonPropKey, "Severity"), + relationRow("rel-legacy", "severity", "Old severity"), + ) + + // when / then — step 2, exact, before any folding + key, ok := r.PropertyKey("severity") + assert.False(t, ok) + assert.Equal(t, "severity", key, "an exact stored key is never folded away") + }) + + t.Run("an ambiguous fold degrades verbatim, never guesses", func(t *testing.T) { + // given — two live relations whose names fold together + r := vocabFixture(t, + relationRow("rel-a", bsonPropKey, "Mood level"), + relationRow("rel-b", bsonTwinKey, "Moodlevel"), + ) + + // when + key, ok := r.PropertyKey("MOOD_LEVEL") + + // then + assert.False(t, ok) + assert.Equal(t, "MOOD_LEVEL", key, "the term passes through — never a guess") + }) + + t.Run("a hidden holder does not answer the fold", func(t *testing.T) { + r := vocabFixture(t, hiddenRelationRow("rel-hidden", bsonPropKey, "Severity")) + + key, ok := r.PropertyKey("severity") + assert.False(t, ok) + assert.Equal(t, "severity", key) + }) + + t.Run("the bundled fold table is consulted too", func(t *testing.T) { + r := vocabFixture(t) + + // the pre-change derived slug: fold(ToSnake(key)) == fold(key), so + // it lands in the stored key's class with no compatibility table + key, ok := r.PropertyKey("due_date") + require.True(t, ok) + assert.Equal(t, "dueDate", key) + }) + + t.Run("a space claimant makes a bundled fold class ambiguous", func(t *testing.T) { + // given — a custom relation named so that it folds together with a + // bundled key's class: the layer must see BOTH candidates and refuse + r := vocabFixture(t, relationRow("rel-b", bsonPropKey, "Due-Date")) + + key, ok := r.PropertyKey("due_date") + assert.False(t, ok) + assert.Equal(t, "due_date", key, "two candidates in one class: never a guess") + }) + + t.Run("a term nothing folds to passes through", func(t *testing.T) { + key, ok := vocabFixture(t).PropertyKey("no_such_thing") + assert.False(t, ok) + assert.Equal(t, "no_such_thing", key, "a miss must return the term, not an empty key") + }) + + t.Run("the type namespace folds the same way", func(t *testing.T) { + r := vocabFixture(t, typeRow("type-inv", bsonTypeKey, "Invoice")) + + key, ok := r.TypeKey("invoice") + require.True(t, ok) + assert.Equal(t, bsonTypeKey, key) + }) +} + +// TestCorpseNameLifecycle — the corpse (uninstalled) story, both store +// shapes a real UI delete can leave (flag-only {isUninstalled}, and the +// prod double-flag {isUninstalled, isDeleted}). +func TestCorpseNameLifecycle(t *testing.T) { + corpse := func(prodShape bool) spaceindex.TestObject { + row := relationRow("rel-corpse", bsonPropKey, "Warranty until") + row[bundle.RelationKeyIsUninstalled] = domain.Bool(true) + if prodShape { + row[bundle.RelationKeyIsDeleted] = domain.Bool(true) + } + return row + } + + for _, shape := range []struct { + name string + prod bool + }{{"flag-only shape", false}, {"prod shape", true}} { + t.Run(shape.name+": the name is severed in both directions", func(t *testing.T) { + // given — the corpse is the ONLY holder of "Warranty until" + r := vocabFixture(t, corpse(shape.prod)) + + // then: emit degrades to the stored key… + assert.Equal(t, bsonPropKey, r.PropertySlug(bsonPropKey)) + // …and the name no longer resolves — a document written BEFORE + // the uninstall keeps its term verbatim on import, landing the + // value under a string key no relation owns (executed below) + key, ok := r.PropertyKey("Warranty until") + assert.False(t, ok) + assert.Equal(t, "Warranty until", key) + }) + } + + t.Run("a pre-uninstall document imports its value onto a key no relation owns", func(t *testing.T) { + // given — the export produced while the property was live spells the + // name; the property has since been UI-deleted. (A REAL export also + // carries the legend line that would resolve this — the fixture + // drops it to show the legendless degradation.) + index := spaceindex.NewStoreFixture(t) + index.AddObjects(t, []spaceindex.TestObject{corpse(true)}) + doc := []byte(`{"version":2,"id":"obj1","type":"Page","properties":{"Name":"Doc","Warranty until":"2027-01-01"}}`) + + // when + _, snapshot, err := anyblockjson.Unmarshal(doc, New(index).Options()) + + // then — the term passes through verbatim, never a guess: the value + // lands under the literal name string, NOT under the corpse's stored + // key. Byte-safe, address-orphaned. + require.NoError(t, err) + assert.Equal(t, "2027-01-01", snapshot.Details.Fields["Warranty until"].GetStringValue()) + assert.Nil(t, snapshot.Details.Fields[bsonPropKey]) + }) + + t.Run("after a recreate the corpse-era name re-aims onto the new holder", func(t *testing.T) { + // given — P held "Warranty until" and was uninstalled; Q was minted + // fresh under the same name + recreated := relationRow("rel-recreated", bsonTwinKey, "Warranty until") + r := vocabFixture(t, corpse(true), recreated) + + // when + key, ok := r.PropertyKey("Warranty until") + + // then — same bytes, different property: a legendless document + // exported while P was live now binds Q. This is §6's freed-name + // hazard, pinned as current documented behavior; an EXPORTED + // document is protected by its legend line. + require.True(t, ok) + assert.Equal(t, bsonTwinKey, key) + assert.Equal(t, "Warranty until", r.PropertySlug(bsonTwinKey), "the new holder owns the spelling") + }) +} + +// TestCorpseStoredKeyStillNamesItsObjects is the corpse story from the OTHER +// side, and the one that loses data: not the corpse's name, but the corpse's +// STORED KEY, which every object it ever typed or tagged still carries. +// +// The delete vacates the name namespace, so `initiative` stops being a live +// stored key — while a live entity is NAMED "initiative", which is how a +// user frees a name and reuses it. The vocabulary emits the corpse's stored +// key verbatim (nothing else is an address) and grants the live holder its +// name only where no live stored key owns the string; the DOCUMENT is what +// says which of the two it means, and the identity entry is that statement. +// +// Both arms run the real exporter over the real resolver. +func TestCorpseStoredKeyStillNamesItsObjects(t *testing.T) { + t.Run("a type whose key the space no longer reserves", func(t *testing.T) { + // given + dead := typeRow("t-dead", "initiative", "Initiative") + dead[bundle.RelationKeyIsUninstalled] = domain.Bool(true) + r := vocabFixture(t, dead, typeRow("t-live", bsonTypeKey, "initiative")) + require.Equal(t, "initiative", r.TypeSlug("initiative"), + "the corpse's stored key is its own address — there is no name to spell it with") + key, ok := r.TypeKey("initiative") + require.True(t, ok) + require.Equal(t, bsonTypeKey, key, + "the corpse vacated the string, and the live holder is NAMED it — the fixture is the collision") + + snapshot := &model.SmartBlockSnapshotBase{ + Details: &types.Struct{Fields: map[string]*types.Value{"id": strValue("obj1")}}, + ObjectTypes: []string{"ot-initiative"}, + } + + // when + data, err := anyblockjson.Marshal(model.SmartBlockType_Page, snapshot, r.Options()) + require.NoError(t, err) + + // then + require.NoError(t, anyblockjson.Validate(data)) + var doc struct { + Type string `json:"type"` + TypeKeys map[string]string `json:"type_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "initiative", doc.Type) + assert.Equal(t, map[string]string{"initiative": "initiative"}, doc.TypeKeys, + "the document says the term is a stored key, or a reader whose space "+ + "binds the name takes it for the live holder") + + _, back, err := anyblockjson.Unmarshal(data, r.Options()) + require.NoError(t, err) + assert.Equal(t, []string{"ot-initiative"}, back.ObjectTypes) + }) + + t.Run("a property whose key the space no longer reserves", func(t *testing.T) { + // given — the live holder is NAMED "initiative", the exact string + // the corpse is stored under, so the ladder gives it the suffixed + // spelling: the plain string could never resolve to it anywhere + dead := relationRow("rel-dead", "initiative", "Initiative") + dead[bundle.RelationKeyIsUninstalled] = domain.Bool(true) + r := vocabFixture(t, dead, named(relationRow("rel-live", bsonPropKey, ""), "initiative")) + + snapshot := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{ + "id": strValue("obj1"), + "initiative": strValue("value of the deleted property"), + bsonPropKey: strValue("value of the live one"), + }}} + + // when + data, err := anyblockjson.Marshal(model.SmartBlockType_Page, snapshot, r.Options()) + require.NoError(t, err) + + // then + require.NoError(t, anyblockjson.Validate(data)) + var doc struct { + Properties map[string]string `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "value of the deleted property", doc.Properties["initiative"]) + assert.Equal(t, "value of the live one", doc.Properties["initiative (b9e101)"], + "the live holder's plain name is a stored key on this space, so it "+ + "takes the suffixed form — visibly synthetic, immutable while the key lives") + assert.Equal(t, map[string]string{ + "initiative": "initiative", + "initiative (b9e101)": bsonPropKey, + }, doc.PropertyKeys, + "the identity entry for the stored key, and the suffix's inverse — no "+ + "shipped table binds either term") + + // and both values come home + _, back, err := anyblockjson.Unmarshal(data, r.Options()) + require.NoError(t, err) + assert.Equal(t, "value of the deleted property", + back.Details.Fields["initiative"].GetStringValue()) + assert.Equal(t, "value of the live one", + back.Details.Fields[bsonPropKey].GetStringValue()) + }) +} + +func strValue(s string) *types.Value { + return &types.Value{Kind: &types.Value_StringValue{StringValue: s}} +} + +// The dead machinery, pinned dead: `apiObjectKey` is never read by the +// format. A stored slug neither spells its holder nor binds on the accept +// side — the name is the whole of the spelling rule, and the API surface's +// key convention is a separate decision. +func TestApiObjectKeyLeavesTheFormat(t *testing.T) { + row := relationRow("rel-x", bsonPropKey, "Publish Date") + row[bundle.RelationKeyApiObjectKey] = domain.String("publish_date_custom") + r := vocabFixture(t, row) + + assert.Equal(t, "Publish Date", r.PropertySlug(bsonPropKey), + "the name, not the stored slug") + _, ok := r.PropertyKey("publish_date_custom") + assert.False(t, ok, "the stored slug binds nothing") + // the fold still forgives what the NAME's own class covers — which is + // what keeps most legacy custom spellings resolving without the slug: + // the old label was derived from this very name + key, ok := r.PropertyKey("publish_date") + require.True(t, ok) + assert.Equal(t, bsonPropKey, key) +} + +// ScopedKeyVocabulary — the capability the importer resolves shared names +// through, and diagnoses verbatim terms with. +func TestScopedKeyVocabulary(t *testing.T) { + t.Run("TypePropertyKeys reads the space's own type row", func(t *testing.T) { + task := typeRow("type-task", bsonTypeKey, "Task") + task[bundle.RelationKeyRecommendedRelations] = domain.StringList([]string{"objidA", "objidGone"}) + task[bundle.RelationKeyRecommendedFeaturedRelations] = domain.StringList([]string{"objidB"}) + r := vocabFixture(t, task, + relationRow("objidA", bsonPropKey, "Projects"), + relationRow("objidB", bsonTwinKey, "Assignee 2"), + ) + + keys := r.TypePropertyKeys(bsonTypeKey) + assert.ElementsMatch(t, []string{bsonPropKey, bsonTwinKey}, keys, + "the four recommended lists, translated id→key; an id the space cannot name is dropped — "+ + "a narrower scope only degrades toward the loud error, never toward a wrong resolution") + }) + + t.Run("an uninstalled bundled type falls back to the shipped table", func(t *testing.T) { + r := vocabFixture(t) + + keys := r.TypePropertyKeys("task") + assert.Contains(t, keys, "assignee", "the bundled Task type declares its own properties") + }) + + t.Run("facts: a live stored key, and a glued name", func(t *testing.T) { + r := vocabFixture(t, relationRow("rel-a", bsonPropKey, "Lists [in work]")) + + assert.True(t, r.PropertyTermFacts(bsonPropKey).LiveStoredKey) + facts := r.PropertyTermFacts("Lists [in work] (text)") + assert.False(t, facts.LiveStoredKey) + assert.Equal(t, "Lists [in work]", facts.ExtendsName, + "the eval's one real raw-name failure shape: an annotation glued onto a copied name") + assert.Equal(t, "", r.PropertyTermFacts("Lists [in workshop]").ExtendsName, + "a longer name sharing a prefix is not glue — the boundary rule") + }) +} + +// The §3 label rule inside the space vocabulary, name-era: the rule itself +// is unit-tested in the parent package (label_test.go); what is pinned here +// is the part only this package can get wrong — which stored fact reaches +// it, and what happens when a spelling is unusable. +func TestNameLabels(t *testing.T) { + t.Run("a non-Latin name is the spelling, verbatim", func(t *testing.T) { + r := vocabFixture(t, + named(relationRow("rel-due", "dueDate", ""), "Срок"), + named(relationRow("rel-custom", bsonPropKey, ""), "Срок"), + ) + + // the space's local copy of a bundled key may carry any name; the + // code table is the bundled key's authority in every space + assert.Equal(t, "Due date", r.PropertySlug("dueDate"), + "a renamed local copy does not move a spelling that ships with every reader") + assert.Equal(t, "Срок", r.PropertySlug(bsonPropKey)) + key, ok := r.PropertyKey("Срок") + require.True(t, ok) + assert.Equal(t, bsonPropKey, key) + }) + + t.Run("a hidden relation spells nothing", func(t *testing.T) { + r := vocabFixture(t, named(hiddenRelationRow("rel-secret", bsonPropKey, ""), "Secret")) + + assert.Equal(t, bsonPropKey, r.PropertySlug(bsonPropKey)) + _, ok := r.PropertyKey("Secret") + assert.False(t, ok) + }) + + t.Run("a name the format refuses as a property spelling is never granted", func(t *testing.T) { + // §2 refuses `id` and `type` as property spellings before any + // resolution, so a label export would throw away is not built here + r := vocabFixture(t, named(relationRow("rel-id", bsonPropKey, ""), "id")) + + assert.Equal(t, bsonPropKey, r.PropertySlug(bsonPropKey)) + }) +} + +// A UI-deleted entity vacates the NAME namespace and still answers to its +// ID. The two are different questions and the listing serves both: +// +// - "who owns the spelling `Project`?" — a corpse must not, or a type +// minted under the freed name is shadowed by something no listing shows. +// - "what does this id point at?" — naming a corpse claims nothing, and +// the store never removes the type, so the answer exists. +func TestKeyVocab_ADeletedTypeIsNamedByIdButOwnsNoName(t *testing.T) { + corpse := typeRow("type-corpse", "retired_project", "Project") + corpse[bundle.RelationKeyIsUninstalled] = domain.Bool(true) + + t.Run("its id resolves to its key", func(t *testing.T) { + r := vocabFixture(t, corpse) + key, ok := r.TypeKeyById("type-corpse") + require.True(t, ok, "the store still holds the type; naming it claims nothing") + assert.Equal(t, "retired_project", key) + }) + + t.Run("but it is unreachable by spelling", func(t *testing.T) { + r := vocabFixture(t, corpse) + + key, ok := r.TypeKey("Project") + assert.Falsef(t, ok && key == "retired_project", + "the corpse claimed the spelling it vacated (got %q, ok=%v)", key, ok) + + byId, ok := r.TypeKeyById("type-corpse") + require.True(t, ok) + assert.Equal(t, "retired_project", byId, "while still being nameable by id") + }) +} + +// One entity arriving on TWO rows — the shape the listing does not forbid +// and the rest of this file already guards against with first-wins (keyById, +// idByKey, and relKeyToId one file over; GetRelationByKey answers a +// duplicated relationKey with records[0]). Every SET the vocabulary +// publishes has to survive it, and the candidate list most of all: that list +// is the importer's ambiguity signal — two entries mean two live entities +// and the import stops to ask for a legend — so one entity listed twice +// would refuse a document this very exporter had just written, from a pure +// bookkeeping slip rather than from anything the space actually holds. +func TestOneEntityOnTwoRowsIsStillOneCandidate(t *testing.T) { + t.Run("a property key carried by two rows is one candidate", func(t *testing.T) { + // given — a legacy row and the derived one, both live, both carrying + // the same relationKey and the same name (the relation namespace + // reads its key off the `relationKey` detail, not off the id, so + // nothing about the row identity keeps the two apart) + r := vocabFixture(t, + relationRow("legacy-row", bsonPropKey, "Manual property"), + relationRow("rel-"+bsonPropKey, bsonPropKey, "Manual property"), + ) + want := []string{bsonPropKey} + + // when + got := r.PropertyKeyCandidates("Manual property") + + // then + assert.Equal(t, want, got, "one entity, one candidate") + key, ok := r.PropertyKey("Manual property") + require.True(t, ok, "one entity is not an ambiguity — the name still resolves") + assert.Equal(t, bsonPropKey, key) + }) + + t.Run("a type key carried by two rows is one candidate", func(t *testing.T) { + // given + r := vocabFixture(t, + typeRow("legacy-type-row", bsonTypeKey, "Sprint"), + typeRow("ot-"+bsonTypeKey, bsonTypeKey, "Sprint"), + ) + want := []string{bsonTypeKey} + + // when + got := r.TypeKeyCandidates("Sprint") + + // then + assert.Equal(t, want, got) + key, ok := r.TypeKey("Sprint") + require.True(t, ok, "the type namespace has no wider scope to recover in — it must not be lost here") + assert.Equal(t, bsonTypeKey, key) + }) + + t.Run("a property named by two of the type's four lists is one scope entry", func(t *testing.T) { + // given — nothing declares the four recommended lists disjoint, and + // the scope is COUNTED by the importer: a property listed as both + // featured and ordinary would make its own type unable to single it + // out, which is the opposite of what the scope exists for + task := typeRow("type-task", bsonTypeKey, "Task") + task[bundle.RelationKeyRecommendedRelations] = domain.StringList([]string{"objidA"}) + task[bundle.RelationKeyRecommendedFeaturedRelations] = domain.StringList([]string{"objidA"}) + r := vocabFixture(t, task, relationRow("objidA", bsonPropKey, "Projects")) + want := []string{bsonPropKey} + + // when + got := r.TypePropertyKeys(bsonTypeKey) + + // then + assert.Equal(t, want, got, "the type declares one property, however many of its lists name it") + }) +} diff --git a/pkg/lib/anyblockjson/storeresolver/objectexists_test.go b/pkg/lib/anyblockjson/storeresolver/objectexists_test.go new file mode 100644 index 0000000000..1e084449ae --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/objectexists_test.go @@ -0,0 +1,75 @@ +package storeresolver + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" +) + +// ObjectExists is the existence half of the object-namespace seam (§9): it +// answers from the same cached point lookup ObjectName pays for, and it is +// the answer ObjectName structurally CANNOT give — that seam's ok is +// `name != ""`, so a present-and-unnamed object answers "no" there. The +// missing-reference rule rewrites and drops on this answer, so the two +// negative shapes below are the ones that protect live data. +// +// How these can fail: read existence off ObjectName's ok and the unnamed +// case fails — the exact conflation that would rewrite live references to +// `_missing_object`; key the lookup on anything but the id, or read a +// partial row as absence, and the named case fails. +func TestObjectExists(t *testing.T) { + t.Run("a present, named object exists", func(t *testing.T) { + // given + r := objectNameFixture(t) + + // when + exists, known := r.ObjectExists("bafyreinamedpage") + + // then + require.True(t, known) + assert.True(t, exists) + }) + + t.Run("a present object with NO name still exists", func(t *testing.T) { + // given — the trap: ObjectName answers ("", false) for this id + r := objectNameFixture(t) + + // when + exists, known := r.ObjectExists("bafyreinameless") + _, named := r.ObjectName("bafyreinameless") + + // then + require.True(t, known) + assert.True(t, exists, "untitled is not missing") + assert.False(t, named, "and namedness stays a separate question") + }) + + t.Run("an id the index has no row for does not exist", func(t *testing.T) { + // given + r := objectNameFixture(t) + + // when + exists, known := r.ObjectExists("bafyreineverseen") + + // then + require.True(t, known) + assert.False(t, exists) + }) + + t.Run("the capability is discoverable off Options", func(t *testing.T) { + // given — the codec finds it by type assertion on + // Options.ResolveObjectNames (the TypeResolver pattern), so the + // standard wiring arms the missing-reference rule with no extra step + r := objectNameFixture(t) + + // when + opts := r.Options() + _, ok := opts.ResolveObjectNames.(anyblockjson.ObjectExistenceResolver) + + // then + assert.True(t, ok) + }) +} diff --git a/pkg/lib/anyblockjson/storeresolver/objectname_test.go b/pkg/lib/anyblockjson/storeresolver/objectname_test.go new file mode 100644 index 0000000000..29ba38c74c --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/objectname_test.go @@ -0,0 +1,101 @@ +package storeresolver + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// objectNameFixture builds a resolver over a space holding one named page +// and one object that never got a name. +func objectNameFixture(t *testing.T) *Resolvers { + index := spaceindex.NewStoreFixture(t) + index.AddObjects(t, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String("bafyreinamedpage"), + bundle.RelationKeyName: domain.String("Local-first UX"), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_basic)), + }, + { + bundle.RelationKeyId: domain.String("bafyreinameless"), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_basic)), + }, + }) + return New(index) +} + +// ObjectName is the object-namespace seam the `#name` reference suffix reads +// from (§9). The two negative answers matter as much as the positive one: +// export writes a suffix only when this says yes, so a "yes, empty string" +// would put a dangling `#` on every reference to an unnamed object. +// +// How these can fail: a lookup keyed on anything but the object id misses +// the named row and the first case fails; reading any field but `name` fails +// it too; returning true for an empty name fails the second and third. +func TestObjectName(t *testing.T) { + t.Run("a named object resolves to its display name", func(t *testing.T) { + // given + r := objectNameFixture(t) + + // when + name, ok := r.ObjectName("bafyreinamedpage") + + // then + require.True(t, ok) + assert.Equal(t, "Local-first UX", name) + }) + + t.Run("an object with no name answers no", func(t *testing.T) { + // given + r := objectNameFixture(t) + + // when + name, ok := r.ObjectName("bafyreinameless") + + // then + assert.False(t, ok, "no name is an answer of no, never a blank suffix") + assert.Empty(t, name) + }) + + t.Run("an id this space has no row for answers no", func(t *testing.T) { + // given + r := objectNameFixture(t) + + // when + name, ok := r.ObjectName("bafyreiunknown") + + // then + assert.False(t, ok) + assert.Empty(t, name) + }) +} + +// Options() is the one line every wiring copies, so what it pre-wires is +// what every export/import actually runs with. The object-name seam and the +// space id ride it like the four resolvers before them: forget either and +// the suffix never fires (silently) or the participant fold never fires +// (silently), with nothing else failing. +// +// How this can fail: drop `ResolveObjectNames: r` or `SpaceId: +// r.index.SpaceId()` from Options() and the matching assertion fails. +func TestOptions_WiresObjectNamesAndSpaceId(t *testing.T) { + // given + r := objectNameFixture(t) + + // when + opts := r.Options() + + // then + require.NotNil(t, opts.ResolveObjectNames, "the suffix seam is pre-wired") + name, ok := opts.ResolveObjectNames.ObjectName("bafyreinamedpage") + require.True(t, ok) + assert.Equal(t, "Local-first UX", name) + assert.Equal(t, "test", opts.SpaceId, + "the index's own space id rides along — the participant fold needs it (§9)") +} diff --git a/pkg/lib/anyblockjson/storeresolver/objecttypes_test.go b/pkg/lib/anyblockjson/storeresolver/objecttypes_test.go new file mode 100644 index 0000000000..add1c00c4e --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/objecttypes_test.go @@ -0,0 +1,156 @@ +package storeresolver + +// PropertyDefinition.ObjectTypes — `type_settings.property_definitions[].object_types`, SPEC §2a +// — had no node-backed emitter at all: the resolver left the field empty, so a +// node export of a type document silently dropped every property's target +// types and the property came back accepting any object. These tests drive the +// real exporter, not just the resolver method, because the resolver hook that +// feeds that slot is PropertyById (via the recommended-relation lists) and a +// test calling anything else would never reach it. + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// customTypeKey is a space-minted (bson) type key, the shape a real space +// gives a user-created type. Its display name is what the document must spell. +const customTypeKey = "69bbfc78877a91b1d12d1a7c" + +// newTargetsFixture is a space holding one `objects` relation that targets two +// types by OBJECT ID — the form the store actually keeps +// (objectcreator.fillRelationFormatObjectTypes rewrites bundled urls to derived +// ids at creation) — plus one target the store cannot resolve at all. +func newTargetsFixture(t *testing.T) *fixture { + index := spaceindex.NewStoreFixture(t) + index.AddObjects(t, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String("rel-assignee"), + bundle.RelationKeyRelationKey: domain.String("assignee"), + bundle.RelationKeyName: domain.String("Assignee"), + bundle.RelationKeyRelationFormat: domain.Int64(int64(model.RelationFormat_object)), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeyRelationFormatObjectTypes: domain.StringList([]string{"type-person", "type-participant", "type-vanished"}), + }, + { + bundle.RelationKeyId: domain.String("type-person"), + bundle.RelationKeyUniqueKey: domain.String(domain.TypeKey(customTypeKey).URL()), + bundle.RelationKeyApiObjectKey: domain.String("person"), + bundle.RelationKeyName: domain.String("Person"), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_objectType)), + }, + { + // a bundled type installed into the space: the store row carries + // the same shape, and hidden entities still have an identity + bundle.RelationKeyId: domain.String("type-participant"), + bundle.RelationKeyUniqueKey: domain.String(bundle.TypeKeyParticipant.URL()), + bundle.RelationKeyName: domain.String("Participant"), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_objectType)), + }, + }) + return &fixture{Resolvers: New(index), index: index} +} + +func TestPropertyDefinitionObjectTypes(t *testing.T) { + t.Run("stored target ids resolve to stored type keys", func(t *testing.T) { + // given + fx := newTargetsFixture(t) + + // when + def, ok := fx.PropertyById("rel-assignee") + + // then: ids in, KEYS out, in the stored order — and the id nothing in + // the space answers for is dropped rather than exported as a target + // no reader could find + require.True(t, ok) + assert.Equal(t, []string{customTypeKey, "participant"}, def.ObjectTypes) + }) + + t.Run("a bundled url target resolves without a store row", func(t *testing.T) { + // given: legacy rows that predate fillRelationFormatObjectTypes keep + // the bundled url form + index := spaceindex.NewStoreFixture(t) + index.AddObjects(t, []spaceindex.TestObject{{ + bundle.RelationKeyId: domain.String("rel-assignee"), + bundle.RelationKeyRelationKey: domain.String("assignee"), + bundle.RelationKeyRelationFormat: domain.Int64(int64(model.RelationFormat_object)), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeyRelationFormatObjectTypes: domain.StringList([]string{bundle.TypeKeyTask.BundledURL()}), + }}) + fx := &fixture{Resolvers: New(index), index: index} + + // when + def, ok := fx.PropertyById("rel-assignee") + + // then + require.True(t, ok) + assert.Equal(t, []string{"task"}, def.ObjectTypes) + }) + + t.Run("a property with no targets stays untargeted", func(t *testing.T) { + // given: the "empty means any object" case must not become [""] + fx := newFixture(t) + + // when + def, ok := fx.PropertyById("rel-priority") + + // then + require.True(t, ok) + assert.Empty(t, def.ObjectTypes) + }) +} + +// The end-to-end guard: run the real exporter over a type snapshot and read +// the document. This is the only assertion that proves the resolver hook the +// exporter actually calls is the one that was fixed — the emitter reads +// ObjectTypes off whatever PropertyById returns for a recommended-relation id. +func TestTypeDocumentCarriesObjectTypes(t *testing.T) { + // given + fx := newTargetsFixture(t) + snapshot := &model.SmartBlockSnapshotBase{ + Key: "task", + Details: &types.Struct{Fields: map[string]*types.Value{ + "recommendedFeaturedRelations": {Kind: &types.Value_ListValue{ListValue: &types.ListValue{ + Values: []*types.Value{{Kind: &types.Value_StringValue{StringValue: "rel-assignee"}}}, + }}}, + }}, + } + + // when + data, err := anyblockjson.Marshal(model.SmartBlockType_STType, snapshot, fx.Options()) + require.NoError(t, err) + + // then + var doc struct { + TypeKeys map[string]string `json:"type_internal_keys"` + TypeSettings struct { + PropertyDefinitions []struct { + Key string `json:"property"` + ObjectTypes []string `json:"object_types"` + } `json:"property_definitions"` + } `json:"type_settings"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + require.Len(t, doc.TypeSettings.PropertyDefinitions, 1) + assert.Equal(t, "Assignee", doc.TypeSettings.PropertyDefinitions[0].Key) + // the slots are spelled as display names and the legend inverts the one + // the bundled table cannot (§3) — the whole point of carrying the + // targets is that a reader can bind them back + assert.Equal(t, []string{"Person", "Space member"}, doc.TypeSettings.PropertyDefinitions[0].ObjectTypes) + assert.Equal(t, map[string]string{"Person": customTypeKey}, doc.TypeKeys) + + // and: the document reads back onto the very same stored keys + _, back, err := anyblockjson.Unmarshal(data, anyblockjson.Options{}) + require.NoError(t, err) + assert.NotNil(t, back) +} diff --git a/pkg/lib/anyblockjson/storeresolver/participant_test.go b/pkg/lib/anyblockjson/storeresolver/participant_test.go new file mode 100644 index 0000000000..e79b3afd0e --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/participant_test.go @@ -0,0 +1,120 @@ +package storeresolver + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The participant ids are the real shape: `_participant__`, +// 135 characters, which is what makes naming the member worth doing at all. +const ( + namedParticipant = "_participant_bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq_30afw2fe3tvff_AASdKiEGfcyhxX3ufr4auHRviACUXxkF68uZwtSb2AnyRoMA" + unnamedParticipant = "_participant_bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq_30afw2fe3tvff_AAJmVGvhCctWjPUeQxpZjbQCbLcVeCTHM7LmJRCzEnHwqXBt" + absentParticipant = "_participant_bafyreid62d5e6hny6mv6zass2zg73nxyhjzhjasx7imvzxvqz6rcnjqcgq_30afw2fe3tvff_AAHTtt8gtEhk9vPBFdrpxNXFrYaZQBS4rjhBLTGRRfDrDwLA" +) + +// participantFixture builds a resolver over a space holding two members: one +// with a profile name, one who never set one. +func participantFixture(t *testing.T) *Resolvers { + index := spaceindex.NewStoreFixture(t) + index.AddObjects(t, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String(namedParticipant), + bundle.RelationKeyName: domain.String("Roman"), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_participant)), + }, + { + bundle.RelationKeyId: domain.String(unnamedParticipant), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_participant)), + }, + }) + return New(index) +} + +// ParticipantName is what turns a 135-character id into the one thing a +// reader of the document wants from it. Three answers, and the two negative +// ones matter as much as the positive: export writes the property only when +// this says yes, so a "yes, empty string" would put a blank `creator` on +// every object whose author never named themselves. +// +// How these can fail: a lookup keyed on anything but the object id (a unique +// key, a relation key) misses the named row and the first case fails; reading +// any field but `name` — `identity`, `globalName`, the id itself — fails it +// too; and returning true for an empty name fails the second and third. +func TestParticipantName(t *testing.T) { + t.Run("a named member resolves to the name", func(t *testing.T) { + // given + r := participantFixture(t) + + // when + name, ok := r.ParticipantName(namedParticipant) + + // then + require.True(t, ok) + assert.Equal(t, "Roman", name) + }) + + t.Run("a member with no profile name answers no", func(t *testing.T) { + // given + r := participantFixture(t) + + // when + name, ok := r.ParticipantName(unnamedParticipant) + + // then + assert.False(t, ok, "an unnamed member must not export as a blank name") + assert.Empty(t, name) + }) + + t.Run("an id this space has no participant for answers no", func(t *testing.T) { + // given + r := participantFixture(t) + + // when + name, ok := r.ParticipantName(absentParticipant) + + // then + assert.False(t, ok) + assert.Empty(t, name, "the id must never be its own answer (§3)") + }) + + t.Run("the answer is cached, misses included", func(t *testing.T) { + // given + r := participantFixture(t) + + // when + first, _ := r.ParticipantName(namedParticipant) + _, missOk := r.ParticipantName(absentParticipant) + second, secondOk := r.ParticipantName(namedParticipant) + + // then + assert.Equal(t, first, second) + assert.True(t, secondOk) + assert.False(t, missOk) + assert.Len(t, r.participantNames, 2, "one entry per id asked about, hit or miss") + }) +} + +// The wiring: a node-backed export gets the participant resolver without the +// caller naming it, exactly as it gets the option and property resolvers. +// Without this the seam exists and nothing in the product reaches it. +func TestOptions_CarriesParticipantResolver(t *testing.T) { + // given + r := participantFixture(t) + + // when + opts := r.Options() + + // then + require.NotNil(t, opts.ResolveParticipants) + name, ok := opts.ResolveParticipants.ParticipantName(namedParticipant) + require.True(t, ok) + assert.Equal(t, "Roman", name) +} diff --git a/pkg/lib/anyblockjson/storeresolver/relationformat_test.go b/pkg/lib/anyblockjson/storeresolver/relationformat_test.go new file mode 100644 index 0000000000..45de379f92 --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/relationformat_test.go @@ -0,0 +1,154 @@ +package storeresolver + +// relationformat_test.go — the anyblockjson.TypeResolver capability (SPEC +// §2d): the id↔key translation behind a relation document's `object_types` +// envelope field. The codec discovers it by type assertion on +// Options.ResolveProperties, so these tests drive both the methods and the +// real codec through fx.Options(). + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// TypeKeyById answers for a space row and for a legacy bundled url, and +// misses honestly for an id nothing serves; TypeIdByKey inverts the space +// half. Both must miss rather than invent, because the codec's degradation +// for an unanswered entry is verbatim pass-through — an invented answer +// would translate one direction with nothing to invert it on the other. +// +// How this can fail: fill idByKey from a different listing than keyById and +// the two directions stop being inverses; drop the bundled-url arm and a +// legacy `_ot…` entry stops resolving. +func TestTypeResolver_TranslatesIdsAndKeys(t *testing.T) { + // given + fx := newTargetsFixture(t) + + // then: the space row, both directions + key, ok := fx.TypeKeyById("type-person") + require.True(t, ok) + assert.Equal(t, customTypeKey, key) + id, ok := fx.TypeIdByKey(customTypeKey) + require.True(t, ok) + assert.Equal(t, "type-person", id) + + // the legacy bundled-url form, no store row needed + key, ok = fx.TypeKeyById(bundle.TypeKeyTask.BundledURL()) + require.True(t, ok) + assert.Equal(t, "task", key) + + // and honest misses + _, ok = fx.TypeKeyById("type-vanished") + assert.False(t, ok, "an id nothing serves must miss, not invent") + _, ok = fx.TypeIdByKey("vanishedKey") + assert.False(t, ok) +} + +// The end-to-end guard, the objecttypes_test shape: a relation object whose +// stored targets are ids exports `object_types` as type keys, and the import +// through the same resolver stores the same ids back — the §2d round trip is +// id-exact under the capability, which is what lets snapshotdiff run with no +// new rule. +// +// How this can fail: remove the TypeResolver methods and the assertion on +// the document shows raw ids; break TypeIdByKey and the re-imported detail +// holds keys where the store wants ids. +func TestRelationDocumentTranslatesTargetTypes(t *testing.T) { + // given a relation snapshot in the store's own shape: targets by id, + // plus one id the space no longer serves + fx := newTargetsFixture(t) + snapshot := &model.SmartBlockSnapshotBase{ + Key: "assignee", + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "rel-assignee"}}, + "relationFormat": {Kind: &types.Value_NumberValue{ + NumberValue: float64(model.RelationFormat_object)}}, + "relationFormatObjectTypes": {Kind: &types.Value_ListValue{ListValue: &types.ListValue{ + Values: []*types.Value{ + {Kind: &types.Value_StringValue{StringValue: "type-person"}}, + {Kind: &types.Value_StringValue{StringValue: "type-vanished"}}, + }, + }}}, + }}, + } + + // when + data, err := anyblockjson.Marshal(model.SmartBlockType_STRelation, snapshot, fx.Options()) + require.NoError(t, err) + _, got, err := anyblockjson.Unmarshal(data, fx.Options()) + require.NoError(t, err) + + // then: the resolved key on the wire in its §3 name spelling, with the + // legend entry that inverts it — and the unresolvable id verbatim, its + // own address, never dropped + var doc struct { + PropertySettings struct { + Format string `json:"format"` + ObjectTypes []string `json:"object_types"` + } `json:"property_settings"` + TypeKeys map[string]string `json:"type_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + assert.Equal(t, "objects", doc.PropertySettings.Format) + assert.Equal(t, []string{"Person", "type-vanished"}, doc.PropertySettings.ObjectTypes) + assert.Equal(t, customTypeKey, doc.TypeKeys["Person"], + "the name owes the legend entry that inverts it (§3)") + + // and ids back in the snapshot + gotTargets := got.Details.Fields["relationFormatObjectTypes"].GetListValue() + require.NotNil(t, gotTargets) + values := make([]string, 0, len(gotTargets.Values)) + for _, v := range gotTargets.Values { + values = append(values, v.GetStringValue()) + } + assert.Equal(t, []string{"type-person", "type-vanished"}, values) +} + +// A relation document from a space whose type listing is empty still +// round-trips: the capability answers nothing, entries pass through +// verbatim, and nothing errors. +// +// How this can fail: make TypeIdByKey (or TypeKeyById) panic or invent on an +// empty vocabulary, or make the codec require an answer. +func TestRelationDocumentPassThroughOnEmptySpace(t *testing.T) { + // given a space with no type rows at all + index := spaceindex.NewStoreFixture(t) + index.AddObjects(t, []spaceindex.TestObject{{ + bundle.RelationKeyId: domain.String("rel-assignee"), + bundle.RelationKeyRelationKey: domain.String("assignee"), + bundle.RelationKeyRelationFormat: domain.Int64(int64(model.RelationFormat_object)), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_relation)), + }}) + fx := &fixture{Resolvers: New(index), index: index} + snapshot := &model.SmartBlockSnapshotBase{ + Key: "assignee", + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "rel-assignee"}}, + "relationFormat": {Kind: &types.Value_NumberValue{ + NumberValue: float64(model.RelationFormat_object)}}, + "relationFormatObjectTypes": {Kind: &types.Value_ListValue{ListValue: &types.ListValue{ + Values: []*types.Value{{Kind: &types.Value_StringValue{StringValue: "bafyreisometype"}}}, + }}}, + }}, + } + + // when + data, err := anyblockjson.Marshal(model.SmartBlockType_STRelation, snapshot, fx.Options()) + require.NoError(t, err) + _, got, err := anyblockjson.Unmarshal(data, fx.Options()) + + // then + require.NoError(t, err) + assert.Equal(t, "bafyreisometype", + got.Details.Fields["relationFormatObjectTypes"].GetListValue().Values[0].GetStringValue()) +} diff --git a/pkg/lib/anyblockjson/storeresolver/storeresolver.go b/pkg/lib/anyblockjson/storeresolver/storeresolver.go new file mode 100644 index 0000000000..b2deafcb64 --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/storeresolver.go @@ -0,0 +1,365 @@ +// Package storeresolver provides objectstore-backed resolvers for the +// anyblockjson Options: property formats, select/multiSelect option names, +// and property definitions, all read from one space's index. It is the +// standard wiring for exports/imports that run inside a full node +// (API v2 reads, cmd/anyblockroundtrip). +package storeresolver + +import ( + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// Resolvers implements anyblockjson.OptionResolver and +// anyblockjson.PropertyResolver over a space index, with per-instance +// caching. Instances are not safe for concurrent use; create one per +// export/import operation. +type Resolvers struct { + index spaceindex.Store + optionsFor map[domain.RelationKey][]*model.RelationOption + + // participantNames caches ParticipantName's answers, misses included — + // an empty value IS the miss, so an unnamed or unknown participant is + // asked about once per export rather than once per object. + participantNames map[string]string + + // objectRows caches one point lookup per referenced object id, answering + // BOTH object-namespace questions from it: ObjectName's (the display + // name; miss = "") and ObjectExists's (whether the index holds a row at + // all). One cache rather than two because they are one GetDetails — an + // export asks about the same referenced id once per slot it appears in, + // and splitting the caches would pay the lookup twice. The two answers + // MUST stay distinct inside the entry: a present-and-unnamed object and + // an absent id both answer ObjectName with "", and collapsing them is + // exactly the conflation ObjectExists exists to avoid (§9). A store + // error caches nothing: an unasked question has no answer worth + // remembering, and a transient failure must not become a persistent + // wrong claim of absence. + objectRows map[string]objectRow + + relsLoaded bool + relById map[string]anyblockjson.PropertyDefinition + relKeyToId map[string]string + + // the §3 key vocabulary, primed lazily — see keyvocab.go + relVocab *keyMaps + typeVocab *keyMaps +} + +// objectRow is one cached point lookup: what the object-namespace seams know +// about one referenced id. +type objectRow struct { + name string + exists bool + // deleted marks a TOMBSTONE: the row survives stripped to its + // bookkeeping. It still counts as exists — see ObjectExists — and only + // the icon rule asks this narrower question. + deleted bool +} + +// New creates resolvers over one space's index. +func New(index spaceindex.Store) *Resolvers { + return &Resolvers{ + index: index, + optionsFor: map[domain.RelationKey][]*model.RelationOption{}, + participantNames: map[string]string{}, + objectRows: map[string]objectRow{}, + } +} + +// Options returns anyblockjson.Options pre-wired with the resolvers and the +// index's own space id (which enables the participant fold, §9); callers set +// the remaining fields (compaction flags, RefNames, etc.) on the returned +// value. +func (r *Resolvers) Options() anyblockjson.Options { + return anyblockjson.Options{ + ResolveFormat: r.ResolveFormat, + ResolveOptions: r, + ResolveProperties: r, + ResolveParticipants: r, + ResolveObjectNames: r, + SpaceId: r.index.SpaceId(), + Keys: r, + } +} + +// loadRelations snapshots the space's relation objects once: the point +// lookups (GetRelationByKey) miss for some legacy relations that the full +// listing still returns, so the map is the primary source and the point +// lookups are the fallback. +func (r *Resolvers) loadRelations() { + if r.relsLoaded { + return + } + r.relsLoaded = true + r.relById = map[string]anyblockjson.PropertyDefinition{} + r.relKeyToId = map[string]string{} + rels, err := r.index.ListAllRelations() + if err != nil { + return + } + for _, rel := range rels { + if rel == nil || rel.Relation == nil { + continue + } + def := anyblockjson.PropertyDefinition{ + Key: domain.RelationKey(rel.Key), + Name: rel.Name, + Format: rel.Format, + ObjectTypes: r.targetTypeKeys(rel.ObjectTypes), + } + r.relById[rel.Id] = def + if _, taken := r.relKeyToId[rel.Key]; !taken { + r.relKeyToId[rel.Key] = rel.Id + } + } +} + +// targetTypeKeys turns a relation's stored `relationFormatObjectTypes` into +// the third type slot of §2a — `type_properties[].object_types`. +// +// It is a translation, not a copy: the store holds the target types' OBJECT +// IDs (objectcreator.fillRelationFormatObjectTypes rewrites bundled urls to +// derived ids at creation), while PropertyDefinition.ObjectTypes is defined in +// stored type KEYS, which the codec then spells at the document boundary. Left +// empty — as it was — a node export of a type document silently dropped every +// property's target types: the slot had no node-backed emitter at all, so an +// `objects` property came back untargeted and would accept any object. +// +// The mapping rides the one bounded type listing the vocabulary already pays +// for (§3 budgets one query per kind per resolver, never one per +// reference), plus the bundled-url arm for ids that were never rewritten. An +// id that resolves to nothing is DROPPED, which is the policy export already +// applies to a recommended-list entry that no longer resolves (§2a) — a +// dangling target is not a type key, and writing one would export a document +// naming a type no reader can find. +func (r *Resolvers) targetTypeKeys(ids []string) []string { + if len(ids) == 0 { + return nil + } + var out []string + for _, id := range ids { + if key, err := bundle.TypeKeyFromUrl(id); err == nil && key != "" { + out = append(out, string(key)) + continue + } + if key := r.typeKeyMaps().keyById[id]; key != "" { + out = append(out, key) + } + } + return out +} + +// ResolveFormat implements anyblockjson.FormatResolver. +func (r *Resolvers) ResolveFormat(key domain.RelationKey) (model.RelationFormat, bool) { + rel, err := r.index.GetRelationByKey(string(key)) + if err != nil || rel == nil { + return 0, false + } + return rel.Format, true +} + +func (r *Resolvers) options(key domain.RelationKey) []*model.RelationOption { + if cached, ok := r.optionsFor[key]; ok { + return cached + } + opts, err := r.index.ListRelationOptions(key) + if err != nil { + opts = nil + } + r.optionsFor[key] = opts + return opts +} + +// OptionName implements anyblockjson.OptionResolver. +func (r *Resolvers) OptionName(key domain.RelationKey, id string) (string, bool) { + for _, o := range r.options(key) { + if o.Id == id { + return o.Text, true + } + } + return "", false +} + +// OptionId implements anyblockjson.OptionResolver. +func (r *Resolvers) OptionId(key domain.RelationKey, name string) (string, bool) { + for _, o := range r.options(key) { + if o.Text == name { + return o.Id, true + } + } + return "", false +} + +// ParticipantName implements anyblockjson.ParticipantResolver: the display +// name of the space member a participant id names (§3). +// +// A participant is an ordinary indexed object whose id is +// `_participant__`, so one point lookup answers it — there is +// no listing to load and no vocabulary to prime. The answer is the `name` the +// space last saw on that member's profile. +// +// The lookup is by id and asks nothing about layout, which is what makes it +// answer for the one attribution value that is NOT a participant: +// `_anytype_profile`, the app itself, stands in `creator` on 7.9% of a +// 36,966-object corpus (bundled types and relations copied into a space) and +// is indexed per space by `reindexIDs`. It resolves to "Anytype", which is +// the true answer to who wrote those objects. +// +// **No name is an answer of "no", not an empty string.** A member who never +// set a profile name has none here, and so does an id this space has no +// participant row for (a member of a space this export is not running in, an +// account whose participant object was never indexed). Both make export omit +// the property, which is the same thing it does with no resolver at all — +// the format's rule is that `creator` is a name or is absent, never a blank. +// +// **Only `name`, deliberately.** A member may also carry `globalName` (their +// any-name) and `identity`; neither is substituted for a missing `name`, +// because a document that falls back to an address has re-introduced the +// address this spelling exists to remove. +func (r *Resolvers) ParticipantName(id string) (string, bool) { + if name, cached := r.participantNames[id]; cached { + return name, name != "" + } + name := "" + if details, err := r.index.GetDetails(id); err == nil && details != nil { + name = details.GetString(bundle.RelationKeyName) + } + r.participantNames[id] = name + return name, name != "" +} + +// objectRow answers one referenced id's row from the cache or one GetDetails. +// ok is false only when the store errored: nothing is cached then and the +// caller answers as if it had not been asked. +// +// GetDetails is what makes the existence answer precise: it returns an EMPTY +// details struct for an id the index has no row for (spaceindex/objects.go), +// while a stored row always carries at least its own id — so emptiness IS +// the absence signal, and it is available on the very lookup the name +// already pays for. A deleted object's tombstone row ({id, isDeleted}) is a +// row: the id still means something in this space, and the missing-reference +// rule (§9) leaves references to it alone. +func (r *Resolvers) objectRow(id string) (objectRow, bool) { + if row, cached := r.objectRows[id]; cached { + return row, true + } + details, err := r.index.GetDetails(id) + if err != nil { + return objectRow{}, false + } + row := objectRow{} + if details != nil && details.Len() > 0 { + row.exists = true + row.name = details.GetString(bundle.RelationKeyName) + row.deleted = details.GetBool(bundle.RelationKeyIsDeleted) + } + r.objectRows[id] = row + return row, true +} + +// ObjectName implements anyblockjson.ObjectNameResolver: the display name of +// the object a reference points at, for the informative `#name` suffix (§9). +// +// The same one-point-lookup shape as ParticipantName, deliberately kept +// separate from it: this one answers for EVERY indexed object — pages, +// files, types, participants alike — because the suffix rides any object +// reference, while the participant seam serves exactly two derived +// properties. No name is an answer of "no", never an empty string: the +// format's rule for the suffix is a name or nothing (a bare reference), +// never a dangling `#`. +func (r *Resolvers) ObjectName(id string) (string, bool) { + row, ok := r.objectRow(id) + return row.name, ok && row.name != "" +} + +// ObjectExists implements anyblockjson.ObjectExistenceResolver: whether the +// space's index holds a row for id at all — the question behind the +// missing-reference rule (§9), riding the same cached point lookup +// ObjectName pays for. +// +// It exists because ObjectName cannot answer it: that seam's ok is +// `name != ""`, so it says "no" for an object that exists UNTITLED — and +// untitled objects are common. known is false only when the store errored, +// and the codec then leaves the reference untouched: a claim of absence has +// to come from the store actually answering, never from a failure to ask. +func (r *Resolvers) ObjectExists(id string) (exists, known bool) { + row, ok := r.objectRow(id) + return row.exists, ok +} + +// ObjectDeleted implements anyblockjson.ObjectDeletionResolver: whether the +// row the space kept for id is a TOMBSTONE — deleted, stripped to its +// bookkeeping. It rides the same cached point lookup ObjectExists pays for, +// and is a strictly narrower question: every deleted id also exists. +// +// Only the icon rule consults it (§2b). A deleted object stays a live +// reference everywhere else, which is ObjectExists's documented position. +func (r *Resolvers) ObjectDeleted(id string) (deleted, known bool) { + row, ok := r.objectRow(id) + return row.deleted, ok +} + +// PropertyById implements anyblockjson.PropertyResolver. +func (r *Resolvers) PropertyById(id string) (anyblockjson.PropertyDefinition, bool) { + r.loadRelations() + if def, ok := r.relById[id]; ok { + return def, true + } + rel, err := r.index.GetRelationById(id) + if err != nil || rel == nil { + return anyblockjson.PropertyDefinition{}, false + } + def := anyblockjson.PropertyDefinition{ + Key: domain.RelationKey(rel.Key), + Name: rel.Name, + Format: rel.Format, + ObjectTypes: r.targetTypeKeys(rel.ObjectTypes), + } + // cache the point-lookup hit both ways: some relations resolve by id but + // are absent from the listing AND the by-key lookup (deleted or index + // gap — anomaly #9 class), so without this PropertyId cannot invert the + // key export just produced and the entry is dropped on re-export + // (resolvers must be equivalent both directions, SPEC §2a/§13) + r.relById[id] = def + if _, taken := r.relKeyToId[rel.Key]; !taken { + r.relKeyToId[rel.Key] = id + } + return def, true +} + +// SeedProperty registers a definition for an id the index cannot currently +// answer for. The one caller class is the API v2 wiring's tombstone window: a +// deleted relation's index row is stripped to {id, isDeleted} until the next +// space load, so GetRelationById fails on an id the surviving TREE still +// fully describes — the wiring reads the live object and seeds what the +// index will hold again after reindex. Seeds never override a loaded row +// (the by-key map keeps its first binding), matching the cache discipline of +// PropertyById's point-lookup arm. +func (r *Resolvers) SeedProperty(id string, def anyblockjson.PropertyDefinition) { + r.loadRelations() + if _, ok := r.relById[id]; ok { + return + } + r.relById[id] = def + if _, taken := r.relKeyToId[string(def.Key)]; !taken { + r.relKeyToId[string(def.Key)] = id + } +} + +// PropertyId implements anyblockjson.PropertyResolver. +func (r *Resolvers) PropertyId(def anyblockjson.PropertyDefinition) (string, bool) { + r.loadRelations() + if id, ok := r.relKeyToId[string(def.Key)]; ok { + return id, true + } + rel, err := r.index.GetRelationByKey(string(def.Key)) + if err != nil || rel == nil { + return "", false + } + r.relKeyToId[rel.Key] = rel.Id + return rel.Id, true +} diff --git a/pkg/lib/anyblockjson/storeresolver/storeresolver_test.go b/pkg/lib/anyblockjson/storeresolver/storeresolver_test.go new file mode 100644 index 0000000000..53742a2a41 --- /dev/null +++ b/pkg/lib/anyblockjson/storeresolver/storeresolver_test.go @@ -0,0 +1,113 @@ +package storeresolver + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/anyblockjson" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +type fixture struct { + *Resolvers + index *spaceindex.StoreFixture +} + +func newFixture(t *testing.T) *fixture { + index := spaceindex.NewStoreFixture(t) + index.AddObjects(t, []spaceindex.TestObject{ + { + bundle.RelationKeyId: domain.String("rel-priority"), + bundle.RelationKeyRelationKey: domain.String("priority"), + bundle.RelationKeyName: domain.String("Priority"), + bundle.RelationKeyRelationFormat: domain.Int64(int64(model.RelationFormat_status)), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_relation)), + }, + { + bundle.RelationKeyId: domain.String("opt-high"), + bundle.RelationKeyRelationKey: domain.String("priority"), + bundle.RelationKeyName: domain.String("High"), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_relationOption)), + }, + }) + return &fixture{Resolvers: New(index), index: index} +} + +func TestResolveFormat(t *testing.T) { + t.Run("known custom key resolves", func(t *testing.T) { + // given + fx := newFixture(t) + + // when + format, ok := fx.ResolveFormat("priority") + + // then + require.True(t, ok) + assert.Equal(t, model.RelationFormat_status, format) + }) + + t.Run("unknown key does not resolve", func(t *testing.T) { + // given + fx := newFixture(t) + + // when + _, ok := fx.ResolveFormat("nope") + + // then + assert.False(t, ok) + }) +} + +func TestOptionResolution(t *testing.T) { + // given + fx := newFixture(t) + + // when / then: both directions invert each other + name, ok := fx.OptionName("priority", "opt-high") + require.True(t, ok) + assert.Equal(t, "High", name) + + id, ok := fx.OptionId("priority", "High") + require.True(t, ok) + assert.Equal(t, "opt-high", id) + + _, ok = fx.OptionName("priority", "missing") + assert.False(t, ok) +} + +func TestPropertyResolution(t *testing.T) { + // given + fx := newFixture(t) + want := anyblockjson.PropertyDefinition{ + Key: "priority", + Name: "Priority", + Format: model.RelationFormat_status, + } + + // when / then: id -> definition -> id round-trips + def, ok := fx.PropertyById("rel-priority") + require.True(t, ok) + assert.Equal(t, want, def) + + id, ok := fx.PropertyId(def) + require.True(t, ok) + assert.Equal(t, "rel-priority", id) +} + +func TestOptionsWiring(t *testing.T) { + // given + fx := newFixture(t) + + // when + opts := fx.Options() + + // then + assert.NotNil(t, opts.ResolveFormat) + assert.Equal(t, fx.Resolvers, opts.ResolveOptions) + assert.Equal(t, fx.Resolvers, opts.ResolveProperties) +} diff --git a/pkg/lib/anyblockjson/systemtrim.go b/pkg/lib/anyblockjson/systemtrim.go new file mode 100644 index 0000000000..c4335fb193 --- /dev/null +++ b/pkg/lib/anyblockjson/systemtrim.go @@ -0,0 +1,101 @@ +package anyblockjson + +// systemtrim.go — the seven system-stamped properties whose EMPTY value is +// not written (§3, §15 #12). +// +// §3's rule is that the presence of a property key is meaningful: it records +// that the property was set on this object, so values are written verbatim, +// empty ones included. That rule earns its keep for anything a person +// touched — an empty `tag` list is a person having cleared the tags — and it +// costs nothing on an ordinary page, which carries few system keys. +// +// It costs a great deal on the documents this format exists to be read: +// measured over 36,967 production documents, empty system-stamped values are +// 1.13% of all bytes but the distribution is bimodal — p50 1.21%, p90 +// 13.55%, max 23.22% — because `relationDefaultValue`, `relationMaxCount` +// and their neighbours appear only on RELATION and TYPE documents. An agent +// reading a space's SCHEMA reads exactly the documents that pay ~20%. +// +// So the carve-out is a WHITELIST, not a rule over a category. The blanket +// form — every key in bundle.SystemRelations minus an exception list — was +// declined: it admits every system relation added in future sight-unseen, +// and it buys almost nothing, because the saving is top-heavy. These seven +// keys carry ~50% of it; the thirty-key tail carries 3.6% of 1.13%, which is +// 0.04% of all bytes. An explicit list gets nearly all of the benefit with +// every omission vetted. +// +// Admission test, applied to each key below: does anything distinguish +// present-and-empty from absent? For a system-stamped flag whose empty value +// IS the proto zero, nothing does — every reader reaches it through a typed +// getter that answers the same for both. Keys that failed the test and are +// deliberately absent from this list: +// +// - `relationFormat` — 0 is `longtext`, a real format, not "unset" (§15 #14). +// - `relationFormatObjectTypes` — list-valued and user-intent-bearing, the +// same empty-vs-absent shape GO-7451 settled the other way for a type's +// recommended lists: an empty list is how a cleared set is expressed, so +// it has to survive. +// - `featuredRelations` was excluded here for the same reason, and the +// reason was wrong: an empty list there is not a cleared set but the +// LAYOUT SYNCER's output (layout/syncer.go), since no UI sets a +// per-object featured list. The key is now deprecated outright and never +// reaches this rule. +// +// This is a state normalization, recorded in `N(S)` (§11): such a key comes +// back ABSENT. DroppedEmptySystemProperty exists so the round-trip +// comparator can suppress exactly that step — the rule lives here, in one +// place, so the comparator and the exporter cannot disagree about it. + +import ( + "github.com/gogo/protobuf/types" +) + +// trimmedWhenEmpty maps each admitted stored key to why its empty value says +// nothing. Every entry is a system relation whose empty value is the proto +// zero AND whose zero is the semantic default, so a reader that asks gets the +// same answer whether the key is absent or present-and-empty. +var trimmedWhenEmpty = map[string]string{ + "relationDefaultValue": "no default value: empty IS the absence of one", + "relationReadonlyValue": "false: the relation is writable, the default", + "revision": "0: no bundled revision recorded", + "isHidden": "false: the object is visible, the default", + "isHiddenDiscovery": "false: the object is discoverable, the default", + "isArchived": "false: the object is not archived, the default", + "relationMaxCount": "0: unlimited, the default", +} + +// DroppedEmptySystemProperty reports a stored detail that export omits +// because it is one of the admitted system-stamped keys (§15 #12) and its +// value is empty. It is the exported half of the rule, for the round-trip +// comparator; the predicate is the format's own, not a copy of it. +// +// Scoped to empty-and-admitted and nothing else: a NON-empty value on the +// same key still reports as loss if it ever goes missing, and a key outside +// the list reports whatever it did before. +func DroppedEmptySystemProperty(key string, v *types.Value) bool { + _, admitted := trimmedWhenEmpty[key] + return admitted && isEmptySystemValue(v) +} + +// isEmptySystemValue is the emptiness rule the export path applies, in one +// place so the comparator and the builder cannot disagree about which values +// are empty. A bool is spelled out rather than falling through, because +// `false` is precisely the value this rule is about — unlike the icon/cover +// rule beside it (liftedValueIsSource), where no source is a bool. +func isEmptySystemValue(v *types.Value) bool { + switch k := v.GetKind().(type) { + case *types.Value_StringValue: + return k.StringValue == "" + case *types.Value_NumberValue: + return k.NumberValue == 0 + case *types.Value_BoolValue: + return !k.BoolValue + case *types.Value_ListValue: + return len(k.ListValue.GetValues()) == 0 + case *types.Value_StructValue: + return len(k.StructValue.GetFields()) == 0 + case *types.Value_NullValue: + return true + } + return v.GetKind() == nil +} diff --git a/pkg/lib/anyblockjson/systemtrim_test.go b/pkg/lib/anyblockjson/systemtrim_test.go new file mode 100644 index 0000000000..c6b3a98211 --- /dev/null +++ b/pkg/lib/anyblockjson/systemtrim_test.go @@ -0,0 +1,141 @@ +package anyblockjson + +// systemtrim_test.go — the seven system-stamped keys whose empty value is not +// written (§15 #12), and the keys deliberately kept out of that list. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func boolValue(b bool) *types.Value { + return &types.Value{Kind: &types.Value_BoolValue{BoolValue: b}} +} + +func trimSnapshot(details map[string]*types.Value) *model.SmartBlockSnapshotBase { + details["id"] = str("bafyreitrimroot") + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: "bafyreitrimroot", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(details), + } +} + +// Each admitted key, empty, is absent from the document — and the SAME key +// with a value is written, so the rule is about the value and never about +// the key. +// +// How this can fail: drop the buildProperties skip and the empty spellings +// appear; widen isEmptySystemValue and the non-empty ones vanish. +func TestSystemTrim_AnEmptyAdmittedKeyIsNotWritten(t *testing.T) { + for key, filled := range map[string]*types.Value{ + "relationDefaultValue": str("unstarted"), + "relationReadonlyValue": boolValue(true), + "revision": num(3), + "isHidden": boolValue(true), + "isHiddenDiscovery": boolValue(true), + "isArchived": boolValue(true), + "relationMaxCount": num(1), + } { + t.Run(key, func(t *testing.T) { + // given the same key, once empty and once carrying a value + empty := map[string]*types.Value{key: emptyLike(filled)} + set := map[string]*types.Value{key: filled} + + // when + emptyDoc, err := Marshal(model.SmartBlockType_Page, trimSnapshot(empty), testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(emptyDoc), "I1: Marshal never emits what its own Validate rejects") + setDoc, err := Marshal(model.SmartBlockType_Page, trimSnapshot(set), testOptions()) + require.NoError(t, err) + + // then + slug := BundledKeyVocabulary{}.PropertySlug(key) + assert.NotContains(t, string(emptyDoc), `"`+slug+`"`, + "an empty %s says nothing a reader could act on (§15 #12)", key) + assert.Contains(t, string(setDoc), `"`+slug+`"`, + "the rule is about the VALUE, never the key") + }) + } +} + +// emptyLike returns the empty value of the same kind, so each case tests the +// emptiness that key actually carries in the store rather than a uniform nil. +func emptyLike(v *types.Value) *types.Value { + switch v.GetKind().(type) { + case *types.Value_BoolValue: + return boolValue(false) + case *types.Value_NumberValue: + return num(0) + } + return str("") +} + +// The keys that FAILED the admission test keep their empty value. Each is one +// line away from being trimmed, and each would be wrong: +// `relationFormat`'s zero is `longtext`, a real format; the list-valued +// user-intent keys express a CLEARED set by being empty, which GO-7451 +// settled for a type's recommended lists. `relationFormat` and +// `relationFormatObjectTypes` have since moved to a relation document's +// envelope (§2d), where the SAME admission verdict holds: the envelope +// fields mirror stored presence, empty values included. +// +// How this can fail: add featuredRelations to trimmedWhenEmpty and the page +// assertion finds the key gone; make buildPropertySettings treat format 0 as +// unset, or omit an empty object_types list, and the relation assertions +// find the fields missing. +func TestSystemTrim_TheExcludedKeysKeepTheirEmptyValue(t *testing.T) { + // given + pageSnap := trimSnapshot(map[string]*types.Value{ + "featuredRelations": strList(), + }) + relSnap := trimSnapshot(map[string]*types.Value{ + "relationFormat": num(0), // 0 is longtext, not "unset" + "relationFormatObjectTypes": strList(), + }) + + // when + pageDoc, err := Marshal(model.SmartBlockType_Page, pageSnap, testOptions()) + require.NoError(t, err) + relDoc, err := Marshal(model.SmartBlockType_STRelation, relSnap, testOptions()) + require.NoError(t, err) + + // then + // featuredRelations used to be this test's example of a key deliberately + // OUTSIDE the whitelist, on the reasoning that an empty list is a cleared + // set. That reasoning was wrong — no UI sets a per-object featured list, + // and an empty one is the layout syncer's output — so the key is now + // deprecated outright and never reaches this rule at all. + assert.NotContains(t, string(pageDoc), `"featured_properties"`, + "deprecated: the type owns an object's featured list") + assert.Contains(t, string(relDoc), `"format": "text"`, + "relationFormat 0 is longtext, a real format, and §2d requires the field") + assert.Contains(t, string(relDoc), `"object_types": []`, + "an empty target set is a CLEARED set; the §2d field mirrors stored presence") +} + +// The whitelist is a list, not a category: a system relation NOT on it keeps +// its empty value, which is the whole difference between this and the +// blanket rule over bundle.SystemRelations that was declined. +// +// How this can fail: replace trimmedWhenEmpty with a SystemRelations +// membership test and this key disappears. +func TestSystemTrim_AnUnlistedSystemRelationIsUntouched(t *testing.T) { + // given `origin` is in bundle.SystemRelations and not on the whitelist + snap := trimSnapshot(map[string]*types.Value{"origin": num(0)}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"Origin"`, + "admission is by explicit entry, never by category") +} diff --git a/pkg/lib/anyblockjson/table.go b/pkg/lib/anyblockjson/table.go new file mode 100644 index 0000000000..582c36bc7f --- /dev/null +++ b/pkg/lib/anyblockjson/table.go @@ -0,0 +1,546 @@ +package anyblockjson + +// table.go maps the internal table block subtree (table → row/column layout +// wrappers → cells with composite - ids) to the §6.1 +// columns/rows JSON form and back. + +import ( + "fmt" + "sort" + "strings" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +const tableWidthField = "width" + +// tableToJSON flattens the table subtree into columns/rows, mirroring the +// editor's normalization: header rows first, cells sorted into column order, +// orphan cells dropped. Only a structurally unrecognizable subtree (missing +// wrappers) is an error (§6.1). +func (e *exporter) tableToJSON(m *omap, b *model.Block) error { + m.set("type", "table") + + var colsWrapper, rowsWrapper *model.Block + for _, id := range b.ChildrenIds { + child := e.blocks[id] + if child == nil { + continue + } + if l, ok := child.Content.(*model.BlockContentOfLayout); ok { + switch l.Layout.Style { + case model.BlockContentLayout_TableColumns: + colsWrapper = child + case model.BlockContentLayout_TableRows: + rowsWrapper = child + } + } + } + if colsWrapper == nil || rowsWrapper == nil { + return fmt.Errorf("table %s: missing row/column wrappers", b.Id) + } + + var colIds []string + var columns []any + for _, colId := range colsWrapper.ChildrenIds { + col := e.blocks[colId] + if col == nil || e.visited[colId] { + continue + } + if _, ok := col.Content.(*model.BlockContentOfTableColumn); !ok { + continue // orphan blocks in the columns wrapper are dropped + } + e.visited[colId] = true + e.recordEmitted(colId) + colIds = append(colIds, colId) + cm := &omap{} + if !e.opts.OmitIds { + cm.setNonEmpty("id", e.tableInnerId(colId)) + } + lifted := map[string]bool{} + if col.Fields != nil { + if w := col.Fields.Fields[tableWidthField]; w != nil { + if _, isNum := w.GetKind().(*types.Value_NumberValue); isNum { + cm.setNonEmpty("width", w.GetNumberValue()) + lifted[tableWidthField] = true + } + } + } + cm.setNonEmpty("fields", e.fieldsToJSON(col.Fields, lifted)) + columns = append(columns, cm) + } + + // header rows come first (editor invariant); stable to keep row order + var rowBlocks []*model.Block + for _, rowId := range rowsWrapper.ChildrenIds { + row := e.blocks[rowId] + if row == nil || e.visited[rowId] { + continue + } + if _, ok := row.Content.(*model.BlockContentOfTableRow); !ok { + continue + } + e.visited[rowId] = true + rowBlocks = append(rowBlocks, row) + } + isHeader := func(b *model.Block) bool { + return orEmpty(b.Content.(*model.BlockContentOfTableRow).TableRow).IsHeader + } + sort.SliceStable(rowBlocks, func(i, j int) bool { + return isHeader(rowBlocks[i]) && !isHeader(rowBlocks[j]) + }) + + var rows []any + for _, row := range rowBlocks { + rm := &omap{} + e.recordEmitted(row.Id) + if !e.opts.OmitIds { + rm.setNonEmpty("id", e.tableInnerId(row.Id)) + } + rm.setNonEmpty("is_header", isHeader(row)) + + // cells sorted into column order; orphans dropped + byCol := map[string]*model.Block{} + for _, cellId := range row.ChildrenIds { + colId, ok := strings.CutPrefix(cellId, row.Id+"-") + if !ok { + continue + } + if cell := e.blocks[cellId]; cell != nil { + byCol[colId] = cell + } + } + cells := make([]any, 0, len(colIds)) + for _, colId := range colIds { + cell := byCol[colId] + cv, err := e.cellToJSON(cell) + if err != nil { + return err + } + cells = append(cells, cv) + } + // trailing empty cells are omitted (import pads, §6.1) + for len(cells) > 0 && cells[len(cells)-1] == nil { + cells = cells[:len(cells)-1] + } + rm.setNonEmpty("cells", cells) + rows = append(rows, rm) + } + + m.setNonEmpty("columns", columns) + m.setNonEmpty("rows", rows) + return nil +} + +// cellToJSON renders a cell: nil for empty, the string shorthand for a plain +// paragraph, a block object (without id — derived) otherwise. A cell whose +// block has descendants renders as an array of flat blocks — the cell block +// first at indent 0, the descendants following with their depths (§6.1 F10). +func (e *exporter) cellToJSON(cell *model.Block) (any, error) { + if cell == nil { + return nil, nil + } + // §7a: a transparent container has no block of its own, and a cell is a + // position rather than a run — there is nowhere to lift to. The cell + // renders empty, and a container that held a subtree says so, because + // the subtree goes with it. Unreachable from normalization (a cell's + // parent is a TableRow, which normalizeTreeBranch never wraps) and + // absent from the production corpus; it is corrupt input, not a shape + // the editor makes. + if isTransparentContainer(cell) { + e.visited[cell.Id] = true + if len(cell.ChildrenIds) > 0 { + e.warn("", "cell %s is a transparent container: a cell cannot be lifted, so it renders empty and its %d children are dropped", + cell.Id, len(cell.ChildrenIds)) + } + return nil, nil + } + if c, ok := cell.Content.(*model.BlockContentOfText); ok { + t := orEmpty(c.Text) + if cell.Id != "" && e.visited[cell.Id] { + // the mark this branch sets, read back. A block reached twice is + // emitted once (§11) — blockToJSON drops the second arrival, and + // the shorthand, which never goes through blockToJSON, has to drop + // it too. Setting the mark without consulting it only ordered the + // two arrivals: a cell reached first silenced the other parent, + // while a cell reached second wrote the block A SECOND TIME, so + // one stored block came back from import as two. + return nil, nil + } + if t.Style == model.BlockContentText_Paragraph && + t.Color == "" && !t.Checked && + cell.Align == model.Block_AlignLeft && + cell.VerticalAlign == model.Block_VerticalAlignTop && + cell.BackgroundColor == "" && + (cell.Fields == nil || len(cell.Fields.Fields) == 0) && + len(cell.ChildrenIds) == 0 { + // the shorthand renders the block without going through + // blockToJSON, which is where the emit-once mark is set (§11). + // Unmarked, a block that is both this cell and a child elsewhere + // is written twice — the second time with its id, which is the + // derived cell id this row already claims. + e.visited[cell.Id] = true + // the shorthand renders without going through textToJSON, so it + // owes the same mention-target check (§8, §9) + md := renderInline(t.Text, e.exportMarks("/blocks", t.Marks.GetMarks())) + if md == "" { + return nil, nil // empty paragraph collapses to an empty cell (§11) + } + return md, nil + } + } + m, withChildren, err := e.blockToJSON(cell, 0) + if err != nil { + return nil, err + } + if m == nil { + return nil, nil + } + // cells cannot contain tables — the schema's recursion cut (§6.1, §12). + // Erring here keeps the invariant that Marshal never emits output its + // own Validate rejects; the prod sweep found zero such cells, so this is + // an adversarial/legacy guard, not a live path. + if blockJSONType(m) == "table" { + return nil, fmt.Errorf("cell %s: a table block cannot be a cell (cells cannot contain tables)", cell.Id) + } + // cell ids are derived, never serialized (§6.1) + if len(m.keys) > 0 && m.keys[0] == "id" { + m.keys = m.keys[1:] + m.vals = m.vals[1:] + } + if withChildren && len(cell.ChildrenIds) > 0 { + arr := []any{m} + if err := e.appendBlocksFlat(&arr, cell.ChildrenIds, 1, false); err != nil { + return nil, err + } + for _, el := range arr[1:] { + if bm, ok := el.(*omap); ok && blockJSONType(bm) == "table" { + return nil, fmt.Errorf("cell %s: a table block among cell descendants cannot be represented (cells cannot contain tables)", cell.Id) + } + } + if len(arr) > 1 { + return arr, nil + } + // every descendant was dropped (visited/content-less): bare form stays + // canonical + } + return m, nil +} + +// blockJSONType reads the rendered block's type discriminator. +func blockJSONType(m *omap) string { + for i, k := range m.keys { + if k == "type" { + s, _ := m.vals[i].(string) + return s + } + } + return "" +} + +// +// ---- import ---- +// + +type jsonTableColumn struct { + Id string `json:"id"` + Width float64 `json:"width"` + Fields map[string]any `json:"fields"` +} + +type jsonTableRow struct { + Id string `json:"id"` + IsHeader bool `json:"is_header"` + Cells []jsonCell `json:"cells"` +} + +// jsonCell is string | null | block object | array of flat blocks (§6.1). +type jsonCell struct { + Text *string + Block *jsonBlock + Blocks []*jsonBlock // array form: cell block first, descendants flat (F10) +} + +func (c *jsonCell) UnmarshalJSON(data []byte) error { + trimmed := strings.TrimSpace(string(data)) + if trimmed == "null" { + return nil + } + if strings.HasPrefix(trimmed, `"`) { + var s string + if err := jsonUnmarshal(data, &s); err != nil { + return err + } + c.Text = &s + return nil + } + if strings.HasPrefix(trimmed, `[`) { + return jsonUnmarshal(data, &c.Blocks) + } + var b jsonBlock + if err := jsonUnmarshal(data, &b); err != nil { + return err + } + c.Block = &b + return nil +} + +// tableFromJSON rebuilds the internal subtree. It returns the table block +// and every block of the subtree (wrappers, columns, rows, cells). +func (imp *importer) tableFromJSON(jb *jsonBlock, tableId string) (*model.Block, []*model.Block, error) { + table := &model.Block{ + Id: tableId, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}, + } + var extra []*model.Block + + colsWrapper := &model.Block{ + Id: imp.genId(), + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}, + } + rowsWrapper := &model.Block{ + Id: imp.genId(), + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}, + } + table.ChildrenIds = []string{colsWrapper.Id, rowsWrapper.Id} + + // the rows' authored ids, known before a single column is minted: a + // generated column id has to keep the whole column of derived ids it + // implies clear, and the authored rows are the half of the grid that + // cannot move. + authoredRowIds := make([]string, 0, len(jb.Rows)) + for _, jr := range jb.Rows { + if jr.Id != "" { + authoredRowIds = append(authoredRowIds, jr.Id) + } + } + + colIds := make([]string, 0, len(jb.Columns)) + for _, jc := range jb.Columns { + id := jc.Id + if id == "" { + id = imp.newTableInnerId(func(colId string) bool { + return imp.derivedIdTaken(authoredRowIds, func(rowId string) string { return rowId + "-" + colId }) + }) + } else { + imp.claimTableInnerId(id) + } + colIds = append(colIds, id) + fields := jsonMapToProtoStruct(jc.Fields) + if jc.Width != 0 { + if fields == nil || fields.Fields == nil { + fields = &types.Struct{Fields: map[string]*types.Value{}} + } + fields.Fields[tableWidthField] = &types.Value{Kind: &types.Value_NumberValue{NumberValue: jc.Width}} + } + if len(fields.GetFields()) == 0 { + fields = nil + } + col := &model.Block{ + Id: id, + Fields: fields, + Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}, + } + colsWrapper.ChildrenIds = append(colsWrapper.ChildrenIds, id) + extra = append(extra, col) + } + + // header rows first: import reorders rather than rejects (§6.1) + rows := make([]jsonTableRow, len(jb.Rows)) + copy(rows, jb.Rows) + sort.SliceStable(rows, func(i, j int) bool { return rows[i].IsHeader && !rows[j].IsHeader }) + + // Every row id is resolved — and the WHOLE grid claimed — before a cell is + // built, because building one generates ids (a cell block's descendants, + // the next table) and the grid is not free for them to take: the table + // owns `-` for every pair, written or not (§6.1). Only the + // authored half of that grid was ever claimed, so a generated row or + // column left its whole row of derived ids unreserved — and an authored + // block sitting on one came back from import as a second block with the + // same id, which is a snapshot no editor can resolve. + rowIds := make([]string, len(rows)) + for i, jr := range rows { + if jr.Id != "" { + rowIds[i] = imp.claimTableInnerId(jr.Id) + continue + } + rowIds[i] = imp.newTableInnerId(func(rowId string) bool { + return imp.derivedIdTaken(colIds, func(colId string) string { return rowId + "-" + colId }) + }) + } + for _, rowId := range rowIds { + for _, colId := range colIds { + imp.claimId(rowId + "-" + colId) + } + } + + for rowIdx, jr := range rows { + rowId := rowIds[rowIdx] + row := &model.Block{ + Id: rowId, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{IsHeader: jr.IsHeader}}, + } + if len(jr.Cells) > len(colIds) { + return nil, nil, fmt.Errorf("table %s row %s: %d cells for %d columns", tableId, rowId, len(jr.Cells), len(colIds)) + } + for i, cell := range jr.Cells { + cellBlocks, err := imp.cellFromJSON(cell, rowId+"-"+colIds[i]) + if err != nil { + return nil, nil, err + } + if len(cellBlocks) > 0 { + row.ChildrenIds = append(row.ChildrenIds, cellBlocks[0].Id) + extra = append(extra, cellBlocks...) + } + } + rowsWrapper.ChildrenIds = append(rowsWrapper.ChildrenIds, rowId) + extra = append(extra, row) + } + + extra = append([]*model.Block{colsWrapper, rowsWrapper}, extra...) + return table, extra, nil +} + +// cellFromJSON builds a cell block (with its derived id) and, for the array +// form, its flat descendants (F10). Empty cells produce no blocks. +func (imp *importer) cellFromJSON(cell jsonCell, cellId string) ([]*model.Block, error) { + if cell.Text != nil { + if *cell.Text == "" { + return nil, nil + } + text, marks, err := parseInline(*cell.Text) + if err != nil { + return nil, fmt.Errorf("cell %s: %w", cellId, err) + } + return []*model.Block{{ + Id: cellId, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Text: text, + Marks: &model.BlockContentTextMarks{Marks: marks}, + }}, + }}, nil + } + if len(cell.Blocks) > 0 { + // array form: first element is the cell block, the rest its + // descendants per the §4 F6 stack rebuild + blocks, err := imp.blockFromJSON(cell.Blocks[0], cellId) + if err != nil { + return nil, err + } + rest := cell.Blocks[1:] + restJbs, restIndents := liftTransparentContainers(rest, imp.blockIndents(rest, 0)) + extra, err := imp.flatSubtree(restJbs, restIndents, blocks[0], 0) + if err != nil { + return nil, err + } + return append(blocks, extra...), nil + } + if cell.Block == nil { + return nil, nil + } + // an empty plain paragraph collapses to an empty cell (§11) + b := cell.Block + if b.Type == "paragraph" && b.Text == "" && b.Color == "" && !b.Checked && + b.Align == "" && b.VerticalAlign == "" && b.BackgroundColor == "" && + len(b.Fields) == 0 { + return nil, nil + } + blocks, err := imp.blockFromJSON(b, cellId) + if err != nil { + return nil, err + } + return blocks, nil +} + +// claimTableInnerId reserves an authored row/column id so a generated one +// cannot collide with it. claimAuthoredIds has normally seen it already; this +// keeps the guarantee local to the caller rather than assuming that. +func (imp *importer) claimTableInnerId(id string) string { + return imp.claimId(id) +} + +// newTableInnerId mints a row or column id that is safe to build a cell id +// from. A cell's id is rowId + "-" + colId, and the whole editor recovers the +// column from it with SplitN(id, "-", 2) (table.ParseCellID, which drives +// every column insert/delete/move, HTML export and table normalization), so a +// row or column id must contain no "-" at all — hence the schema's +// [A-Za-z0-9_]{1,64} on authored ones. +// +// Generated ids have to honour the same rule, and Options.GenerateId belongs +// to the caller: the convert wiring derives ids from file paths, which are +// full of dashes. So sanitize rather than trust, and disambiguate on +// collision instead of hoping the sanitized forms stay distinct. +// +// derived reports whether a candidate would make a cell id that is already +// somebody's: a row or column id is never alone, it names a whole line of the +// grid, and a line landing on an existing id is the same collision as the id +// itself landing on one. The candidate is what moves, because a derived id has +// no spelling of its own (§6.1). +func (imp *importer) newTableInnerId(derived func(string) bool) string { + id := imp.genId() + sanitized := sanitizeTableInnerId(id) + if sanitized == id && !derived(id) { + // nothing to sanitize: genId's answer is already unique and claimed, + // and running it through the disambiguation pass would find it taken + // by its own claim and rename it to _2 — which is what every + // generated row and column id used to be called + return id + } + // the sanitized form is a different string, so it has to be claimed on + // its own; the raw one stays claimed, which costs nothing + return imp.claimId(uniqueLabel(sanitized, func(candidate string) bool { + return imp.idTaken(candidate) || derived(candidate) + })) +} + +// derivedIdTaken reports whether any cell id the candidate implies is already +// claimed — cellId maps each id of the opposite axis to the pair's derived id. +func (imp *importer) derivedIdTaken(others []string, cellId func(string) string) bool { + for _, other := range others { + if imp.idTaken(cellId(other)) { + return true + } + } + return false +} + +// maxTableInnerId mirrors the schema's tableInnerId length bound, so a +// generated id validates on re-export. +const maxTableInnerId = 64 + +func sanitizeTableInnerId(s string) string { + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + out := b.String() + if out == "" { + out = "c" + } + if len(out) > maxTableInnerId { + out = out[:maxTableInnerId] + } + return out +} + +// tableInnerId renders a stored row/column id for output. Stored ids can hold +// characters the format forbids in that position (§6.1): historical data and +// any generator that derives ids from file paths both produce "-", which is +// the cell-id separator. Emitting one verbatim would make Marshal write a +// document its own Validate rejects, so normalize it once here. Only the +// label changes — the cell mapping keys off the stored id. +// +// The uniqueness domain is the whole document, not the table: a column id +// sanitized to "c_1" has to avoid a sibling paragraph already called "c_1" +// just as much as it has to avoid another column (§4). +func (e *exporter) tableInnerId(stored string) string { + return e.idLabel(stored, sanitizeTableInnerId) +} diff --git a/pkg/lib/anyblockjson/tableid_test.go b/pkg/lib/anyblockjson/tableid_test.go new file mode 100644 index 0000000000..97d24bc7a9 --- /dev/null +++ b/pkg/lib/anyblockjson/tableid_test.go @@ -0,0 +1,487 @@ +package anyblockjson + +// A cell's id is rowId + "-" + colId, and the editor recovers the column from +// it with SplitN(id, "-", 2) (table.ParseCellID — the basis of every column +// insert/delete/move, HTML export and table normalization). A row or column +// id containing "-" therefore corrupts column identity, which is why the +// schema pins authored ones to [A-Za-z0-9_]{1,64}. Generated ids must honour +// the same rule, and Options.GenerateId belongs to the caller. + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +const tableDoc = `{"version": 2, "id": "p1", "blocks": [{"type": "table", + "columns": [{}, {}], + "rows": [{"cells": ["a", "b"]}]}]}` + +// the convert wiring derives ids from file paths, so its generator is full of +// dashes — the package cannot assume otherwise +func TestImport_GeneratedTableIdsAreSeparatorFree(t *testing.T) { + n := 0 + _, snap, err := Unmarshal([]byte(tableDoc), Options{ + GenerateId: func() string { + n++ + return "pages-doc-kickoff-onboarding-" + string(rune('0'+n)) + }, + }) + require.NoError(t, err) + + var rows, cols, cells []string + for _, b := range snap.Blocks { + switch b.Content.(type) { + case *model.BlockContentOfTableRow: + rows = append(rows, b.Id) + case *model.BlockContentOfTableColumn: + cols = append(cols, b.Id) + } + } + require.Len(t, rows, 1) + require.Len(t, cols, 2) + for _, id := range append(append([]string{}, rows...), cols...) { + assert.NotContains(t, id, "-", "row/column id must be separator-free") + assert.Regexp(t, `^[A-Za-z0-9_]{1,64}$`, id) + } + + // and the cells built from them carry exactly one separator + for _, row := range rows { + for _, cellId := range blockById(snap, row).ChildrenIds { + cells = append(cells, cellId) + assert.Equal(t, 1, strings.Count(cellId, "-"), + "cell id must split into exactly rowId and colId") + rowId, colId, _ := strings.Cut(cellId, "-") + assert.Equal(t, row, rowId) + assert.Contains(t, cols, colId) + } + } + assert.Len(t, cells, 2) +} + +func blockById(snap *model.SmartBlockSnapshotBase, id string) *model.Block { + for _, b := range snap.Blocks { + if b.Id == id { + return b + } + } + return nil +} + +// distinct source ids must stay distinct after sanitizing +func TestImport_SanitizedTableIdsStayUnique(t *testing.T) { + ids := []string{"a-b", "a_b", "a.b", "a b"} // all sanitize to "a_b" + n := 0 + _, snap, err := Unmarshal([]byte(`{"version": 2, "id": "p1", "blocks": [{"type": "table", + "columns": [{}, {}], "rows": [{"cells": ["x", "y"]}, {"cells": ["z"]}]}]}`), + Options{GenerateId: func() string { + id := ids[n%len(ids)] + n++ + return id + }}) + require.NoError(t, err) + + seen := map[string]bool{} + for _, b := range snap.Blocks { + switch b.Content.(type) { + case *model.BlockContentOfTableRow, *model.BlockContentOfTableColumn: + assert.False(t, seen[b.Id], "duplicate table inner id %q", b.Id) + seen[b.Id] = true + assert.Regexp(t, `^[A-Za-z0-9_]{1,64}$`, b.Id) + } + } +} + +// Marshal must never emit a document its own Validate rejects: stored tables +// predating this rule carry dashed row/column ids. +func TestExport_DashedTableIdsAreNormalized(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "root", ChildrenIds: []string{"tbl"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "tbl", ChildrenIds: []string{"cols", "rows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "cols", ChildrenIds: []string{"pages-doc-12"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: "rows", ChildrenIds: []string{"pages-doc-17"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + {Id: "pages-doc-12", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + {Id: "pages-doc-17", ChildrenIds: []string{"pages-doc-17-pages-doc-12"}, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + textBlock("pages-doc-17-pages-doc-12", model.BlockContentText_Paragraph, "cell"), + }, + Details: fields(map[string]*types.Value{"id": str("root")}), + } + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + + assert.NoError(t, Validate(data), "Marshal must not emit what Validate rejects") + assert.Contains(t, string(data), `"id": "pages_doc_17"`) + assert.Contains(t, string(data), `"id": "pages_doc_12"`) + assert.Contains(t, string(data), "cell", "cell content survives the relabel") + + // and the normalized document round-trips + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + again, err := Marshal(model.SmartBlockType_Page, back, testOptions()) + require.NoError(t, err) + assert.Equal(t, string(data), string(again), "export must be byte-stable (§11)") +} + +// A cell's id is derived, so a table owns every rowId-colId of its grid +// whether or not that cell is materialized (§4, §6.1): validation claims the +// whole grid, and the editor materializes the missing cell at exactly that id +// the first time it is filled. Export has to reserve the same ids, or a +// perfectly legal snapshot — every block id unique — marshals into a document +// Validate rejects. +func TestExport_UnwrittenDerivedCellIdIsReserved(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "root", ChildrenIds: []string{"tbl", "r1-c1"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "tbl", ChildrenIds: []string{"cols", "rows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "cols", ChildrenIds: []string{"c1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: "rows", ChildrenIds: []string{"r1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + {Id: "c1", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + // the (r1,c1) cell is not materialized: the row has no children + {Id: "r1", Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + textBlock("r1-c1", model.BlockContentText_Paragraph, "sibling"), + }, + Details: fields(map[string]*types.Value{"id": str("root")}), + } + // the fixture is not a bad fixture: every stored id in it is unique + seen := map[string]bool{} + for _, b := range snap.Blocks { + require.False(t, seen[b.Id], "fixture id %q is not unique", b.Id) + seen[b.Id] = true + } + + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + assert.NoError(t, Validate(data), "Marshal must not emit what Validate rejects") + + // the table's ids are what the derived id is made of, so they stay; the + // plain block spelling a derived cell id is the one that yields (§4) + assert.Contains(t, string(data), `"id": "c1"`) + assert.Contains(t, string(data), `"id": "r1"`) + assert.Contains(t, string(data), `"id": "r1-c1_2"`) + assert.Contains(t, string(data), "sibling", "the block keeps its content") +} + +// A cell rendered through the string shorthand never reaches blockToJSON, +// which is where the emit-once mark is set (§11). Without the mark, a block +// that is both a cell and a child somewhere else is written twice — once as +// the cell's text and once as a block carrying the derived cell id, which is a +// duplicate id in the output. +// +// The mark has to be READ as well as written, and which of the two failures +// shows up depends on which parent the walk reaches first — so both orders are +// tested. Setting the mark without consulting it covers only the order where +// the cell comes first; reach the other parent first and the cell writes the +// block a SECOND time, and one stored block imports back as two. +func TestExport_StringShorthandCellIsEmittedOnce(t *testing.T) { + // one block, two parents: the row's cell and a child of a top-level block + sharedCellSnapshot := func(rootChildren ...string) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{ + {Id: "root", ChildrenIds: rootChildren, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + {Id: "tbl", ChildrenIds: []string{"cols", "rows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "cols", ChildrenIds: []string{"c1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: "rows", ChildrenIds: []string{"r1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + {Id: "c1", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + {Id: "r1", ChildrenIds: []string{"r1-c1"}, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + {Id: "holder", ChildrenIds: []string{"r1-c1"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "holder"}}}, + textBlock("r1-c1", model.BlockContentText_Paragraph, "shared"), + }, + Details: fields(map[string]*types.Value{"id": str("root")}), + } + } + for name, snap := range map[string]*model.SmartBlockSnapshotBase{ + "the cell is reached first": sharedCellSnapshot("tbl", "holder"), + "the other parent is reached first": sharedCellSnapshot("holder", "tbl"), + } { + t.Run(name, func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + assert.NoError(t, Validate(data), "Marshal must not emit what Validate rejects") + assert.Equal(t, 1, strings.Count(string(data), "shared"), + "the block is emitted once:\n%s", data) + + // and the count the document states is the count that comes back: + // a second emission is a block import has to build, not a phrase + // that happens to appear twice + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + var shared int + for _, b := range back.Blocks { + if t, ok := b.Content.(*model.BlockContentOfText); ok && t.Text.GetText() == "shared" { + shared++ + } + } + assert.Equal(t, 1, shared, "one stored block, one imported block:\n%s", data) + }) + } +} + +// A table's grid belongs to the table whether the ids were authored or +// generated (§6.1). Only the authored half was ever claimed, so a generated +// row or column left every cell id it implies free — free for an authored +// block to be sitting on already, and free for the next generated id to take. +// Either way the document imported to two blocks with one id, a snapshot no +// editor can resolve. +// +// Both fixtures calibrate themselves: they import the table alone to learn +// what the generator will call the row and the column, and only then build the +// collision. Hard-coding the generator's answer would make the test pass +// vacuously the day the number of genId calls changes. +func TestImport_GeneratedGridIsClaimed(t *testing.T) { + tableOnly := `{"version": 2, "id": "p1", "blocks": [ + {"type": "table", "columns": [{}], "rows": [{"cells": ["cell"]}]}, + {"type": "paragraph", "text": "trailing"}]}` + mints := 0 + _, probe, err := Unmarshal([]byte(tableOnly), Options{ + GenerateId: func() string { mints++; return fmt.Sprintf("g%d", mints) }}) + require.NoError(t, err) + var rowId, colId string + for _, b := range probe.Blocks { + switch b.Content.(type) { + case *model.BlockContentOfTableRow: + rowId = b.Id + case *model.BlockContentOfTableColumn: + colId = b.Id + } + } + require.NotEmpty(t, rowId, "the fixture's row id is generated") + require.NotEmpty(t, colId, "the fixture's column id is generated") + derived := rowId + "-" + colId + require.Contains(t, blockIds(t, probe, "p1"), derived, "the cell block is built at the derived id") + require.NotZero(t, mints, "the fixture generates ids") + + noDuplicateIds := func(t *testing.T, snap *model.SmartBlockSnapshotBase) { + t.Helper() + seen := map[string]bool{} + for _, b := range snap.Blocks { + assert.False(t, seen[b.Id], "duplicate block id %q — the generated grid landed on a taken id", b.Id) + seen[b.Id] = true + } + } + + t.Run("an authored block is already sitting on the grid", func(t *testing.T) { + doc := fmt.Sprintf(`{"version": 2, "id": "p1", "blocks": [ + {"type": "paragraph", "id": %q, "text": "authored"}, + {"type": "table", "columns": [{}], "rows": [{"cells": ["cell"]}]}]}`, derived) + require.NoError(t, Validate([]byte(doc)), "the document itself is legal: %s", doc) + + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + noDuplicateIds(t, snap) + // the authored block keeps its id and the generated grid moves off it: + // a derived id has no spelling of its own, so the generated side — the + // only side with a free choice — is the one that yields + assert.Contains(t, blockIds(t, snap, "p1"), derived, "the authored block keeps its id") + assert.NotEqual(t, derived, cellIdOf(t, snap), "the cell moved off the authored id") + }) + + t.Run("a later generated id lands on the grid", func(t *testing.T) { + // the same document and the same generator, except that its LAST + // answer — the trailing paragraph's — is the cell id the table just + // derived. A generator is the caller's (the convert wiring derives ids + // from file paths), so its answers are not the package's to trust. + n := 0 + _, snap, err := Unmarshal([]byte(tableOnly), Options{GenerateId: func() string { + n++ + if n == mints { + return derived + } + return fmt.Sprintf("g%d", n) + }}) + require.NoError(t, err) + noDuplicateIds(t, snap) + assert.Equal(t, derived, cellIdOf(t, snap), + "the table's own cell is unmoved — nothing else claimed its id first") + }) +} + +// cellIdOf returns the id of the single table cell in a snapshot: the one +// child of the one row. +func cellIdOf(t *testing.T, snap *model.SmartBlockSnapshotBase) string { + t.Helper() + for _, b := range snap.Blocks { + if _, ok := b.Content.(*model.BlockContentOfTableRow); ok { + require.Len(t, b.ChildrenIds, 1, "the fixture's row holds one cell") + return b.ChildrenIds[0] + } + } + t.Fatal("no table row in the snapshot") + return "" +} + +// A snapshot's block list is not its block tree: unlinked subtrees survive in +// it, and a table among them is not in the document at all. Its grid of +// derived ids therefore claims nothing — reserving it renamed a block the +// document DOES contain on the authority of one nobody can see, which is +// exactly the rename §9 promises never happens to an id that is already legal. +func TestExport_UnreachableTableReservesNoIds(t *testing.T) { + reachable := []*model.Block{ + {Id: "root", ChildrenIds: []string{"r9-c9"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + textBlock("r9-c9", model.BlockContentText_Paragraph, "in the document"), + } + // an orphan table: nothing links to "orphan", so the emit never arrives + orphan := []*model.Block{ + {Id: "orphan", ChildrenIds: []string{"cols9", "rows9"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "cols9", ChildrenIds: []string{"c9"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: "rows9", ChildrenIds: []string{"r9"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + {Id: "c9", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + {Id: "r9", Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + } + details := fields(map[string]*types.Value{"id": str("root")}) + + withOrphan, err := Marshal(model.SmartBlockType_Page, &model.SmartBlockSnapshotBase{ + Blocks: append(append([]*model.Block{}, reachable...), orphan...), Details: details}, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(withOrphan), `"id": "r9-c9"`, + "the reachable block keeps the id it was authored with:\n%s", withOrphan) + + without, err := Marshal(model.SmartBlockType_Page, &model.SmartBlockSnapshotBase{ + Blocks: reachable, Details: details}, testOptions()) + require.NoError(t, err) + assert.Equal(t, string(without), string(withOrphan), + "a block the document does not contain may not change the document") +} + +// The fragment export's entry point is the caller's subtree[0], not the root +// indexBlocks infers — and an inferred root is what a fragment slice least +// deserves to be judged by: it is "the first block nobody references", which a +// slice carrying its own parent, or any spare entry, moves somewhere else. +// Reserve from the wrong entry point and the table that IS emitted reserves +// nothing, so the plain block on its derived cell id keeps that id and Marshal +// writes a duplicate — a run its own Validate rejects (I1). +func TestMarshalBlockSubtree_ReservesFromTheCallersRoot(t *testing.T) { + subtree := []*model.Block{ + // [0] is what the caller addresses; it holds the table and a plain + // block spelling the (r1,c1) cell id the table derives + {Id: "holder", ChildrenIds: []string{"tbl", "r1-c1"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "holder"}}}, + {Id: "tbl", ChildrenIds: []string{"cols", "rows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + {Id: "cols", ChildrenIds: []string{"c1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}}, + {Id: "rows", ChildrenIds: []string{"r1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}}, + {Id: "c1", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + {Id: "r1", Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + textBlock("r1-c1", model.BlockContentText_Paragraph, "sits on a cell id"), + // a spare entry the emit never visits, and the caller's own parent — + // between them, "the first block nobody references" is neither the + // table's owner nor anything that leads to it + textBlock("spare", model.BlockContentText_Paragraph, "unreferenced"), + {Id: "parent", ChildrenIds: []string{"holder"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "parent"}}}, + } + fragment, err := MarshalBlockSubtree(subtree, Options{}) + require.NoError(t, err) + + // the run is validated the way a fragment is: as the blocks of a document + var env struct { + Blocks json.RawMessage `json:"blocks"` + } + require.NoError(t, json.Unmarshal(fragment, &env)) + doc, err := json.Marshal(map[string]any{"version": FormatVersion, "blocks": env.Blocks}) + require.NoError(t, err) + assert.NoError(t, Validate(doc), "Marshal must not emit what Validate rejects:\n%s", fragment) + assert.Contains(t, string(fragment), `"r1-c1_2"`, "the plain block yields to the grid:\n%s", fragment) +} + +// The mirror of the export rule on the import side: a generated id may not +// land on a cell id the document's table already implies, materialized or not. +func TestImport_GeneratedIdAvoidsUnwrittenCellId(t *testing.T) { + doc := `{"version": 2, "id": "p1", "blocks": [ + {"type": "table", "id": "tbl", "columns": [{"id": "c1"}], "rows": [{"id": "r1"}]}, + {"type": "paragraph", "text": "x"}]}` + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: func() string { return "r1-c1" }}) + require.NoError(t, err) + + ids := map[string]bool{} + for _, b := range snap.Blocks { + assert.False(t, ids[b.Id], "duplicate block id %q", b.Id) + ids[b.Id] = true + } + assert.False(t, ids["r1-c1"], "a generated id took the (r1,c1) cell id") +} + +// Sanitizing is for ids that need it. A generated id that is already a legal +// row/column id keeps its name — nothing renames it, and above all not the +// disambiguation pass, which used to find the id taken by the generator's own +// claim and hand back `_2` for every row and column ever minted. +func TestImport_GeneratedTableInnerIdsKeepTheirName(t *testing.T) { + doc := `{"version": 2, "blocks": [{"type": "table", + "columns": [{}, {}], "rows": [{"cells": ["a", "b"]}, {"cells": ["c", "d"]}]}]}` + + t.Run("legal generated ids are untouched", func(t *testing.T) { + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + var inner []string + for _, b := range snap.Blocks { + switch b.Content.(type) { + case *model.BlockContentOfTableRow, *model.BlockContentOfTableColumn: + inner = append(inner, b.Id) + } + } + require.Len(t, inner, 4) + for _, id := range inner { + assert.Regexp(t, `^g\d+$`, id, "a generated id that needs no sanitizing keeps its name") + } + }) + + t.Run("the default generator's shape survives", func(t *testing.T) { + _, snap, err := Unmarshal([]byte(doc), Options{}) + require.NoError(t, err) + for _, b := range snap.Blocks { + switch b.Content.(type) { + case *model.BlockContentOfTableRow, *model.BlockContentOfTableColumn: + assert.Regexp(t, `^[0-9a-f]{24}$`, b.Id, "24 hex chars, like every other minted id") + } + } + }) + + t.Run("a sanitized id takes the sanitized name, nothing more", func(t *testing.T) { + // a dashed generator (the convert wiring's shape): every id sanitizes + // to a name nothing else holds, so the suffix pass has no work to do + n := 0 + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: func() string { + n++ + return fmt.Sprintf("x-%d", n) + }}) + require.NoError(t, err) + seen := map[string]bool{} + for _, b := range snap.Blocks { + switch b.Content.(type) { + case *model.BlockContentOfTableRow, *model.BlockContentOfTableColumn: + assert.False(t, seen[b.Id], "duplicate table inner id %q", b.Id) + seen[b.Id] = true + assert.Regexp(t, `^x_\d+$`, b.Id, "sanitized, and not suffixed on top of it") + } + } + require.Len(t, seen, 4) + }) +} diff --git a/pkg/lib/anyblockjson/templatelegend_test.go b/pkg/lib/anyblockjson/templatelegend_test.go new file mode 100644 index 0000000000..729a8811f2 --- /dev/null +++ b/pkg/lib/anyblockjson/templatelegend_test.go @@ -0,0 +1,91 @@ +package anyblockjson + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// This file used to pin a refusal: v0.22 made `kind` the sole authority on +// what a template is, and a document that carried the meaning in `type` +// alone — `{"type": "template"}` with no `kind` — was refused rather than +// migrated, because that one shape was well-formed under BOTH readings and +// would otherwise have imported as an ordinary page with nothing anywhere +// saying so. The refusal even read the document's own type_internal_keys +// legend, so a document that renamed the template spelling could not slip +// past it. +// +// The freeze deleted it (§15 #9). Every document written under the old +// reading declares version 1, and checkVersion refuses that outright before +// any semantic check runs — one gate, one verdict, instead of a byte +// comparison standing in for a version marker the format did not have. +// +// What is left is the rule the refusal was standing in front of, and these +// cases pin it: the kind decides, the legend decides only the TYPE, and the +// version gate is what answers for a pre-freeze document. +func TestValidate_ThePreFreezeTemplateShapeIsNoLongerSpecialCased(t *testing.T) { + t.Run("a kindless document is a page, whatever its type spells", func(t *testing.T) { + // under the old reading this WAS a template, and the deleted refusal + // is what said so. At version 2 the kind is absent, so it is a page + // whose object type is the one the term names — no refusal, and + // nothing silent about it either: the type is spelled right there + doc := []byte(`{"version": 2, "type": "template"}`) + + require.NoError(t, Validate(doc)) + sbType, snap, err := Unmarshal(doc, Options{}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + assert.Equal(t, []string{"ot-template"}, snap.GetObjectTypes()) + }) + + t.Run("and so is one whose legend renames the template spelling", func(t *testing.T) { + // `tpl` resolves to the stored key `template` through the document's + // own legend. The deleted refusal read that legend; the kind gate + // does not need to, because it reads a field no chain touches + doc := []byte(`{"version": 2, "type_internal_keys": {"tpl": "template"}, "type": "tpl"}`) + + require.NoError(t, Validate(doc)) + sbType, snap, err := Unmarshal(doc, Options{}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + assert.Equal(t, []string{"ot-template"}, snap.GetObjectTypes()) + }) + + t.Run("a legend that rebinds the spelling away binds the type it names", func(t *testing.T) { + // the mirror case, and the one that never changed: the raw spelling + // IS `template`, but the document's own legend binds it to `custom1` + doc := []byte(`{"version": 2, "type_internal_keys": {"template": "custom1"}, "type": "template"}`) + + require.NoError(t, Validate(doc)) + sbType, snap, err := Unmarshal(doc, Options{}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + assert.Equal(t, []string{"ot-custom1"}, snap.GetObjectTypes(), + "and it binds to the type the legend names") + }) + + t.Run("the version gate is what answers for a pre-freeze document", func(t *testing.T) { + // the positive control the deleted refusal used to be: a document + // written under the old reading is still refused, one version marker + // earlier and for the whole grammar rather than this one shape + err := Validate([]byte(`{"version": 1, "type": "template"}`)) + require.Error(t, err) + var ve *ValidationError + require.ErrorAs(t, err, &ve) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/version", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, "pre-freeze") + assert.False(t, ve.NewerFormat, "a draft is not a newer format") + + _, _, uerr := Unmarshal([]byte(`{"version": 1, "type": "template"}`), Options{}) + require.Error(t, uerr, "Validate and Unmarshal agree (§11 I2)") + }) + + t.Run("a canonical template still validates", func(t *testing.T) { + require.NoError(t, Validate([]byte( + `{"version": 2, "kind": "template", "type": "template", "template_for": "page"}`))) + }) +} diff --git a/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/index.json b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/index.json new file mode 100644 index 0000000000..6e31139ce7 --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/index.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/authoring/index.schema.json", + "version": 2, + "name": "Habit Tracker", + "description": "Small habits, tracked without ceremony.", + "icon": { "format": "emoji", "emoji": "🌱" }, + "entrypoint": "page-start", + "widgets": [ + { "target": "page-start" }, + { "target": "type-habit", "layout": "view", "limit": 6 }, + { "target": "habit-weekly-review", "card_style": "card" }, + { "target": "_favorite", "layout": "compact_list", "limit": 4 } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/morning-run.json b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/morning-run.json new file mode 100644 index 0000000000..648f8467f1 --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/morning-run.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/authoring/object.schema.json", + "version": 2, + "id": "habit-morning-run", + "type": "habit", + "icon": { "format": "emoji", "emoji": "🏃" }, + "properties": { + "Name": "Morning run", + "Frequency": [ "Daily" ], + "Streak": 12, + "Last done": "2026-08-24T07:30:00Z" + }, + "blocks": [ + { "type": "paragraph", "text": "Twenty minutes before breakfast. Shoes by the door the night before — the run starts when the alarm goes off, not when it feels right." }, + { "type": "toggle", "text": "Routes" }, + { "indent": 1, "type": "bulleted_list_item", "text": "Park loop — 2.8 km, flat" }, + { "indent": 1, "type": "bulleted_list_item", "text": "River path — 4 km, one hill" } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/start.json b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/start.json new file mode 100644 index 0000000000..aaae829994 --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/start.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/authoring/object.schema.json", + "version": 2, + "id": "page-start", + "type": "Page", + "icon": { "format": "emoji", "emoji": "👋" }, + "properties": { + "Name": "Start here", + "Description": "How this tracker works.", + "Favorited": true + }, + "blocks": [ + { "type": "paragraph", "text": "Add a **Habit** for anything you want to do regularly, and bump its *streak* every time you follow through." }, + { "type": "bulleted_list_item", "text": "Set the frequency: Daily or Weekly." }, + { "type": "bulleted_list_item", "text": "Update **Streak** and **Last done** when you finish." }, + { "type": "callout", "icon": { "format": "emoji", "emoji": "💡" }, "text": "Broke a streak? Set it to zero and move on — the [By frequency board](anytype://object?objectId=type-habit) shows what needs attention today." }, + { "type": "divider" }, + { "type": "paragraph", "text": "Two habits are already set up:" }, + { "type": "link", "object_id": "habit-morning-run", "card_style": "card" }, + { "type": "link", "object_id": "habit-weekly-review", "card_style": "card" } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/weekly-review.json b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/weekly-review.json new file mode 100644 index 0000000000..f6743d064b --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/objects/weekly-review.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/authoring/object.schema.json", + "version": 2, + "id": "habit-weekly-review", + "type": "habit", + "icon": { "format": "emoji", "emoji": "🗓️" }, + "properties": { + "Name": "Weekly review", + "Frequency": [ "Weekly" ], + "Streak": 3, + "Last done": "2026-08-22T18:00:00Z" + }, + "blocks": [ + { "type": "paragraph", "text": "Half an hour on Friday evening: close the week before it closes you." }, + { "type": "checkbox", "text": "Clear the inbox to zero" }, + { "type": "checkbox", "text": "Review next week's calendar" }, + { "type": "checkbox", "text": "Pick the one thing that must happen" } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/properties.json b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/properties.json new file mode 100644 index 0000000000..48583f29cc --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/properties.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/authoring/properties.schema.json", + "version": 2, + "properties": [ + { + "name": "Frequency", + "format": "select", + "options": [ + { "name": "Daily", "color": "lime" }, + { "name": "Weekly", "color": "blue" } + ], + "description": "How often the habit repeats." + }, + { + "name": "Streak", + "format": "number", + "description": "Days in a row, so far." + }, + { + "name": "Last done", + "format": "date", + "description": "When you last did it." + } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/types/habit.json b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/types/habit.json new file mode 100644 index 0000000000..7e0e9255d5 --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/authoring/habit_tracker/types/habit.json @@ -0,0 +1,65 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/authoring/object.schema.json", + "version": 2, + "kind": "object_type", + "id": "type-habit", + "internal_key": "habit", + "icon": { "format": "icon", "name": "repeat", "color": "teal" }, + "properties": { + "Name": "Habit", + "Description": "One habit you are building." + }, + "type_settings": { + "layout": "basic", + "plural_name": "Habits", + "default_view": "table", + "property_definitions": [ + { "property": "Frequency", "section": "featured" }, + { "property": "Streak", "section": "featured" }, + { "property": "Last done" } + ] + }, + "blocks": [ + { + "type": "dataview", + "properties": [ + { "property": "Name", "format": "text" }, + { "property": "Frequency", "format": "select" }, + { "property": "Streak", "format": "number" }, + { "property": "Last done", "format": "date" } + ], + "views": [ + { + "name": "All habits", + "sorts": [ { "property": "Streak", "direction": "desc" } ], + "columns": [ + { "property": "Name" }, + { "property": "Frequency" }, + { "property": "Streak" }, + { "property": "Last done" } + ] + }, + { + "type": "kanban", + "name": "By frequency", + "group_by": "Frequency", + "columns": [ + { "property": "Name" }, + { "property": "Streak" } + ] + }, + { + "name": "Daily", + "filters": [ + { "property": "Frequency", "condition": "in", "value": [ "Daily" ] } + ], + "columns": [ + { "property": "Name" }, + { "property": "Streak" }, + { "property": "Last done" } + ] + } + ] + } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/containers.json b/pkg/lib/anyblockjson/testdata/containers.json new file mode 100644 index 0000000000..504fe4a4ad --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/containers.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "version": 2, + "id": "obj1", + "blocks": [ + { + "id": "h1", + "type": "heading_1", + "text": "Heading" + }, + { + "id": "p1", + "type": "paragraph", + "text": "one" + }, + { + "id": "p2", + "type": "paragraph", + "text": "two" + }, + { + "id": "p3", + "type": "paragraph", + "text": "three" + }, + { + "id": "row1", + "type": "row" + }, + { + "indent": 1, + "id": "col1", + "type": "column" + }, + { + "indent": 2, + "id": "p5", + "type": "paragraph", + "text": "in a column" + }, + { + "id": "p4", + "type": "paragraph", + "text": "four" + }, + { + "id": "table1", + "type": "table", + "columns": [ + { + "id": "c1" + } + ], + "rows": [ + { + "id": "r1", + "cells": [ + [ + { + "type": "paragraph", + "text": "cell" + }, + { + "indent": 1, + "id": "p6", + "type": "paragraph", + "text": "under the cell's container" + } + ] + ] + } + ] + } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/rich.json b/pkg/lib/anyblockjson/testdata/rich.json new file mode 100644 index 0000000000..c8585621b7 --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/rich.json @@ -0,0 +1,237 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "version": 2, + "id": "bafyreiobject", + "type": "Page", + "icon": { + "format": "emoji", + "emoji": "🔥" + }, + "properties": { + "Name": "Project Phoenix", + "Description": "The subtitle", + "Assignee": [ + "bafyreiroman" + ], + "Creation date": "2025-07-06T08:44:05Z", + "customDate": "2025-07-06T08:44:05Z", + "customStatus": [ + "In progress" + ] + }, + "property_internal_keys": { + "customDate": "customDate", + "customStatus": "customStatus" + }, + "option_ids": { + "customStatus": { + "Done": "opt2", + "In progress": "opt1" + } + }, + "blocks": [ + { + "id": "b1", + "type": "heading_2", + "text": "Goals" + }, + { + "id": "b2", + "type": "paragraph", + "text": "Ship the **new export** with Roman" + }, + { + "id": "b3", + "type": "bulleted_list_item", + "text": "Nested item" + }, + { + "id": "b5", + "type": "checkbox", + "text": "Draft spec" + }, + { + "id": "b6", + "type": "code", + "language": "go", + "text": "func main() {\n\tprintln(\"hi\")\n}" + }, + { + "id": "b7", + "type": "divider", + "style": "dots" + }, + { + "id": "b8", + "type": "image", + "object_id": "bafyreiimage", + "name": "cat.png", + "mime_type": "image/png", + "size": 2048, + "added_at": "2025-07-06T08:44:05Z" + }, + { + "id": "b9", + "type": "bookmark", + "url": "https://anytype.io", + "object_id": "bafyreibookmark" + }, + { + "id": "row1", + "type": "row" + }, + { + "indent": 1, + "id": "col1", + "type": "column" + }, + { + "indent": 2, + "id": "b11", + "type": "paragraph", + "text": "left" + }, + { + "indent": 1, + "id": "col2", + "type": "column" + }, + { + "indent": 2, + "id": "b12", + "type": "paragraph", + "text": "right" + }, + { + "id": "table1", + "type": "table", + "columns": [ + { + "id": "c1" + }, + { + "id": "c2", + "width": 120 + } + ], + "rows": [ + { + "id": "r1", + "is_header": true, + "cells": [ + "Name", + "Status" + ] + }, + { + "id": "r2", + "cells": [ + null, + { + "type": "checkbox", + "checked": true, + "text": "done" + } + ] + } + ] + }, + { + "id": "b10", + "type": "embed", + "processor": "mermaid", + "text": "graph TD; A-->B" + }, + { + "id": "dv1", + "type": "dataview", + "object_id": "bafyreitasks", + "properties": [ + { + "property": "Name", + "format": "text" + }, + { + "property": "customStatus", + "format": "select" + }, + { + "property": "Due date", + "format": "date" + } + ], + "views": [ + { + "id": "v1", + "type": "kanban", + "name": "By status", + "group_by": "customStatus", + "sorts": [ + { + "property": "Due date", + "empty_placement": "end", + "id": "s1" + } + ], + "filters": [ + { + "property": "Due date", + "condition": "less", + "date_preset": "current_week", + "id": "f1" + }, + { + "operator": "or", + "filters": [ + { + "property": "customStatus", + "condition": "in", + "value": [ + "In progress", + "Done" + ], + "id": "f2" + }, + { + "property": "Done", + "condition": "empty", + "id": "f3" + } + ] + } + ], + "columns": [ + { + "property": "Name" + }, + { + "property": "Due date", + "hidden": true, + "width": 120, + "aggregation": "count_distinct", + "align": "right" + } + ], + "groups": [ + { + "id": "g1", + "background_color": "red" + }, + { + "id": "g2", + "hidden": true + } + ], + "object_orders": [ + { + "group_id": "g1", + "object_ids": [ + "bafyreitask1" + ] + } + ] + } + ] + } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/rich_compact_ids.json b/pkg/lib/anyblockjson/testdata/rich_compact_ids.json new file mode 100644 index 0000000000..c8585621b7 --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/rich_compact_ids.json @@ -0,0 +1,237 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "version": 2, + "id": "bafyreiobject", + "type": "Page", + "icon": { + "format": "emoji", + "emoji": "🔥" + }, + "properties": { + "Name": "Project Phoenix", + "Description": "The subtitle", + "Assignee": [ + "bafyreiroman" + ], + "Creation date": "2025-07-06T08:44:05Z", + "customDate": "2025-07-06T08:44:05Z", + "customStatus": [ + "In progress" + ] + }, + "property_internal_keys": { + "customDate": "customDate", + "customStatus": "customStatus" + }, + "option_ids": { + "customStatus": { + "Done": "opt2", + "In progress": "opt1" + } + }, + "blocks": [ + { + "id": "b1", + "type": "heading_2", + "text": "Goals" + }, + { + "id": "b2", + "type": "paragraph", + "text": "Ship the **new export** with Roman" + }, + { + "id": "b3", + "type": "bulleted_list_item", + "text": "Nested item" + }, + { + "id": "b5", + "type": "checkbox", + "text": "Draft spec" + }, + { + "id": "b6", + "type": "code", + "language": "go", + "text": "func main() {\n\tprintln(\"hi\")\n}" + }, + { + "id": "b7", + "type": "divider", + "style": "dots" + }, + { + "id": "b8", + "type": "image", + "object_id": "bafyreiimage", + "name": "cat.png", + "mime_type": "image/png", + "size": 2048, + "added_at": "2025-07-06T08:44:05Z" + }, + { + "id": "b9", + "type": "bookmark", + "url": "https://anytype.io", + "object_id": "bafyreibookmark" + }, + { + "id": "row1", + "type": "row" + }, + { + "indent": 1, + "id": "col1", + "type": "column" + }, + { + "indent": 2, + "id": "b11", + "type": "paragraph", + "text": "left" + }, + { + "indent": 1, + "id": "col2", + "type": "column" + }, + { + "indent": 2, + "id": "b12", + "type": "paragraph", + "text": "right" + }, + { + "id": "table1", + "type": "table", + "columns": [ + { + "id": "c1" + }, + { + "id": "c2", + "width": 120 + } + ], + "rows": [ + { + "id": "r1", + "is_header": true, + "cells": [ + "Name", + "Status" + ] + }, + { + "id": "r2", + "cells": [ + null, + { + "type": "checkbox", + "checked": true, + "text": "done" + } + ] + } + ] + }, + { + "id": "b10", + "type": "embed", + "processor": "mermaid", + "text": "graph TD; A-->B" + }, + { + "id": "dv1", + "type": "dataview", + "object_id": "bafyreitasks", + "properties": [ + { + "property": "Name", + "format": "text" + }, + { + "property": "customStatus", + "format": "select" + }, + { + "property": "Due date", + "format": "date" + } + ], + "views": [ + { + "id": "v1", + "type": "kanban", + "name": "By status", + "group_by": "customStatus", + "sorts": [ + { + "property": "Due date", + "empty_placement": "end", + "id": "s1" + } + ], + "filters": [ + { + "property": "Due date", + "condition": "less", + "date_preset": "current_week", + "id": "f1" + }, + { + "operator": "or", + "filters": [ + { + "property": "customStatus", + "condition": "in", + "value": [ + "In progress", + "Done" + ], + "id": "f2" + }, + { + "property": "Done", + "condition": "empty", + "id": "f3" + } + ] + } + ], + "columns": [ + { + "property": "Name" + }, + { + "property": "Due date", + "hidden": true, + "width": 120, + "aggregation": "count_distinct", + "align": "right" + } + ], + "groups": [ + { + "id": "g1", + "background_color": "red" + }, + { + "id": "g2", + "hidden": true + } + ], + "object_orders": [ + { + "group_id": "g1", + "object_ids": [ + "bafyreitask1" + ] + } + ] + } + ] + } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/rich_compact_omit.json b/pkg/lib/anyblockjson/testdata/rich_compact_omit.json new file mode 100644 index 0000000000..85f1911416 --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/rich_compact_omit.json @@ -0,0 +1,187 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "version": 2, + "id": "bafyreiobject", + "type": "Page", + "icon": { + "format": "emoji", + "emoji": "🔥" + }, + "properties": { + "Name": "Project Phoenix", + "Description": "The subtitle", + "Assignee": [ + "bafyreiroman" + ], + "Creation date": "2025-07-06T08:44:05Z", + "customDate": "2025-07-06T08:44:05Z", + "customStatus": [ + "In progress" + ] + }, + "property_internal_keys": { + "customDate": "customDate", + "customStatus": "customStatus" + }, + "blocks": [ + { + "type": "heading_2", + "text": "Goals" + }, + { + "type": "paragraph", + "text": "Ship the **new export** with Roman" + }, + { + "type": "bulleted_list_item", + "text": "Nested item" + }, + { + "type": "checkbox", + "text": "Draft spec" + }, + { + "type": "code", + "language": "go", + "text": "func main() {\n\tprintln(\"hi\")\n}" + }, + { + "type": "divider", + "style": "dots" + }, + { + "type": "image", + "object_id": "bafyreiimage", + "name": "cat.png", + "mime_type": "image/png", + "size": 2048, + "added_at": "2025-07-06T08:44:05Z" + }, + { + "type": "bookmark", + "url": "https://anytype.io", + "object_id": "bafyreibookmark" + }, + { + "type": "row" + }, + { + "indent": 1, + "type": "column" + }, + { + "indent": 2, + "type": "paragraph", + "text": "left" + }, + { + "indent": 1, + "type": "column" + }, + { + "indent": 2, + "type": "paragraph", + "text": "right" + }, + { + "type": "table", + "columns": [ + {}, + { + "width": 120 + } + ], + "rows": [ + { + "is_header": true, + "cells": [ + "Name", + "Status" + ] + }, + { + "cells": [ + null, + { + "type": "checkbox", + "checked": true, + "text": "done" + } + ] + } + ] + }, + { + "type": "embed", + "processor": "mermaid", + "text": "graph TD; A-->B" + }, + { + "type": "dataview", + "object_id": "bafyreitasks", + "properties": [ + { + "property": "Name", + "format": "text" + }, + { + "property": "customStatus", + "format": "select" + }, + { + "property": "Due date", + "format": "date" + } + ], + "views": [ + { + "type": "kanban", + "name": "By status", + "group_by": "customStatus", + "sorts": [ + { + "property": "Due date", + "empty_placement": "end" + } + ], + "filters": [ + { + "property": "Due date", + "condition": "less", + "date_preset": "current_week" + }, + { + "operator": "or", + "filters": [ + { + "property": "customStatus", + "condition": "in", + "value": [ + "In progress", + "Done" + ] + }, + { + "property": "Done", + "condition": "empty" + } + ] + } + ], + "columns": [ + { + "property": "Name" + }, + { + "property": "Due date", + "hidden": true, + "width": 120, + "aggregation": "count_distinct", + "align": "right" + } + ] + } + ] + } + ] +} diff --git a/pkg/lib/anyblockjson/testdata/rich_omit_ids.json b/pkg/lib/anyblockjson/testdata/rich_omit_ids.json new file mode 100644 index 0000000000..85f1911416 --- /dev/null +++ b/pkg/lib/anyblockjson/testdata/rich_omit_ids.json @@ -0,0 +1,187 @@ +{ + "$schema": "https://schemas.anytype.io/anyblock/2/object.schema.json", + "version": 2, + "id": "bafyreiobject", + "type": "Page", + "icon": { + "format": "emoji", + "emoji": "🔥" + }, + "properties": { + "Name": "Project Phoenix", + "Description": "The subtitle", + "Assignee": [ + "bafyreiroman" + ], + "Creation date": "2025-07-06T08:44:05Z", + "customDate": "2025-07-06T08:44:05Z", + "customStatus": [ + "In progress" + ] + }, + "property_internal_keys": { + "customDate": "customDate", + "customStatus": "customStatus" + }, + "blocks": [ + { + "type": "heading_2", + "text": "Goals" + }, + { + "type": "paragraph", + "text": "Ship the **new export** with Roman" + }, + { + "type": "bulleted_list_item", + "text": "Nested item" + }, + { + "type": "checkbox", + "text": "Draft spec" + }, + { + "type": "code", + "language": "go", + "text": "func main() {\n\tprintln(\"hi\")\n}" + }, + { + "type": "divider", + "style": "dots" + }, + { + "type": "image", + "object_id": "bafyreiimage", + "name": "cat.png", + "mime_type": "image/png", + "size": 2048, + "added_at": "2025-07-06T08:44:05Z" + }, + { + "type": "bookmark", + "url": "https://anytype.io", + "object_id": "bafyreibookmark" + }, + { + "type": "row" + }, + { + "indent": 1, + "type": "column" + }, + { + "indent": 2, + "type": "paragraph", + "text": "left" + }, + { + "indent": 1, + "type": "column" + }, + { + "indent": 2, + "type": "paragraph", + "text": "right" + }, + { + "type": "table", + "columns": [ + {}, + { + "width": 120 + } + ], + "rows": [ + { + "is_header": true, + "cells": [ + "Name", + "Status" + ] + }, + { + "cells": [ + null, + { + "type": "checkbox", + "checked": true, + "text": "done" + } + ] + } + ] + }, + { + "type": "embed", + "processor": "mermaid", + "text": "graph TD; A-->B" + }, + { + "type": "dataview", + "object_id": "bafyreitasks", + "properties": [ + { + "property": "Name", + "format": "text" + }, + { + "property": "customStatus", + "format": "select" + }, + { + "property": "Due date", + "format": "date" + } + ], + "views": [ + { + "type": "kanban", + "name": "By status", + "group_by": "customStatus", + "sorts": [ + { + "property": "Due date", + "empty_placement": "end" + } + ], + "filters": [ + { + "property": "Due date", + "condition": "less", + "date_preset": "current_week" + }, + { + "operator": "or", + "filters": [ + { + "property": "customStatus", + "condition": "in", + "value": [ + "In progress", + "Done" + ] + }, + { + "property": "Done", + "condition": "empty" + } + ] + } + ], + "columns": [ + { + "property": "Name" + }, + { + "property": "Due date", + "hidden": true, + "width": 120, + "aggregation": "count_distinct", + "align": "right" + } + ] + } + ] + } + ] +} diff --git a/pkg/lib/anyblockjson/transient_test.go b/pkg/lib/anyblockjson/transient_test.go new file mode 100644 index 0000000000..57447a5a25 --- /dev/null +++ b/pkg/lib/anyblockjson/transient_test.go @@ -0,0 +1,410 @@ +package anyblockjson + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// transientProperties describe the MOMENT an object was written rather than the +// object: internalFlags carries editor state ("this object was just created, +// offer the type picker"), which a restored object is never in. Export drops +// them; import drops them too, silently, because a document carrying one is +// stale rather than wrong. +// +// These can only fail if a transient key starts reaching the snapshot or starts +// being refused: each asserts the RESULTING DETAILS, not merely that the +// document validates, so a rule that stopped firing would have to keep both +// the acceptance and the absence to pass. +func TestTransientProperties_DroppedNotRefused(t *testing.T) { + for name, doc := range map[string]string{ + "empty, the shape 18,647 real objects carry": `{"version": 2, "properties": {"internal_flags": []}}`, + "populated": `{"version": 2, "properties": {"internal_flags": ["editor_select_type"]}}`, + } { + t.Run(name, func(t *testing.T) { + require.NoError(t, Validate([]byte(doc)), + "a stale export must still import: transient state is dropped, not refused") + + _, snap, err := Unmarshal([]byte(doc), Options{}) + require.NoError(t, err, "Validate and Unmarshal agree (§11 I2)") + assert.NotContains(t, snap.GetDetails().GetFields(), "internalFlags", + "and it must not reach the snapshot") + }) + } + + t.Run("a merge-resolution vector is still REFUSED, not dropped", func(t *testing.T) { + // the control that keeps the exemption honest: neverWritableProperties + // aims a document at an object it did not create, and stays an error. + // The spelling that RESOLVES onto the vector is its stored key + // (verbatim-first); the old derived slug resolves nothing at all — + // a denied key's fold class answers nothing — so it is an ordinary + // custom key that lands on no vector, which the second line pins. + require.Error(t, Validate([]byte(`{"version": 2, "properties": {"oldAnytypeID": "x"}}`))) + _, snap, err := Unmarshal([]byte(`{"version": 2, "properties": {"old_anytype_id": "x"}}`), Options{}) + require.NoError(t, err) + assert.NotContains(t, snap.GetDetails().GetFields(), "oldAnytypeID") + }) + + t.Run("an ordinary property still lands", func(t *testing.T) { + _, snap, err := Unmarshal([]byte(`{"version": 2, "properties": {"name": "keep me"}}`), Options{}) + require.NoError(t, err) + assert.Equal(t, "keep me", snap.GetDetails().GetFields()["name"].GetStringValue()) + }) +} + +// Export's half: a snapshot carrying transient state must not write it. +func TestTransientProperties_NeverExported(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), + "name": str("Real"), + "internalFlags": {Kind: &types.Value_ListValue{ListValue: &types.ListValue{}}}, + }), + ObjectTypes: []string{"ot-page"}, + } + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + require.NoError(t, err) + assert.NotContains(t, string(data), "internal_flags", + "transient state describes the moment, not the object (§3)") + assert.Contains(t, string(data), `"Name"`, "and the rest of the object is untouched") +} + +// Whether a transient key is a BUNDLED relation is a per-key verdict, and +// this pins each one so a change fires here instead of silently changing +// what gets eaten. +// +// The two justifications are different, and only one of them tolerates a +// bundled key. `internalFlags` IS a bundled relation and is stripped anyway, +// because what it holds is editor state — "this object was just created, +// offer the type picker" — which a restored object is never in. The +// analytics triple is stripped for the opposite reason: nothing defines +// those keys at all, so no reader can name them, give them a format, or act +// on them. If a bundled relation ever takes one of those three spellings +// they stop being nameless, the justification evaporates, and dropping them +// would delete real schema shipped with every reader. +// +// How this can fail: add a bundled relation named `data`, `isNew` or +// `layoutFormat`; remove the bundled `internalFlags`; or add a key to +// transientProperties without deciding which case it is. +func TestTransientProperties_BundledVerdictPerKey(t *testing.T) { + want := map[string]bool{ + "internalFlags": true, // bundled, and stripped regardless: editor state + "data": false, // not a relation at all — the whole justification + "isNew": false, + "layoutFormat": false, + // the source space's live session: bundled every one, and stripped + // for a third reason again — they belong to the space a bundle came + // FROM, and three of them are secrets + "spaceInviteFileKey": true, + "spaceInviteGuestFileKey": true, + "oneToOneRequestMetadataKey": true, + "spaceInviteFileCid": true, + "spaceInviteGuestFileCid": true, + "spaceInvitePermissions": true, + "spaceInviteType": true, + "spaceInviteHeldByOwner": true, + "oneToOneInboxSentStatus": true, + "analyticsSpaceId": true, + // deprecated space details + "spaceDashboardId": true, + "spaceUxType": true, + "hasChat": true, + // deprecated: the type owns which properties an instance features + "featuredRelations": true, + // the file machinery's per-device answers: bundled both, and stripped + // because they are the moment's sync/index state of the device that + // exported — the class fileSyncStatus was always in, via + // bundle.LocalAndDerivedRelationKeys + "fileBackupStatus": true, + "fileIndexingStatus": true, + // the file's variant machinery. The first is a SECRET — the + // per-variant encryption keys — and the API layer already refuses to + // emit all seven "so a future change cannot accidentally leak file + // keys / CIDs". No import path reads any of them; a restored file is + // re-indexed and gets its own. + "fileVariantKeys": true, + "fileVariantIds": true, + "fileVariantChecksums": true, + "fileVariantMills": true, + "fileVariantOptions": true, + "fileVariantPaths": true, + "fileVariantWidths": true, + // the file's own content addresses, the last two of the API's + // refusal list; fileExt and fileMimeType stay, describing the file + // rather than addressing it + "fileId": true, + "fileSourceChecksum": true, + } + assert.Equal(t, len(want), len(transientProperties), + "every transient key owes a verdict here — a new one must say which case it is") + for key, why := range transientProperties { + t.Run(key, func(t *testing.T) { + verdict, listed := want[key] + require.Truef(t, listed, "%q was added to transientProperties with no bundled verdict (%s)", key, why) + assert.Equalf(t, verdict, bundle.HasRelation(domain.RelationKey(key)), + "%q changed sides: a nameless key that became bundled is real schema now, "+ + "and dropping it would delete it (%s)", key, why) + }) + } +} + +// The analytics triple: 35 type objects across 7 spaces carry +// `data: {"route":"SettingsSpace"}`, `isNew: true`, `layoutFormat: 0` — the +// client's analytics route context persisted onto the object instead of +// sent as an event. `data` is a MAP-shaped value that no relation defines, +// so no reader can name it, give it a format, or act on it. +// +// How this can fail: drop any of the three from transientProperties and its +// value reaches the snapshot. +func TestTransientProperties_TheAnalyticsTripleIsDropped(t *testing.T) { + // given the exact shape those 35 objects carry + doc := []byte(`{"version": 2, "kind": "object_type", "internal_key": "use_case", + "properties": {"name": "Use Case", "data": {"route": "SettingsSpace"}, + "isNew": true, "layoutFormat": 0}}`) + + // when + require.NoError(t, Validate(doc), "a stale export still imports") + _, snap, err := Unmarshal(doc, Options{}) + require.NoError(t, err, "Validate and Unmarshal agree (§11 I2)") + + // then + for _, key := range []string{"data", "isNew", "layoutFormat"} { + assert.NotContains(t, snap.GetDetails().GetFields(), key, + "%q describes the click that made the object, not the object", key) + } + assert.Contains(t, snap.GetDetails().GetFields(), "name", "the object itself survives") +} + +// The file machinery's per-device answers do not travel. Every one of the +// 10,248 file objects in a 28,604-document corpus carried both keys — +// `file_backup_status` as Synced(4) on 10,246 and Queued(5) on 2, +// `file_indexing_status` as Indexed(1) on ALL of them, one distinct value +// across the whole corpus. Both describe what THIS device's sync and index +// machinery last observed, which the destination's machinery determines for +// itself — and `fileIndexingStatus` is actively harmful on import: the file +// indexer queues exactly the file objects whose status is not Indexed +// (core/files/fileobject/fileindex.go), so an imported Indexed tells it the +// restored file needs no indexing. +// +// How this can fail: drop either key from transientProperties and the value +// reaches the snapshot — and, on the export side, the wire. +func TestTransientProperties_FileStatusDoesNotTravel(t *testing.T) { + t.Run("import drops, not refuses", func(t *testing.T) { + // given the exact shape all 10,248 corpus file objects carry + doc := []byte(`{"version": 2, "properties": {"name": "photo.png", + "file_backup_status": 4, "file_indexing_status": 1}}`) + + // when + require.NoError(t, Validate(doc), "a stale export still imports") + _, snap, err := Unmarshal(doc, Options{}) + require.NoError(t, err, "Validate and Unmarshal agree (§11 I2)") + + // then + for _, key := range []string{"fileBackupStatus", "fileIndexingStatus"} { + assert.NotContains(t, snap.GetDetails().GetFields(), key, + "%q is the exporting device's answer, not a fact about the file", key) + } + assert.Contains(t, snap.GetDetails().GetFields(), "name", "the file object itself survives") + }) + + t.Run("export strips", func(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "f1", Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("f1"), + "name": str("photo.png"), + "fileBackupStatus": num(4), + "fileIndexingStatus": num(1), + }), + } + data, err := Marshal(model.SmartBlockType_FileObject, snap, Options{}) + require.NoError(t, err) + assert.NotContains(t, string(data), "file_backup_status") + assert.NotContains(t, string(data), "file_indexing_status") + assert.Contains(t, string(data), `"Name"`, "and the rest of the object is untouched") + }) +} + +// A bundle is a SHAREABLE artifact — a use case, a template, a backup +// someone sends on — and it was carrying the source space's invite +// encryption keys. `spaceInviteFileKey` is, in the bundled table's own +// words, the "encoded encryption key of invite file for current space". +// +// Measured before the rule: of 77 exported spaces, 74 carried at least one +// of these, 35 carried the invite key, 31 a participant's request-metadata +// key, and 50 carried `analyticsSpaceId` — a stable per-space tracking +// identifier. All ten occur on the space's own document and nowhere else in +// 38,070 corpus documents. +// +// None of them is a fact about any object in the bundle: a restored space +// mints its own invites and its own analytics identity. +// +// How this can fail: drop any of these from transientProperties and the +// value reaches the snapshot — and, on the export side, the wire. +func TestTransientProperties_ASpacesSecretsDoNotTravel(t *testing.T) { + secrets := map[string]string{ + "space_invite_file_key": `"CTSVcbZvejUEhSziyp1c5oFtQaYg"`, + "space_invite_guest_file_key": `"AUhKdNbK3mZcq6taS2qT32Lc92FPM"`, + "one_to_one_request_metadata_key": `"CAISIFALZJQnpN1fVts0VW0oBsKqv"`, + "analytics_space_id": `"3817ea69-b7fa-4d93-ad6d-1b9a8"`, + } + stored := map[string]string{ + "space_invite_file_key": "spaceInviteFileKey", + "space_invite_guest_file_key": "spaceInviteGuestFileKey", + "one_to_one_request_metadata_key": "oneToOneRequestMetadataKey", + "analytics_space_id": "analyticsSpaceId", + } + for spelling, value := range secrets { + t.Run(spelling, func(t *testing.T) { + // given the space's own document carrying it + doc := []byte(`{"version":2,"kind":"space_settings","properties":{"name":"My space","` + + spelling + `":` + value + `}}`) + + // when + require.NoError(t, Validate(doc), "a stale export still imports") + _, snap, err := Unmarshal(doc, Options{}) + require.NoError(t, err) + + // then + assert.NotContains(t, snap.GetDetails().GetFields(), stored[spelling], + "a shareable bundle must not carry the source space's %s", spelling) + assert.Contains(t, snap.GetDetails().GetFields(), "name", + "the space itself survives") + }) + } +} + +// A bundle is built to be SHARED, and it was carrying the per-variant file +// encryption keys of every file in the space. +// +// This package's own API layer already refuses to emit all seven file-variant +// keys, in its words "so a future change to either the bundle or the cache +// subscription cannot accidentally leak file keys / CIDs" +// (core/api/service/property.go) — the export was the change that did. +// +// Nothing needs them on the way back: they are read by core/files/queries.go +// and the file editor, both of which run in a space that already holds the +// file, and by no import path at all. A bundle carries the file itself, so +// the same content imported elsewhere is matched and reused, or uploaded +// fresh under a NEW key that the old one does not open. +// +// How this can fail: let any of the seven back into an export and a shared +// bundle hands its recipient the keys to every file in the source space. +func TestTransientProperties_FileKeysDoNotTravel(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "f1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("f1"), + "name": str("diagram.png"), + "fileVariantKeys": strList("b53rlwqyr64xos4evv5t5vb254qf2dlfcmraphgzgg5et3optun4q"), + "fileVariantIds": strList("bafybeidflnhxa2fu3gkrzcio4ultfrun4eo5oaei3czlmuhohsfsyacpni"), + "fileVariantChecksums": strList("EPDVLKN76F8QG1P33I9DNV12P87SDE5K7J0C3G57RK5QCTULSO00"), + "fileVariantPaths": strList("/0/original/"), + "fileVariantMills": strList("/image/resize"), + "fileVariantOptions": strList("77aUeoeWD8t7zu4QovgeoFKDoCZTtmwvrYADtANdSpS3"), + "fileVariantWidths": strList("192"), + }), + } + + data, err := Marshal(model.SmartBlockType_FileObject, snap, testOptions()) + require.NoError(t, err) + + assert.NotContains(t, string(data), "b53rlwqyr64xos4evv5t5vb254qf2dlfcmraphgzgg5et3optun4q", + "THE ENCRYPTION KEY must not appear anywhere in a shared bundle") + for _, key := range []string{ + "file_variant_keys", "file_variant_ids", "file_variant_checksums", + "file_variant_paths", "file_variant_mills", "file_variant_options", "file_variant_widths", + } { + assert.NotContainsf(t, string(data), key, "%s must not travel", key) + } + assert.Contains(t, string(data), "diagram.png", "the file object itself still travels") + + t.Run("and they do not come back either", func(t *testing.T) { + doc := `{"version": 2, "id": "f1", "kind": "file_object", "properties": { + "name": "diagram.png", "file_variant_keys": ["b53rlwqyr64xos4evv5t5vb254qf2dlfcmrap"]}}` + require.NoError(t, Validate([]byte(doc))) + _, back, err := Unmarshal([]byte(doc), testOptions()) + require.NoError(t, err) + assert.Nil(t, back.Details.Fields["fileVariantKeys"], + "a document that states one must not plant it in the importing space") + }) +} + +// The analytics keys the ROOT BLOCK carries. The format had already ruled +// twice that analytics do not travel — the click-context triple "describes +// the click that made the object, not the object", and `analyticsSpaceId` +// is stripped beside a space's invite keys because "a restored space mints +// its own invites and its own analytics identity". Both rulings watched the +// DETAILS door. These two came through block fields, so the strip list never +// saw them: 1,042 of 38,105 corpus documents shipped one, analyticsOriginalId +// on 872 and analyticsContext on 445. +// +// analyticsOriginalId is the sharper of the two — it is the id of the object +// this one was made FROM, so it is a tracking identifier AND a dangling +// reference: 805 of the 872 name an object present in no bundle at all. +// +// How this can fail: sweep by prefix instead of by name and a user's own tag +// named `analytics` goes with it; drop the whole map and `isLocked` (128 +// documents) and `width` (45) go too — those are real state. +func TestTransientProperties_RootBlockAnalyticsDoNotTravel(t *testing.T) { + rootFields := func(f map[string]*types.Value) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{ + Id: "o1", + Fields: &types.Struct{Fields: f}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + Details: fields(map[string]*types.Value{"id": str("o1")}), + } + } + + t.Run("export strips them and keeps real state beside them", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, rootFields(map[string]*types.Value{ + "analyticsOriginalId": str("bafyreied5biwzt4jqshuhmhmknuu37kvsu4kafezcb5b5bcxf2jivdnntq"), + "analyticsContext": str("empty"), + "isLocked": {Kind: &types.Value_BoolValue{BoolValue: true}}, + "width": num(0.5), + }), Options{}) + require.NoError(t, err) + assert.NotContains(t, string(data), "analyticsOriginalId") + assert.NotContains(t, string(data), "analyticsContext") + assert.Contains(t, string(data), "isLocked", "real state stays") + assert.Contains(t, string(data), "width") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + }) + + // a map that was ONLY analytics leaves no empty `fields` behind. + t.Run("nothing survives means no fields member", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, rootFields(map[string]*types.Value{ + "analyticsContext": str("empty"), + }), Options{}) + require.NoError(t, err) + assert.NotContains(t, string(data), `"fields"`) + }) + + // a bundle written before the rule still imports — the keys are accepted + // and dropped, never refused (the analytics-details rule, §3). + t.Run("a stale bundle imports without them", func(t *testing.T) { + doc := []byte(`{"version": 2, "id": "o1", "root": {"fields": { + "analyticsOriginalId": "bafyreied5biwzt4jqshuhmhmknuu37kvsu4kafezcb5b5bcxf2jivdnntq", + "analyticsContext": "empty", "isLocked": true}}}`) + require.NoError(t, Validate(doc), "a stale export still imports") + _, snap, err := Unmarshal(doc, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "Validate and Unmarshal agree (§11 I2)") + + require.NotEmpty(t, snap.Blocks) + got := snap.Blocks[0].Fields.GetFields() + assert.NotContains(t, got, "analyticsOriginalId") + assert.NotContains(t, got, "analyticsContext") + assert.Contains(t, got, "isLocked", "and the real state still arrives") + }) +} diff --git a/pkg/lib/anyblockjson/transparent_test.go b/pkg/lib/anyblockjson/transparent_test.go new file mode 100644 index 0000000000..4a1f548eb2 --- /dev/null +++ b/pkg/lib/anyblockjson/transparent_test.go @@ -0,0 +1,722 @@ +package anyblockjson + +// transparent_test.go covers §7a — transparent containers. A `Layout/Div` +// (the editor's fan-out wrapper) and a content-less block contribute +// containment and nothing else, so export lifts their children to the +// container's own depth and writes nothing for the container. +// +// The package had no test and no golden carrying one of these before the +// rule existed: `grep '"group"' *_test.go testdata/` matched only dataview +// `group_by`. Every case here is new ground. + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// divBlock builds the wrapper state.wrapChildrenToDiv mints — content +// Layout/Div, id prefixed `div-` exactly as newDiv() spells it. The prefix +// is fixture realism ONLY: the rule keys on content (see the `authored +// group` cases, whose ids carry no prefix and lift just the same). +func divBlock(id string, children ...string) *model.Block { + return &model.Block{Id: id, ChildrenIds: children, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Div}}} +} + +// contentless builds the other §7a shape: a block whose content oneof is +// unset (legacy accounts hold these around a relation object's "used in" +// dataview). +func contentless(id string, children ...string) *model.Block { + return &model.Block{Id: id, ChildrenIds: children} +} + +// withChildren links children under an existing block fixture. +func withChildren(b *model.Block, children ...string) *model.Block { + b.ChildrenIds = children + return b +} + +func layoutBlock(id string, style model.BlockContentLayoutStyle, children ...string) *model.Block { + return &model.Block{Id: id, ChildrenIds: children, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: style}}} +} + +// pageOf wraps blocks into a page snapshot whose root lists rootChildren. +func pageOf(rootChildren []string, blocks ...*model.Block) *model.SmartBlockSnapshotBase { + all := []*model.Block{{Id: "obj1", ChildrenIds: rootChildren, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}} + return &model.SmartBlockSnapshotBase{ + Blocks: append(all, blocks...), + Details: fields(map[string]*types.Value{"id": str("obj1")}), + } +} + +// blockLines renders the served blocks as " " lines — +// the flat run's structure, which is exactly what the lift changes. +func blockLines(t *testing.T, data []byte) []string { + t.Helper() + var doc struct { + Blocks []struct { + Indent int `json:"indent"` + Id string `json:"id"` + Type string `json:"type"` + } `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + out := make([]string, 0, len(doc.Blocks)) + for _, b := range doc.Blocks { + out = append(out, fmt.Sprintf("%d %s %s", b.Indent, b.Id, b.Type)) + } + return out +} + +func TestMarshal_TransparentContainersAreLifted(t *testing.T) { + for _, tc := range []struct { + name string + snap *model.SmartBlockSnapshotBase + want []string + }{ + { + name: "a container at indent 0 lifts its children to indent 0", + snap: pageOf([]string{"div-1", "p3"}, + divBlock("div-1", "p1", "p2"), + textBlock("p1", model.BlockContentText_Paragraph, "one"), + textBlock("p2", model.BlockContentText_Paragraph, "two"), + textBlock("p3", model.BlockContentText_Paragraph, "three")), + want: []string{"0 p1 paragraph", "0 p2 paragraph", "0 p3 paragraph"}, + }, + { + name: "nested containers collapse fully — a chain of three removes three levels", + snap: pageOf([]string{"div-1"}, + divBlock("div-1", "div-2"), + divBlock("div-2", "div-3"), + divBlock("div-3", "p1"), + textBlock("p1", model.BlockContentText_Paragraph, "one")), + want: []string{"0 p1 paragraph"}, + }, + { + name: "a childless container emits nothing", + snap: pageOf([]string{"div-1", "p1"}, + divBlock("div-1"), + textBlock("p1", model.BlockContentText_Paragraph, "one")), + want: []string{"0 p1 paragraph"}, + }, + { + name: "a content-less block with children is a container too", + snap: pageOf([]string{"legacy"}, + contentless("legacy", "p1"), + textBlock("p1", model.BlockContentText_Paragraph, "one")), + want: []string{"0 p1 paragraph"}, + }, + { + name: "a container under real nesting keeps the parent's depth for its children", + snap: pageOf([]string{"t1"}, + withChildren(textBlock("t1", model.BlockContentText_Toggle, "toggle"), "div-1"), + divBlock("div-1", "p1"), + textBlock("p1", model.BlockContentText_Paragraph, "one")), + want: []string{"0 t1 toggle", "1 p1 paragraph"}, + }, + { + // the decoy: row and column are author-created and grammar- + // bearing (§5). A rule that lifted "any layout block" would + // flatten this into two bare paragraphs. + name: "row and column are NOT transparent", + snap: pageOf([]string{"row1"}, + layoutBlock("row1", model.BlockContentLayout_Row, "col1", "col2"), + layoutBlock("col1", model.BlockContentLayout_Column, "p1"), + layoutBlock("col2", model.BlockContentLayout_Column, "p2"), + textBlock("p1", model.BlockContentText_Paragraph, "left"), + textBlock("p2", model.BlockContentText_Paragraph, "right")), + want: []string{ + "0 row1 row", "1 col1 column", "2 p1 paragraph", + "1 col2 column", "2 p2 paragraph", + }, + }, + { + // the live I1 hole this closes: a Layout_Row with more than 40 + // columns normalizes to row → div → columns, which Marshal used + // to emit and its own Validate rejected + // ("a row block can only contain column blocks, got group"). + name: "row > container > column says row > column", + snap: pageOf([]string{"row1"}, + layoutBlock("row1", model.BlockContentLayout_Row, "div-1"), + divBlock("div-1", "col1", "col2"), + layoutBlock("col1", model.BlockContentLayout_Column, "p1"), + layoutBlock("col2", model.BlockContentLayout_Column, "p2"), + textBlock("p1", model.BlockContentText_Paragraph, "left"), + textBlock("p2", model.BlockContentText_Paragraph, "right")), + want: []string{ + "0 row1 row", "1 col1 column", "2 p1 paragraph", + "1 col2 column", "2 p2 paragraph", + }, + }, + { + name: "a structural block under a top-level container is dropped, not preserved", + snap: pageOf([]string{"div-1"}, + divBlock("div-1", "title", "p1"), + textBlock("title", model.BlockContentText_Title, "The title"), + textBlock("p1", model.BlockContentText_Paragraph, "one")), + want: []string{"0 p1 paragraph"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, tc.snap, testOptions()) + require.NoError(t, err) + // I1: Marshal never emits what its own Validate rejects + require.NoError(t, Validate(data)) + assert.Equal(t, tc.want, blockLines(t, data)) + assert.NotContains(t, string(data), `"group"`, "export emits no group, ever") + }) + } +} + +// TestMarshal_ContainerAttributesAreDroppedWithAWarning pins the accepted +// loss: a container's own align/background/fields go with it. The warning is +// the only trace, and it costs nothing on real data — every one of the 7,303 +// wrappers in the production corpus carries no attribute at all. +func TestMarshal_ContainerAttributesAreDroppedWithAWarning(t *testing.T) { + div := divBlock("div-1", "p1") + div.BackgroundColor = "red" + div.Align = model.Block_AlignCenter + div.Fields = fields(map[string]*types.Value{"custom": str("kept nowhere")}) + snap := pageOf([]string{"div-1"}, div, textBlock("p1", model.BlockContentText_Paragraph, "one")) + + var warnings []Issue + opts := testOptions() + opts.OnWarning = func(i Issue) { warnings = append(warnings, i) } + + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + assert.Equal(t, []string{"0 p1 paragraph"}, blockLines(t, data)) + assert.NotContains(t, string(data), "red") + assert.NotContains(t, string(data), "kept nowhere") + + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "div-1") + assert.Contains(t, warnings[0].Message, "attributes on it are dropped") +} + +// TestMarshal_ContainerAttributesWarnOnceUnderCompaction: the id census runs +// the block emit a SECOND time on a throwaway exporter (§9a), so every +// warning the emit raises is raised twice unless the probe's sink is +// silenced. A caller shown each issue twice stops trusting the count. +func TestMarshal_ContainerAttributesWarnOnceUnderCompaction(t *testing.T) { + div := divBlock("div-1", "p1") + div.BackgroundColor = "red" + snap := pageOf([]string{"div-1"}, div, textBlock("p1", model.BlockContentText_Paragraph, "one")) + + var warnings []Issue + opts := testOptions() + opts.CompactIds = true // this is what turns the census probe on + opts.OnWarning = func(i Issue) { warnings = append(warnings, i) } + + _, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + assert.Len(t, warnings, 1, "the census probe must not report the emit's issues a second time") +} + +// TestMarshal_ContainerAttributesSilentWhenBare is the other half: a bare +// container — what normalization actually mints — warns about nothing. +func TestMarshal_ContainerAttributesSilentWhenBare(t *testing.T) { + snap := pageOf([]string{"div-1"}, divBlock("div-1", "p1"), + textBlock("p1", model.BlockContentText_Paragraph, "one")) + + var warnings []Issue + opts := testOptions() + opts.OnWarning = func(i Issue) { warnings = append(warnings, i) } + + _, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + assert.Empty(t, warnings) +} + +// TestMarshal_ContainerCycleTerminates: the lift skips blockToJSON, which is +// where the emit-once mark is set, so the lift has to set it itself. Without +// that, a ChildrenIds cycle through a chain of containers recurses until the +// stack gives out — an untrusted snapshot crashing the process. +func TestMarshal_ContainerCycleTerminates(t *testing.T) { + snap := pageOf([]string{"div-1"}, + divBlock("div-1", "div-2", "p1"), + divBlock("div-2", "div-1"), // back-edge + textBlock("p1", model.BlockContentText_Paragraph, "one")) + + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.Equal(t, []string{"0 p1 paragraph"}, blockLines(t, data)) +} + +// TestMarshal_ContainerSharedByTwoParents: the same mark, read back. A +// container listed by two parents must lift once — its children are blocks, +// and emitting them twice is a document with duplicate ids that Validate +// rejects (I1). +func TestMarshal_ContainerSharedByTwoParents(t *testing.T) { + snap := pageOf([]string{"t1", "t2"}, + withChildren(textBlock("t1", model.BlockContentText_Toggle, "first"), "div-1"), + withChildren(textBlock("t2", model.BlockContentText_Toggle, "second"), "div-1"), + divBlock("div-1", "p1"), + textBlock("p1", model.BlockContentText_Paragraph, "one")) + + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.Equal(t, []string{"0 t1 toggle", "1 p1 paragraph", "0 t2 toggle"}, blockLines(t, data)) +} + +// TestMarshal_ContainerInsideATableCell: the lift lives in appendBlocksFlat, +// which is also what walks a cell's descendants — so cells get the rule for +// free. "For free" is still a claim that needs a test. +func TestMarshal_ContainerInsideATableCell(t *testing.T) { + snap := tableSnapshot( + &model.BlockContentOfText{Text: &model.BlockContentText{Style: model.BlockContentText_Paragraph, Text: "cell"}}, + divBlock("div-1", "p1"), + textBlock("p1", model.BlockContentText_Paragraph, "under the container"), + ) + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.NotContains(t, string(data), `"group"`) + // the cell renders as its array form: the cell block at 0, the lifted + // paragraph at 1 — the depth the container held + assert.Contains(t, string(data), `"indent": 1`) + assert.Contains(t, string(data), "under the container") +} + +// TestMarshal_ContainerAsACellRendersEmpty is the one place the lift cannot +// run: a cell is a position, not a run, so there is nowhere to lift to. +func TestMarshal_ContainerAsACellRendersEmpty(t *testing.T) { + snap := tableSnapshot(nil, textBlock("p1", model.BlockContentText_Paragraph, "lost with the cell")) + + var warnings []Issue + opts := testOptions() + opts.OnWarning = func(i Issue) { warnings = append(warnings, i) } + + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.NotContains(t, string(data), "lost with the cell") + require.Len(t, warnings, 1) + assert.Contains(t, warnings[0].Message, "a cell cannot be lifted") +} + +// TestMarshalBlockSubtree_ContainerRoot: the fragment surface follows the +// rule at every level, its ROOT included — a deliberate divergence from §7's +// structural carve-out, because no read surface ever serves a container id, +// so no caller can address one except out of a stale cache. +func TestMarshalBlockSubtree_ContainerRoot(t *testing.T) { + t.Run("a container root marshals as its lifted children", func(t *testing.T) { + subtree := []*model.Block{ + divBlock("div-1", "p1", "p2"), + textBlock("p1", model.BlockContentText_Paragraph, "one"), + textBlock("p2", model.BlockContentText_Paragraph, "two"), + } + data, err := MarshalBlockSubtree(subtree, testOptions()) + require.NoError(t, err) + assert.Equal(t, []string{"0 p1 paragraph", "0 p2 paragraph"}, blockLines(t, data)) + }) + t.Run("a childless container root marshals as an empty run", func(t *testing.T) { + data, err := MarshalBlockSubtree([]*model.Block{divBlock("div-1")}, testOptions()) + require.NoError(t, err) + assert.Empty(t, blockLines(t, data)) + }) +} + +// TestMarshal_WrappedPrimaryDataviewPinsUnderOmitIds is the live corruption +// this repairs on 160 real objects: their own dataview sits at indent 1 +// inside a wrapper, so §7's primary-dataview pin — which fires only at +// indent 0 — never fires, the id `dataview` is lost under OmitIds, and the +// editor adds a SECOND, empty dataview on open. Lifted, the pin fires. +func TestMarshal_WrappedPrimaryDataviewPinsUnderOmitIds(t *testing.T) { + snap := pageOf([]string{"div-1"}, + divBlock("div-1", "dv1"), + &model.Block{Id: "dv1", Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + RelationLinks: []*model.RelationLink{{Key: "name", Format: model.RelationFormat_shorttext}}, + Views: []*model.BlockContentDataviewView{{Id: "v1", Type: model.BlockContentDataviewView_Table, Name: "All"}}, + }}}) + + opts := testOptions() + opts.OmitIds = true + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.Equal(t, []string{"0 dataview"}, blockLines(t, data), "the dataview must be served at indent 0") + + _, reimported, err := Unmarshal(data, opts) + require.NoError(t, err) + var dvId string + for _, b := range reimported.Blocks { + if b.GetDataview() != nil { + dvId = b.Id + } + } + assert.Equal(t, dataviewBlockId, dvId, + "the object's own dataview must come back at the editor's fixed id, or the editor adds a second one") +} + +// TestMarshal_TransparentContainersGolden freezes the served bytes for a +// document carrying containers in every shape the corpus has: at indent 0, +// nested, childless, attributed, content-less, inside a row, and inside a +// table cell. +func TestMarshal_TransparentContainersGolden(t *testing.T) { + attributed := divBlock("div-attributed", "p4") + attributed.BackgroundColor = "red" + attributed.Align = model.Block_AlignCenter + + snap := pageOf( + []string{"div-top", "div-empty", "legacy", "row1", "div-attributed", "table1"}, + divBlock("div-top", "h1", "div-nested"), + textBlock("h1", model.BlockContentText_Header1, "Heading"), + divBlock("div-nested", "p1", "p2"), + textBlock("p1", model.BlockContentText_Paragraph, "one"), + textBlock("p2", model.BlockContentText_Paragraph, "two"), + divBlock("div-empty"), + contentless("legacy", "p3"), + textBlock("p3", model.BlockContentText_Paragraph, "three"), + layoutBlock("row1", model.BlockContentLayout_Row, "div-inrow"), + divBlock("div-inrow", "col1"), + layoutBlock("col1", model.BlockContentLayout_Column, "p5"), + textBlock("p5", model.BlockContentText_Paragraph, "in a column"), + attributed, + textBlock("p4", model.BlockContentText_Paragraph, "four"), + // a table whose single cell holds a container below it + &model.Block{Id: "table1", ChildrenIds: []string{"tcols", "trows"}, + Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}}, + layoutBlock("tcols", model.BlockContentLayout_TableColumns, "c1"), + layoutBlock("trows", model.BlockContentLayout_TableRows, "r1"), + &model.Block{Id: "c1", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}, + &model.Block{Id: "r1", ChildrenIds: []string{"r1-c1"}, + Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}}, + &model.Block{Id: "r1-c1", ChildrenIds: []string{"div-incell"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{Style: model.BlockContentText_Paragraph, Text: "cell"}}}, + divBlock("div-incell", "p6"), + textBlock("p6", model.BlockContentText_Paragraph, "under the cell's container"), + ) + + opts := testOptions() + opts.OnWarning = func(Issue) {} // the attributed container warns; not the subject here + data, err := Marshal(model.SmartBlockType_Page, snap, opts) + require.NoError(t, err) + require.NoError(t, Validate(data)) + require.False(t, strings.Contains(string(data), `"group"`)) + checkGolden(t, "containers.json", data) +} + +// +// ---- import (§7a) ---- +// + +// importedTree renders a snapshot's block graph as " " lines +// in document order, so a lift can be read off the rebuilt tree. +func importedTree(t *testing.T, s *model.SmartBlockSnapshotBase) []string { + t.Helper() + byId := map[string]*model.Block{} + child := map[string]bool{} + for _, b := range s.Blocks { + byId[b.Id] = b + for _, c := range b.ChildrenIds { + child[c] = true + } + } + var root *model.Block + for _, b := range s.Blocks { + if !child[b.Id] { + root = b + break + } + } + var out []string + var walk func(ids []string, depth int) + walk = func(ids []string, depth int) { + for _, id := range ids { + b := byId[id] + if b == nil { + continue + } + out = append(out, fmt.Sprintf("%d %s", depth, blockKind(b))) + walk(b.ChildrenIds, depth+1) + } + } + require.NotNil(t, root) + walk(root.ChildrenIds, 0) + return out +} + +// blockKind names a model block the way the JSON type does, closely enough +// to read a tree by. +func blockKind(b *model.Block) string { + switch c := b.Content.(type) { + case *model.BlockContentOfText: + return "text:" + orEmpty(c.Text).Text + case *model.BlockContentOfLayout: + switch orEmpty(c.Layout).Style { + case model.BlockContentLayout_Row: + return "row" + case model.BlockContentLayout_Column: + return "column" + case model.BlockContentLayout_Div: + return "DIV" + } + return "layout" + case *model.BlockContentOfDataview: + return "dataview" + case nil: + return "CONTENT-LESS" + } + return fmt.Sprintf("%T", b.Content) +} + +func TestUnmarshal_TransparentContainersAreLifted(t *testing.T) { + for _, tc := range []struct { + name string + blocks string + want []string + }{ + { + name: "a container contributes no block and its children re-base", + blocks: `{"type":"group"},{"indent":1,"type":"paragraph","text":"one"},{"indent":1,"type":"paragraph","text":"two"}`, + want: []string{"0 text:one", "0 text:two"}, + }, + { + name: "nested containers re-base recursively", + blocks: `{"type":"group"},{"indent":1,"type":"group"},{"indent":2,"type":"paragraph","text":"deep"}`, + want: []string{"0 text:deep"}, + }, + { + name: "a childless container is simply gone", + blocks: `{"type":"group"},{"type":"paragraph","text":"after"}`, + want: []string{"0 text:after"}, + }, + { + name: "only the container's own subtree re-bases", + blocks: `{"type":"toggle","text":"t"},{"indent":1,"type":"group"},{"indent":2,"type":"paragraph","text":"inside"},{"type":"paragraph","text":"after"}`, + want: []string{"0 text:t", "1 text:inside", "0 text:after"}, + }, + { + name: "attributes on a container are ignored", + blocks: `{"type":"group","background_color":"red","align":"center","fields":{"x":"y"}},{"indent":1,"type":"paragraph","text":"one"}`, + want: []string{"0 text:one"}, + }, + { + // the decoy again, on the read side + name: "row and column are rebuilt, not lifted", + blocks: `{"type":"row"},{"indent":1,"type":"column"},{"indent":2,"type":"paragraph","text":"left"}`, + want: []string{"0 row", "1 column", "2 text:left"}, + }, + { + name: "row > group > column reads as row > column", + blocks: `{"type":"row"},{"indent":1,"type":"group"},{"indent":2,"type":"column"},{"indent":3,"type":"paragraph","text":"left"}`, + want: []string{"0 row", "1 column", "2 text:left"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + doc := fmt.Sprintf(`{"version":2,"type":"page","blocks":[%s]}`, tc.blocks) + require.NoError(t, Validate([]byte(doc))) + + _, snapshot, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, tc.want, importedTree(t, snapshot)) + for _, b := range snapshot.Blocks { + assert.NotEqual(t, "DIV", blockKind(b), + "import must not mint a Layout_Div: no read would ever show it, and normalization never removes one that has children") + } + }) + } +} + +// TestUnmarshal_ContainerLiftRunsBeforeTheStructuralRules pins §7a's +// ordering. The lift is positional, so a lifted title is at indent 0 for +// every purpose: absorbed into properties.name exactly as a title written at +// indent 0 is, and the wrapped primary dataview reaches the position §7's pin +// requires. +func TestUnmarshal_ContainerLiftRunsBeforeTheStructuralRules(t *testing.T) { + t.Run("a wrapped title is absorbed into the name property", func(t *testing.T) { + doc := `{"version":2,"type":"page","blocks":[ + {"type":"group"}, + {"indent":1,"type":"title","text":"Wrapped title"}, + {"indent":1,"type":"paragraph","text":"body"}]}` + _, snapshot, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "Wrapped title", snapshot.Details.Fields["name"].GetStringValue()) + assert.Equal(t, []string{"0 text:body"}, importedTree(t, snapshot)) + }) + t.Run("a wrapped primary dataview is pinned to the editor's fixed id", func(t *testing.T) { + doc := `{"version":2,"type":"page","blocks":[ + {"type":"group"}, + {"indent":1,"type":"dataview","views":[{"type":"table","name":"All"}]}]}` + _, snapshot, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + var dvId string + for _, b := range snapshot.Blocks { + if b.GetDataview() != nil { + dvId = b.Id + } + } + assert.Equal(t, dataviewBlockId, dvId, + "the pin fires only at indent 0, which is where the lift puts it") + }) +} + +// TestUnmarshal_ContainerInsideATableCell covers the second of the three +// flatSubtree entry points. An unfixed cell path is worse than shipping +// nothing: it mints a real Layout_Div inside a table that no read ever shows. +func TestUnmarshal_ContainerInsideATableCell(t *testing.T) { + doc := `{"version":2,"type":"page","blocks":[{"type":"table", + "columns":[{"id":"c1"}], + "rows":[{"id":"r1","cells":[[ + {"type":"paragraph","text":"cell"}, + {"indent":1,"type":"group"}, + {"indent":2,"type":"paragraph","text":"under the container"}]]}]}]}` + require.NoError(t, Validate([]byte(doc))) + + _, snapshot, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + for _, b := range snapshot.Blocks { + require.NotEqual(t, "DIV", blockKind(b), "a container inside a cell must not become a Layout_Div") + } + // the paragraph re-bases to the cell's own first level + var cell *model.Block + for _, b := range snapshot.Blocks { + if b.Id == "r1-c1" { + cell = b + } + } + require.NotNil(t, cell) + require.Len(t, cell.ChildrenIds, 1) + byId := map[string]*model.Block{} + for _, b := range snapshot.Blocks { + byId[b.Id] = b + } + assert.Equal(t, "text:under the container", blockKind(byId[cell.ChildrenIds[0]])) +} + +// TestUnmarshalBlocks_ContainerOnTheWritePath covers the third entry point — +// the API's block-write surface. A container pasted through it used to mint a +// Layout_Div straight into a live object. +func TestUnmarshalBlocks_ContainerOnTheWritePath(t *testing.T) { + run := []json.RawMessage{ + json.RawMessage(`{"type":"group"}`), + json.RawMessage(`{"indent":1,"type":"paragraph","text":"one"}`), + json.RawMessage(`{"indent":1,"type":"paragraph","text":"two"}`), + } + blocks, topIds, err := UnmarshalBlocks(run, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + require.Len(t, blocks, 2, "the container contributes no block") + assert.Equal(t, []string{blocks[0].Id, blocks[1].Id}, topIds, + "both paragraphs are top-level in the run — they take the container's position") + for _, b := range blocks { + assert.NotEqual(t, "DIV", blockKind(b)) + } +} + +// TestUnmarshalBlock_LoneContainerIsRefused: this entry point's contract is +// exactly one block. Returning zero would leave a replaceBlock silently +// replacing nothing. +func TestUnmarshalBlock_LoneContainerIsRefused(t *testing.T) { + _, err := UnmarshalBlock(json.RawMessage(`{"type":"group"}`), "b1", Options{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "transparent container") + assert.Contains(t, err.Error(), "/blocks/0/type") +} + +// TestValidate_ContainmentIsJudgedOnTheLiftedTree is §7a's containment rule: +// the grammar is checked against the tree import builds, and the message names the effective +// parent — or it reads as wrong to whoever wrote the group. +func TestValidate_ContainmentIsJudgedOnTheLiftedTree(t *testing.T) { + for _, tc := range []struct { + name string + blocks string + want string // "" = valid + }{ + { + // this document is REJECTED before §7a, and it is one + // wrapChildrenToDiv produces from a row with 41 columns + name: "row > group > column is valid: it says row > column", + blocks: `{"type":"row"},{"indent":1,"type":"group"},{"indent":2,"type":"column"}`, + }, + { + name: "row > group (childless) is valid: a row with no columns is a legal document", + blocks: `{"type":"row"},{"indent":1,"type":"group"}`, + }, + { + name: "row > group > paragraph is reported against the row", + blocks: `{"type":"row"},{"indent":1,"type":"group"},{"indent":2,"type":"paragraph","text":"x"}`, + want: "nested under a group inside a row — a row block can only contain column blocks, got paragraph", + }, + { + name: "divider > group > paragraph is reported against the divider", + blocks: `{"type":"divider"},{"indent":1,"type":"group"},{"indent":2,"type":"paragraph","text":"x"}`, + want: "nested under a group inside a divider block — divider blocks cannot have children", + }, + { + name: "the direct message is unchanged when no container is between", + blocks: `{"type":"row"},{"indent":1,"type":"paragraph","text":"x"}`, + want: "a row block can only contain column blocks, got paragraph", + }, + { + name: "a container under a leaf is exempt: it becomes nothing", + blocks: `{"type":"divider"},{"indent":1,"type":"group"}`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + doc := fmt.Sprintf(`{"version":2,"type":"page","blocks":[%s]}`, tc.blocks) + err := Validate([]byte(doc)) + if tc.want == "" { + require.NoError(t, err) + // I2: whatever Validate accepts, Unmarshal reads + _, _, uerr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, uerr) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + }) + } +} + +// TestValidate_ContainerCannotBeACellBlock: a cell is a position, not a run, +// so it is the one spelling of a container the format cannot read back — +// and Validate has to say so, or it would accept what Unmarshal refuses. +func TestValidate_ContainerCannotBeACellBlock(t *testing.T) { + doc := `{"version":2,"type":"page","blocks":[{"type":"table", + "columns":[{"id":"c1"}], + "rows":[{"id":"r1","cells":[[{"type":"group"},{"indent":1,"type":"paragraph","text":"x"}]]}]}]}` + + err := Validate([]byte(doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), "a cell block cannot be a group") + + _, _, uerr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, uerr, "Validate and Unmarshal must agree (I2)") +} + +// TestUnmarshal_ContainerRoundTripsToNothing is guarantee 2 (§11) on a +// document carrying containers: `group` is a readable input token that no +// export produces, so the canonical form of a document with one is the same +// document without it. +func TestUnmarshal_ContainerRoundTripsToNothing(t *testing.T) { + doc := []byte(`{"version":2,"type":"page","blocks":[` + + `{"type":"group"},{"indent":1,"type":"paragraph","text":"one"},` + + `{"type":"paragraph","text":"two"}]}`) + + sbType, snapshot, err := Unmarshal(doc, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + first, err := Marshal(sbType, snapshot, Options{OmitIds: true}) + require.NoError(t, err) + + _, again, err := Unmarshal(first, Options{GenerateId: seqIds("h")}) + require.NoError(t, err) + second, err := Marshal(sbType, again, Options{OmitIds: true}) + require.NoError(t, err) + + assert.Equal(t, string(first), string(second)) + assert.NotContains(t, string(first), "group") + assert.Equal(t, []string{"0 paragraph", "0 paragraph"}, blockLines(t, first)) +} diff --git a/pkg/lib/anyblockjson/transparentcell_test.go b/pkg/lib/anyblockjson/transparentcell_test.go new file mode 100644 index 0000000000..16da2d361b --- /dev/null +++ b/pkg/lib/anyblockjson/transparentcell_test.go @@ -0,0 +1,71 @@ +package anyblockjson + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// §7a lifts a transparent container everywhere a run exists. A table cell is a +// POSITION, not a run — there is nowhere to lift to — so a cell spelled as a +// container is the one shape the format cannot read back, and both sides must +// refuse it. +// +// A cell has two spellings (§6.1): the object form and the array form. The +// array form reached the refusal through checkFlatRun's `inCell` branch; the +// object form went to walkBlock, which had no such check, so Validate accepted +// a document Unmarshal hard-refused — an I2 breach, and a regression: before +// §7a both sides accepted it and minted a Layout_Div cell. +// +// This can only fail if one side stops refusing: the test asserts the two +// VERDICTS agree, not that either one is an error, so a rule that stopped +// firing on both sides at once would still have to keep them equal — and the +// separate "both refuse" assertion catches that. +func TestValidate_ACellCannotBeATransparentContainer_BothSpellings(t *testing.T) { + for _, attrs := range []string{ + ``, + `, "background_color": "red"`, + `, "align": "center"`, + `, "vertical_align": "middle"`, + } { + cells := map[string]string{ + "object form": fmt.Sprintf(`{"type": "group"%s}`, attrs), + "array form": fmt.Sprintf(`[{"type": "group"%s}]`, attrs), + } + for form, cell := range cells { + t.Run(form+attrs, func(t *testing.T) { + doc := []byte(fmt.Sprintf( + `{"version": 2, "type": "page", "blocks": [{"type": "table", + "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [%s]}]}]}`, cell)) + + verr := Validate(doc) + _, _, uerr := Unmarshal(doc, Options{}) + + require.Error(t, verr, "a cell cannot be a transparent container (§7a)") + require.Error(t, uerr, "…and import refuses it too") + assert.Equal(t, verr != nil, uerr != nil, + "Validate and Unmarshal must agree (§11 I2)") + }) + } + } +} + +// the control: an ordinary cell block in BOTH spellings still passes, so the +// refusal above cannot pass by rejecting every cell. +func TestValidate_AnOrdinaryCellStillPasses_BothSpellings(t *testing.T) { + for form, cell := range map[string]string{ + "object form": `{"type": "paragraph", "text": "x"}`, + "array form": `[{"type": "paragraph", "text": "x"}]`, + } { + t.Run(form, func(t *testing.T) { + doc := []byte(fmt.Sprintf( + `{"version": 2, "type": "page", "blocks": [{"type": "table", + "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [%s]}]}]}`, cell)) + require.NoError(t, Validate(doc)) + _, _, err := Unmarshal(doc, Options{}) + require.NoError(t, err) + }) + } +} diff --git a/pkg/lib/anyblockjson/transparentchain_test.go b/pkg/lib/anyblockjson/transparentchain_test.go new file mode 100644 index 0000000000..57015dac93 --- /dev/null +++ b/pkg/lib/anyblockjson/transparentchain_test.go @@ -0,0 +1,125 @@ +package anyblockjson + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// §7a's containment check walks PAST a container to find the effective parent, +// because a container becomes nothing and its children take its place. The walk +// has to skip a CHAIN, not one level: 3,121 of the corpus's 7,463 containers sit +// inside another, so a chain is the common shape rather than the exotic one. +// +// A single-step walk leaves `row > group > group > paragraph` validating, which +// imports to `row > paragraph` — and Marshal of that snapshot emits a document +// its own Validate rejects. That is the I1 hole §7a exists to close, reopened +// one level down. +// +// These fail only if the walk stops skipping: each asserts a REFUSAL on a +// document whose effective parent is reachable solely through two containers, +// so a walk that gives up after one step accepts it. +func TestValidate_ContainmentIsJudgedThroughAChainOfContainers(t *testing.T) { + for name, doc := range map[string]string{ + "a row cannot hold a paragraph behind two containers": `{"version": 2, "type": "page", "blocks": [ + {"type": "row"}, + {"indent": 1, "type": "group"}, + {"indent": 2, "type": "group"}, + {"indent": 3, "type": "paragraph", "text": "x"}]}`, + "a divider holds nothing, behind any number of containers": `{"version": 2, "type": "page", "blocks": [ + {"type": "divider"}, + {"indent": 1, "type": "group"}, + {"indent": 2, "type": "group"}, + {"indent": 3, "type": "paragraph", "text": "x"}]}`, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(doc)) + require.Error(t, err, "the effective parent is reachable only through the chain") + assert.Contains(t, err.Error(), "group", + "and the message names the container between, or it reads as wrong to whoever wrote it") + }) + } +} + +// the control: what the chain lifts TO must still be accepted, or the test above +// could pass by refusing every chain. +func TestValidate_AChainOfContainersOverALegalChildIsFine(t *testing.T) { + require.NoError(t, Validate([]byte(`{"version": 2, "type": "page", "blocks": [ + {"type": "row"}, + {"indent": 1, "type": "group"}, + {"indent": 2, "type": "group"}, + {"indent": 3, "type": "column"}, + {"indent": 4, "type": "paragraph", "text": "x"}]}`)), + "row > group > group > column says row > column, which is legal") +} + +// The import lift's indent arithmetic is only OBSERVABLE where import reads an +// ABSOLUTE indent — §7's structural absorption and the primary-dataview pin, +// both of which fire at indent 0. flatSubtree tolerates indent gaps when +// rebuilding a tree, so an off-by-one inside a container is invisible +// everywhere else. These pin the two shapes that make it visible: NESTED +// containers and SIBLING containers. +// +// The 160-object dataview-pin repair is part of what justifies §7a, and before +// this it was pinned for the single-container shape only. +func TestImport_TheLiftsIndentArithmeticSurvivesEveryWrappingShape(t *testing.T) { + cases := map[string]struct{ doc, wantName, wantId string }{ + "title under two NESTED containers is absorbed": { + doc: `{"version": 2, "type": "page", "blocks": [ + {"type": "group"}, {"indent": 1, "type": "group"}, + {"indent": 2, "type": "title", "text": "Absorbed"}]}`, + wantName: "Absorbed", + }, + "title under the second of two SIBLING containers is absorbed": { + doc: `{"version": 2, "type": "page", "blocks": [ + {"type": "group"}, {"indent": 1, "type": "paragraph", "text": "a"}, + {"type": "group"}, {"indent": 1, "type": "title", "text": "Absorbed"}]}`, + wantName: "Absorbed", + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + _, snap, err := Unmarshal([]byte(c.doc), Options{}) + require.NoError(t, err) + assert.Equal(t, c.wantName, snap.GetDetails().GetFields()["name"].GetStringValue(), + "a lifted title lands at indent 0 for every purpose (§7, §7a)") + }) + } +} + +// UnmarshalBlock is the fragment surface behind replaceBlock: it addresses +// exactly ONE block, so a container — which contributes none — has to be +// refused rather than silently turned into a wrapper no read will ever show +// the caller again. This pins that CONTRACT. +// +// What it does not pin, said plainly rather than implied: the refusal inside +// blockFromJSON (import.go, the transparentBlockTypes case). Replacing that +// line with the pre-§7a behaviour leaves the whole package green, including +// this test — because UnmarshalBlock validates first and the validation side +// refuses the container on its own. Probed: with the line mutated, +// UnmarshalBlock still returns "validation failed". The line is therefore +// DEFENSIVE ONLY, unreachable through every public entry point, and a future +// reader should not delete it on the strength of a coverage report — it is the +// backstop for a new caller that skips validation. +func TestUnmarshalBlock_RefusesATransparentContainer(t *testing.T) { + for _, spelling := range []string{ + `{"type": "group"}`, + `{"type": "group", "background_color": "red"}`, + } { + blocks, err := UnmarshalBlock([]byte(spelling), "b1", Options{}) + require.Error(t, err, + "a caller that asked for one block must be told a container is not one (§7a)") + assert.Contains(t, err.Error(), "transparent container") + assert.Nil(t, blocks, "and gets no Layout_Div minted behind its back") + } +} + +// the control: an ordinary single block still comes back, so the refusal above +// cannot pass by rejecting everything. +func TestUnmarshalBlock_StillReturnsAnOrdinaryBlock(t *testing.T) { + blocks, err := UnmarshalBlock([]byte(`{"type": "paragraph", "text": "x"}`), "b1", Options{}) + require.NoError(t, err) + require.Len(t, blocks, 1) + assert.Equal(t, "b1", blocks[0].Id) +} diff --git a/pkg/lib/anyblockjson/typekeys_test.go b/pkg/lib/anyblockjson/typekeys_test.go new file mode 100644 index 0000000000..3d66cc63f5 --- /dev/null +++ b/pkg/lib/anyblockjson/typekeys_test.go @@ -0,0 +1,877 @@ +package anyblockjson + +// The type namespace gets the same verbatim-first treatment as the property +// namespace (§3): a term that names a stored type key IS that key, the +// bundled slug table applies only to terms that are not stored keys, and the +// document carries its own inverse — the `type_internal_keys` envelope legend — +// wherever the shipped table would give a package-only reader the wrong +// answer. Before the legend existed, a node-backed vocabulary slugging a +// custom type `69bbfc…` to `task` exported `"type": "task"` with nothing to +// invert it, and a package-only reader bound it to the bundled Task type — a +// different type, silently. Same for `type_properties[].object_types`. + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// customTypeKey is a space-minted (bson) type key of the shape real spaces +// produce for user-created types. +const customTypeKey = "69bbfc78877a91b1d12d1a7c" + +// typedSpaceVocabulary is a node-backed vocabulary for BOTH namespaces: it +// knows the space's stored slugs for properties and for types, which the +// bundled table cannot. +type typedSpaceVocabulary struct { + propSlugOf map[string]string + typeSlugOf map[string]string +} + +func (v typedSpaceVocabulary) PropertySlug(key string) string { + if slug, ok := v.propSlugOf[key]; ok { + return slug + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (v typedSpaceVocabulary) PropertyKey(slug string) (string, bool) { + for key, s := range v.propSlugOf { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (v typedSpaceVocabulary) TypeSlug(key string) string { + if slug, ok := v.typeSlugOf[key]; ok { + return slug + } + return BundledKeyVocabulary{}.TypeSlug(key) +} + +func (v typedSpaceVocabulary) TypeKey(slug string) (string, bool) { + for key, s := range v.typeSlugOf { + if s == slug { + return key, true + } + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +func typedSnapshot(objectTypes ...string) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{"id": str("o1")}), + ObjectTypes: objectTypes, + } +} + +type envelopeTypeDoc struct { + Kind string `json:"kind"` + Type string `json:"type"` + TemplateFor string `json:"template_for"` + PropertyKeys map[string]string `json:"property_internal_keys"` + TypeKeys map[string]string `json:"type_internal_keys"` + Properties map[string]any `json:"properties"` + TypeSettings struct { + PropertyDefinitions []TypeProperty `json:"property_definitions"` + } `json:"type_settings"` +} + +// TypeProps reads the property-definition list wherever the tests consult +// it — inside type_settings since v0.32. +func (d envelopeTypeDoc) TypeProps() []TypeProperty { + return d.TypeSettings.PropertyDefinitions +} + +func decodeEnvelope(t *testing.T, data []byte) envelopeTypeDoc { + t.Helper() + var doc envelopeTypeDoc + require.NoError(t, json.Unmarshal(data, &doc)) + return doc +} + +// censusPropResolver serves two recommended-list entries, one of which +// buildTypeProperties drops: its definition carries no key, which real type +// objects hold whenever a vocabulary once resolved a spelling onto the empty +// key. Both name target types, and only the surviving one's target reaches +// the document. +type censusPropResolver struct{} + +func (censusPropResolver) PropertyById(id string) (PropertyDefinition, bool) { + switch id { + case "k1": + return PropertyDefinition{Key: "", Format: model.RelationFormat_object, + ObjectTypes: []string{"cust"}}, true + case "k2": + return PropertyDefinition{Key: "owner", Format: model.RelationFormat_object, + ObjectTypes: []string{"custom1"}}, true + } + return PropertyDefinition{}, false +} + +func (censusPropResolver) PropertyId(def PropertyDefinition) (string, bool) { + if def.Key == "owner" { + return "k2", true + } + return "", false +} + +// TestExport_IsAFixpointWhenTheCensusShrinks: exporting an object, importing +// it and exporting it again must produce the same document (§9 — "provided +// ids are preserved so re-exports diff cleanly" is worth nothing if the terms +// move instead). +// +// The census is what threatened it. It reserved every stored type key the +// SNAPSHOT named, while the document spells only the keys §2 models — one +// type, plus a template's target — and only the type properties export +// actually writes. Every key in the gap was reserved for a term no reader +// ever sees, and it backed a real slug off: generation 1 wrote the stored key +// verbatim, generation 2 — one round trip later, with the extra keys gone +// from the snapshot — wrote the slug and a legend line to invert it. Same +// object, two documents. +// +// Both halves of the gap are here. Neither needs a hostile vocabulary: the +// vocabulary is an ordinary space-backed one, and the shapes are an object +// with a second type the format does not model and a type object holding a +// keyless entry in a recommended list. +func TestExport_IsAFixpointWhenTheCensusShrinks(t *testing.T) { + regenerate := func(t *testing.T, sbType model.SmartBlockType, + snap *model.SmartBlockSnapshotBase, opts Options) (string, string) { + t.Helper() + gen1, err := Marshal(sbType, snap, opts) + require.NoError(t, err) + read := opts + read.GenerateId = seqIds("g") + _, back, err := Unmarshal(gen1, read) + require.NoError(t, err) + gen2, err := Marshal(sbType, back, opts) + require.NoError(t, err) + return string(gen1), string(gen2) + } + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{"custom1": "cust"}} + + t.Run("an object type the envelope does not model", func(t *testing.T) { + // given: `cust` is the first type's slug AND the second type's stored + // key, and the second type is past the only position §2 models + snap := typedSnapshot("ot-custom1", "ot-cust") + + // when + gen1, gen2 := regenerate(t, model.SmartBlockType_Page, snap, Options{Keys: vocab}) + + // then + assert.Equal(t, gen1, gen2, "the second generation must repeat the first") + assert.Contains(t, gen1, `"type": "cust"`, + "the truncated entry is not in the document, so its key reserves nothing") + assert.Contains(t, gen1, `"cust": "custom1"`, "and the spelling owes its legend line") + }) + + t.Run("a type property no slot writes", func(t *testing.T) { + // given: the keyless entry names `cust` and is dropped; the entry that + // survives targets `custom1`, whose slug is `cust` + snap := typedSnapshot("ot-page") + snap.Details.Fields["recommendedFeaturedRelations"] = strList("k1", "k2") + + // when + gen1, gen2 := regenerate(t, model.SmartBlockType_STType, snap, + Options{Keys: vocab, ResolveProperties: censusPropResolver{}}) + + // then + assert.Equal(t, gen1, gen2, "the second generation must repeat the first") + assert.Contains(t, gen1, `"cust"`, "the dropped definition's target reserves nothing") + }) +} + +// The legend carries exactly what the bundled table cannot invert — the +// mirror of TestExport_PropertyKeysLegendCarriesWhatTheTableCannot for the +// type namespace. +func TestExport_TypeKeysLegendCarriesWhatTheTableCannot(t *testing.T) { + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "task"}} + snap := typedSnapshot("ot-" + customTypeKey) + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "task", doc.Type, "the custom type key is spelled as its slug") + assert.Equal(t, map[string]string{"task": customTypeKey}, doc.TypeKeys, + "the slug shadows the bundled task key, so the document owes the entry that inverts it") + assert.Empty(t, doc.PropertyKeys, "the type legend is not the property legend") +} + +// A bundled type spelled as its derived slug needs no entry: the table ships +// with every reader. +func TestExport_TypeKeysLegendOmitsWhatTheTableInverts(t *testing.T) { + for _, key := range []string{"page", "objectType"} { + data, err := Marshal(model.SmartBlockType_Page, typedSnapshot("ot-"+key), Options{}) + require.NoError(t, err) + doc := decodeEnvelope(t, data) + assert.Empty(t, doc.TypeKeys, "bundled key %q owes no legend entry", key) + } +} + +// The identity entry (§3, type namespace): a stored type key written verbatim +// whose spelling the bundled table binds to a DIFFERENT key. `object_type` +// the custom stored key beside bundled `objectType` is exactly the §3 shadow +// shape — without the entry, a package-only reader resolves the spelling +// through the table and lands on the bundled twin. +func TestExport_TypeKeysIdentityEntryForAShadowStoredKey(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, typedSnapshot("ot-object_type"), Options{}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "object_type", doc.Type) + assert.Equal(t, map[string]string{"object_type": "object_type"}, doc.TypeKeys, + "the document's only way to say the term is a stored key, not the bundled table's objectType") + + _, snap, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-object_type"}, snap.ObjectTypes, + "a package-only reader lands on the stored key, not the bundled twin") +} + +// The point of the legend: a reader with no space gets the stored type key +// back, and the legend outranks the reader's own vocabulary. +func TestImport_TypeKeysLegendInvertsWithoutTheSpace(t *testing.T) { + doc := `{"version": 2, "type_internal_keys": {"task": "` + customTypeKey + `"}, "type": "task"}` + + t.Run("package-only reader", func(t *testing.T) { + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-" + customTypeKey}, snap.ObjectTypes) + }) + + t.Run("legend outranks the reader's vocabulary", func(t *testing.T) { + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{"readerLocalKey": "task"}} + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: vocab}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-" + customTypeKey}, snap.ObjectTypes, + "the legend is the document's own statement; a vocabulary belongs to the reader") + }) +} + +// The confirmed defect, end to end: a node-backed writer slugs a custom type, +// and the archive's consumer is a package-only reader. Before the legend the +// reader bound the slug to the bundled Task type — a different type, silently. +func TestRoundTrip_TypeSlugIsInvertibleInAPackageOnlyReader(t *testing.T) { + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "task"}} + data, err := Marshal(model.SmartBlockType_Page, typedSnapshot("ot-"+customTypeKey), Options{Keys: vocab}) + require.NoError(t, err) + + _, snap, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-" + customTypeKey}, snap.ObjectTypes, + "the document alone must invert its own spellings (§3)") +} + +// type_properties[].object_types is a type-key slot like the envelope type, +// and it owes (and reads) the same legend. +func TestTypeKeysLegendCoversObjectTypes(t *testing.T) { + t.Run("export records the entry", func(t *testing.T) { + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "task"}} + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "t1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("t1"), + "recommendedRelations": strList("rel-owner"), + }), + ObjectTypes: []string{"ot-objectType"}, + Key: "k", + } + resolver := &staticPropertyResolver{def: PropertyDefinition{ + Key: "owner", Name: "Owner", Format: model.RelationFormat_object, + ObjectTypes: []string{customTypeKey}, + }} + + data, err := Marshal(model.SmartBlockType_STType, snap, + Options{Keys: vocab, ResolveProperties: resolver}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + require.Len(t, doc.TypeProps(), 1) + assert.Equal(t, []string{"task"}, doc.TypeProps()[0].ObjectTypes) + assert.Equal(t, map[string]string{"task": customTypeKey}, doc.TypeKeys, + "a slot that writes the slug without recording the entry inverts only by luck") + }) + + t.Run("import reads the legend first", func(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_internal_keys": {"task": "` + customTypeKey + `"}, + "type_settings": {"property_definitions": [{"property": "owner", "name": "Owner", "format": "objects", + "object_types": ["task", "participant"]}]}}` + r := &recordingPropertyResolver{} + + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), ResolveProperties: r}) + require.NoError(t, err) + require.Len(t, r.defs, 1) + assert.Equal(t, []string{customTypeKey, "participant"}, r.defs[0].ObjectTypes, + "the legend inverts task; participant stays the bundled key") + }) +} + +// Decision: one ledger and one legend PER NAMESPACE. A property slug and a +// type slug may coincide without conflict (§3: `objectType` the layout value +// coexists with `object_type` the type key, and that is intended) — sharing +// one ledger would make one namespace's claim back the other namespace off +// its own slug. +func TestExportImport_PropertyAndTypeNamespacesShareATerm(t *testing.T) { + const customPropKey = "6a32d4856761631534b22f85" + vocab := typedSpaceVocabulary{ + propSlugOf: map[string]string{customPropKey: "task"}, + typeSlugOf: map[string]string{customTypeKey: "task"}, + } + snap := typedSnapshot("ot-" + customTypeKey) + snap.Details.Fields[customPropKey] = str("both namespaces spell task") + + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "task", doc.Type) + assert.Contains(t, doc.Properties, "task") + assert.Equal(t, map[string]string{"task": customPropKey}, doc.PropertyKeys) + assert.Equal(t, map[string]string{"task": customTypeKey}, doc.TypeKeys) + + _, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-" + customTypeKey}, snap2.ObjectTypes) + assert.Equal(t, "both namespaces spell task", + snap2.Details.Fields[customPropKey].GetStringValue()) +} + +// One term, one key — document-wide, per namespace: a stored type key named +// anywhere in the document always keeps its own term, so no other key's +// spelling may take it, and the contested claimant degrades through the +// ladder — its key is a minted bson id, so it takes ` ()`. +func TestExport_TypeTermLedgerBacksACollidingSlugOff(t *testing.T) { + // the envelope names customTypeKey, whose vocabulary spelling is + // `wiki_person` — but the document ALSO names the stored key + // `wiki_person` in object_types, so the spelling is taken + // (verbatim-first) and the envelope claimant degrades to the suffixed + // form, deterministic off the name and the key's own tail. + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "wiki_person"}} + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "t1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("t1"), + "recommendedRelations": strList("rel-owner"), + }), + ObjectTypes: []string{"ot-" + customTypeKey}, + Key: "k", + } + resolver := &staticPropertyResolver{def: PropertyDefinition{ + Key: "owner", Name: "Owner", Format: model.RelationFormat_object, + ObjectTypes: []string{"wiki_person"}, + }} + + data, err := Marshal(model.SmartBlockType_STType, snap, + Options{Keys: vocab, ResolveProperties: resolver}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "wiki_person (2d1a7c)", doc.Type, + "the plain spelling belongs to the stored key wiki_person; the claimant takes the suffix") + require.Len(t, doc.TypeProps(), 1) + assert.Equal(t, []string{"wiki_person"}, doc.TypeProps()[0].ObjectTypes) + assert.Equal(t, map[string]string{ + "wiki_person": "wiki_person", + "wiki_person (2d1a7c)": customTypeKey, + }, doc.TypeKeys, + "the bundled table is silent on both keys, but THIS vocabulary binds the "+ + "spelling `wiki_person` to customTypeKey — so the stored key written verbatim "+ + "owes the identity entry, or its own space reads the target type back as the "+ + "type that took its spelling; and the suffixed spelling owes its inverse, "+ + "because no shipped table has ever heard of it") + + // a package-only reader, which has no vocabulary at all + _, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-" + customTypeKey}, snap2.ObjectTypes) + + // and the writer's OWN reader, which is the one the entry is for + r := &recordingPropertyResolver{} + _, snap3, err := Unmarshal(data, Options{GenerateId: seqIds("h"), Keys: vocab, ResolveProperties: r}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-" + customTypeKey}, snap3.ObjectTypes) + require.Len(t, r.defs, 1) + assert.Equal(t, []string{"wiki_person"}, r.defs[0].ObjectTypes, + "without the entry this reads back as customTypeKey — the target type re-pointed, silently") +} + +// `template` used to be an envelope-semantic spelling: export keyed +// template_for emission off it, validation gated template_for on it, and +// import derived the smartblock kind from it, so a vocabulary that moved the +// spelling in either direction silently dropped a template's target type or +// handed the machinery to the wrong type. Since v0.22 `kind` carries all +// three, the spelling is an ordinary type term, and the vocabulary may move +// it — which the legend records and a reader inverts, exactly as for any +// other key. +// +// This is a DELETION, so what is asserted is that the same two vocabularies +// now round-trip whole, by the legend rather than by the refusal. +func TestExport_TemplateSpellingIsNoLongerReserved(t *testing.T) { + t.Run("a vocabulary may spell the template type its own way", func(t *testing.T) { + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{ + "template": "tmpl", + customTypeKey: "task", + }} + var warned []Issue + snap := typedSnapshot("ot-template", "ot-"+customTypeKey) + + data, err := Marshal(model.SmartBlockType_Template, snap, + Options{Keys: vocab, OnWarning: func(i Issue) { warned = append(warned, i) }}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "tmpl", doc.Type, "the vocabulary's spelling is honoured now") + assert.Equal(t, "template", doc.Kind, "and the kind, not the spelling, says what this is") + assert.Equal(t, "task", doc.TemplateFor, "the target type survives") + assert.Equal(t, map[string]string{"tmpl": "template", "task": customTypeKey}, doc.TypeKeys, + "the legend is what makes the moved spelling invertible") + assert.Empty(t, warned, "nothing was refused, so there is nothing to report") + + _, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-template", "ot-" + customTypeKey}, snap2.ObjectTypes) + }) + + t.Run("another key may take the spelling", func(t *testing.T) { + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "template"}} + var warned []Issue + + data, err := Marshal(model.SmartBlockType_Page, typedSnapshot("ot-"+customTypeKey), + Options{Keys: vocab, OnWarning: func(i Issue) { warned = append(warned, i) }}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "template", doc.Type, "the spelling reserves nothing") + assert.Equal(t, "page", doc.Kind, + "but the kind is spelled out anyway: `{\"type\": \"template\"}` with no kind is the "+ + "shape that used to mean a template, and the authoring subset still refuses it") + assert.Equal(t, map[string]string{"template": customTypeKey}, doc.TypeKeys) + assert.Empty(t, warned) + + sbType, snap2, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + assert.Equal(t, []string{"ot-" + customTypeKey}, snap2.ObjectTypes) + }) +} + +// The loss the change was FOR. A template's object types need not begin with +// the template key — nothing in the model requires it, and a real store holds +// such snapshots — but the second envelope slot used to exist only when +// keys[0] was the template key. So this snapshot kept one slot and its target +// type was dropped, with a warning and no way to express it. +func TestExport_ATemplateNotLedByTheTemplateKeyKeepsItsTarget(t *testing.T) { + var warned []Issue + data, err := Marshal(model.SmartBlockType_Template, typedSnapshot("ot-task", "ot-"+customTypeKey), + Options{OnWarning: func(i Issue) { warned = append(warned, i) }}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "template", doc.Kind) + assert.Equal(t, "Task", doc.Type) + assert.Equal(t, customTypeKey, doc.TemplateFor, "the target type used to be dropped here") + assert.Empty(t, warned, "and the drop used to be the only thing said about it") + + sbType, snap, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Template, sbType) + assert.Equal(t, []string{"ot-task", "ot-" + customTypeKey}, snap.ObjectTypes) +} + +// blankTypeVocab resolves a type spelling to the empty string — the type +// namespace's twin of blankKeyVocab. Unrefused, the empty key became the +// ObjectTypes entry "ot-", and the re-export then dropped the type with no +// error anywhere. +type blankTypeVocab struct{ BundledKeyVocabulary } + +func (blankTypeVocab) TypeKey(slug string) (string, bool) { + if slug == "blanktype" { + return "", true + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +func TestImport_SeamRefusesAnEmptyResolvedTypeKey(t *testing.T) { + opts := func() Options { return Options{GenerateId: seqIds("g"), Keys: blankTypeVocab{}} } + + t.Run("envelope type", func(t *testing.T) { + doc := `{"version": 2, "type": "blanktype"}` + require.NoError(t, Validate([]byte(doc)), + "the document's own chain resolves blanktype verbatim — Validate cannot see the vocabulary") + _, _, err := Unmarshal([]byte(doc), opts()) + require.Error(t, err) + var ve *ValidationError + require.ErrorAs(t, err, &ve, "the refusal is path-addressed") + assert.Contains(t, err.Error(), "/type") + }) + + t.Run("template_for", func(t *testing.T) { + doc := `{"version": 2, "kind": "template", "type": "template", "template_for": "blanktype"}` + require.NoError(t, Validate([]byte(doc))) + _, _, err := Unmarshal([]byte(doc), opts()) + require.Error(t, err) + assert.Contains(t, err.Error(), "/template_for") + }) + + t.Run("object_types", func(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "owner", "format": "objects", + "object_types": ["page", "blanktype"]}]}}` + require.NoError(t, Validate([]byte(doc))) + o := opts() + o.ResolveProperties = &recordingPropertyResolver{} + _, _, err := Unmarshal([]byte(doc), o) + require.Error(t, err) + assert.Contains(t, err.Error(), "/type_settings/property_definitions/0/object_types/1") + }) +} + +// The template_for gate and the kind read `kind`, and the type term says +// nothing about either (§2). Both used to run on the STORED key the type +// spelling resolved to through the document's own chain — legend, bundled +// table, verbatim — which is a private copy of §3 that Validate and the +// importer each had to keep, and which made the same field answer two +// unrelated questions. +func TestTemplateGateRunsOnTheKind(t *testing.T) { + t.Run("the type term does not make a template", func(t *testing.T) { + // a page whose object type IS the template type: legal, and the one + // shape the old rule could not tell apart from a template + doc := `{"version": 2, "kind": "page", "type": "template", "template_for": "page"}` + err := Validate([]byte(doc)) + require.Error(t, err, "template_for on a document whose kind is page") + assert.Contains(t, err.Error(), "/template_for") + assert.Contains(t, err.Error(), `kind "template"`) + _, _, unmErr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr, "Unmarshal agrees (I2)") + }) + + t.Run("the kind does, whatever the type is spelled", func(t *testing.T) { + doc := `{"version": 2, "kind": "template", "type_internal_keys": {"tpl": "template"}, + "type": "tpl", "template_for": "page"}` + require.NoError(t, Validate([]byte(doc))) + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Template, sbType) + assert.Equal(t, []string{"ot-template", "ot-page"}, snap.ObjectTypes) + }) + + // and a legend rebinding the spelling no longer moves the kind with it: + // the document is a template because it says so, and its type is whatever + // the legend says + t.Run("a rebound template spelling is still a template if the kind says so", func(t *testing.T) { + doc := `{"version": 2, "kind": "template", "type_internal_keys": {"template": "custom1"}, + "type": "template", "template_for": "page"}` + require.NoError(t, Validate([]byte(doc))) + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Template, sbType) + assert.Equal(t, []string{"ot-custom1", "ot-page"}, snap.ObjectTypes) + }) + + // The pre-freeze spelling — `{"type": "template"}` with no kind — had a + // refusal of its own until the freeze, because it was well-formed under + // the reading that preceded `kind` too and would otherwise have imported + // as a silent page. It declares version 1, which the version gate now + // refuses for the whole grammar (§15 #9), so the type spelling has no say + // left at all: at version 2 a kindless document is a page, and its `type` + // is only ever a type. + t.Run("a kindless document is a page whatever the type spells", func(t *testing.T) { + doc := `{"version": 2, "type": "template"}` + require.NoError(t, Validate([]byte(doc))) + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType) + assert.Equal(t, []string{"ot-template"}, snap.ObjectTypes) + }) + + // and template_for, which only a template may carry, is refused on its + // own member rather than by prescribing a kind that would change what the + // document is + t.Run("template_for on a kindless document is refused at template_for", func(t *testing.T) { + doc := `{"version": 2, "type": "template", "template_for": "task"}` + err := Validate([]byte(doc)) + require.Error(t, err, doc) + assert.Contains(t, err.Error(), "/template_for") + assert.Contains(t, err.Error(), `only valid on templates`) + _, _, unmErr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr, "Unmarshal agrees (I2): %s", doc) + }) + + // template_for names object_types[1], and there is no [1] without a [0]. + // The old gate refused this as a side effect of resolving `type`; reading + // `kind` instead, it has to be said outright or the field is discarded in + // silence. + t.Run("template_for needs a type beside it", func(t *testing.T) { + doc := `{"version": 2, "kind": "template", "template_for": "task"}` + err := Validate([]byte(doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/template_for") + _, _, unmErr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr, "Unmarshal agrees (I2)") + }) +} + +// A type legend value is a stored key and obeys the writable-key rule, like a +// property legend value (§3). +func TestValidate_TypeKeysLegendShape(t *testing.T) { + for name, doc := range map[string]string{ + "empty value": `{"version": 2, "type_internal_keys": {"t": ""}}`, + "control value": `{"version": 2, "type_internal_keys": {"t": "a` + "\\n" + `b"}}`, + "over-long value": `{"version": 2, "type_internal_keys": {"t": "` + strings.Repeat("k", 129) + `"}}`, + "empty spelling": `{"version": 2, "type_internal_keys": {"": "task"}}`, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/type_internal_keys") + }) + } + assert.NoError(t, Validate([]byte(`{"version": 2, "type_internal_keys": {"task": "`+customTypeKey+`"}}`))) +} + +// A snapshot's ObjectTypes is untrusted data, and real stores hold entries +// with no type key in them — a bare "ot-", which older builds of this very +// package wrote back whenever a vocabulary resolved a spelling to "". Such an +// entry has no spelling, so setNonEmpty omits the slot it lands in; written +// positionally, it was therefore CONTAGIOUS. An empty `type` slot makes +// `template_for` inexpressible (export keys it off the spelled term), so +// ["ot-", "ot-task"] came back as no types at all — the good sibling gone +// with the bad one, and OnWarning never called, while the import seam refuses +// exactly this shape loudly and path-addressed. +func TestExport_AKeylessObjectTypeIsDroppedAndDoesNotTakeItsSiblings(t *testing.T) { + marshal := func(t *testing.T, sbType model.SmartBlockType, ots ...string) (envelopeTypeDoc, []Issue, []string) { + t.Helper() + var warned []Issue + data, err := Marshal(sbType, typedSnapshot(ots...), + Options{OnWarning: func(i Issue) { warned = append(warned, i) }}) + require.NoError(t, err) + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + return decodeEnvelope(t, data), warned, back.ObjectTypes + } + + t.Run("a hole in front: the target type moves up into the type slot", func(t *testing.T) { + doc, warned, back := marshal(t, model.SmartBlockType_Template, "ot-", "ot-task") + + assert.Equal(t, "Task", doc.Type, "the good sibling survives its bad neighbour") + assert.Equal(t, "template", doc.Kind, + "the type term is no longer `template`, so the kind must be spelled out") + assert.Equal(t, []string{"ot-task"}, back) + require.Len(t, warned, 1, "a dropped type owes a diagnostic, as a dropped property key does") + assert.Equal(t, "/type", warned[0].Path) + assert.Contains(t, warned[0].Message, "no type key") + }) + + t.Run("a hole behind: the type survives and the drop is still reported", func(t *testing.T) { + doc, warned, back := marshal(t, model.SmartBlockType_Template, "ot-template", "ot-") + + assert.Equal(t, "Template", doc.Type) + assert.Empty(t, doc.TemplateFor, "there is no second type to name") + assert.Equal(t, []string{"ot-template"}, back) + require.Len(t, warned, 1) + assert.Equal(t, "/type", warned[0].Path) + }) + + t.Run("a hole between: template_for takes the next real entry", func(t *testing.T) { + doc, warned, back := marshal(t, model.SmartBlockType_Template, "ot-template", "ot-", "ot-"+customTypeKey) + + assert.Equal(t, "Template", doc.Type) + assert.Equal(t, customTypeKey, doc.TemplateFor) + assert.Equal(t, []string{"ot-template", "ot-" + customTypeKey}, back) + require.Len(t, warned, 1) + }) + + t.Run("nothing but holes is nothing, quietly reported", func(t *testing.T) { + doc, warned, back := marshal(t, model.SmartBlockType_Page, "ot-", "") + + assert.Empty(t, doc.Type) + assert.Empty(t, back) + assert.Len(t, warned, 2, "one per dropped entry") + }) + + t.Run("a whole list needs no warning", func(t *testing.T) { + _, warned, back := marshal(t, model.SmartBlockType_Template, "ot-template", "ot-task") + + assert.Equal(t, []string{"ot-template", "ot-task"}, back) + assert.Empty(t, warned) + }) + + // The OTHER drop, and the one that was silent: an entry with a perfectly + // good key that the envelope has no position for (§2 models one type, + // plus the target type on a template). §3 says every drop is reported + // through OnWarning, and this one was not — a user's second type left the + // archive with nothing said, in a document whose own shape gives the + // caller no way to notice. + t.Run("a keyed entry past the modelled positions is reported too", func(t *testing.T) { + doc, warned, back := marshal(t, model.SmartBlockType_Page, "ot-page", "ot-task") + + assert.Equal(t, "Page", doc.Type) + assert.Equal(t, []string{"ot-page"}, back, "the second type is not in the document") + require.Len(t, warned, 1) + assert.Equal(t, "/type", warned[0].Path) + assert.Contains(t, warned[0].Message, `object type 1 ("ot-task")`, + "the message names the entry that was lost, at the position it stood in") + assert.Contains(t, warned[0].Message, "the envelope carries one type", + "and why: there is no position for it, not that something went wrong") + }) + + t.Run("a template reports only what is past its two positions", func(t *testing.T) { + _, warned, back := marshal(t, model.SmartBlockType_Template, "ot-template", "ot-task", "ot-page") + + assert.Equal(t, []string{"ot-template", "ot-task"}, back) + require.Len(t, warned, 1, "the target type has a slot; the third entry does not") + assert.Contains(t, warned[0].Message, `object type 2 ("ot-page")`) + }) + + // the index is the one the SNAPSHOT holds, not the one the survivor took + // after closing ranks: a caller matching the warning against its own + // ObjectTypes must land on the entry that was dropped + t.Run("the reported position survives a keyless entry in front", func(t *testing.T) { + _, warned, back := marshal(t, model.SmartBlockType_Page, "ot-", "ot-page", "ot-task") + + assert.Equal(t, []string{"ot-page"}, back) + require.Len(t, warned, 2) + assert.Contains(t, warned[0].Message, `object type 0 ("ot-")`) + assert.Contains(t, warned[1].Message, `object type 2 ("ot-task")`) + }) +} + +// The type legend must name only types the document actually mentions. +// envelopeTypeTerms slugged every ObjectTypes entry, and typeSlug is the term +// ledger's CLAIM step — it records the legend entry the spelling owes — so a +// document carried a `type_internal_keys` line for a type no slot names, publishing a +// space's slug→key mapping for nothing. buildProperties cannot do this, +// because it filters before it slugs. +func TestExport_TypeLegendNamesOnlyTypesTheDocumentMentions(t *testing.T) { + vocab := typedSpaceVocabulary{typeSlugOf: map[string]string{customTypeKey: "task"}} + + t.Run("a second type no slot writes leaves no legend line", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, + typedSnapshot("ot-page", "ot-"+customTypeKey), Options{Keys: vocab}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "Page", doc.Type) + assert.Empty(t, doc.TypeKeys, + "the document never spells `task`, so it owes no entry inverting it") + assert.NotContains(t, string(data), customTypeKey, + "and the space's stored key does not appear anywhere") + }) + + // the control: when the second slot IS written, the line is owed and + // written — without this the test above would pass on a legend that never + // works at all + t.Run("a second type template_for writes keeps its legend line", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Template, + typedSnapshot("ot-template", "ot-"+customTypeKey), Options{Keys: vocab}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, "task", doc.TemplateFor) + assert.Equal(t, map[string]string{"task": customTypeKey}, doc.TypeKeys) + }) +} + +// templateMovingVocab is the hand-written third-party vocabulary the +// `template` reservation used to exist for: it answers a different stored key +// for that spelling, and binds another spelling onto the template key. No +// shipped vocabulary can produce either — storeresolver's keyMaps.key refuses +// any slug the bundled table binds elsewhere — but Options.Keys is a public +// interface, so a caller can. +type templateMovingVocab struct{ BundledKeyVocabulary } + +func (templateMovingVocab) TypeKey(slug string) (string, bool) { + switch slug { + case "template": + return customTypeKey, true + case "tpl": + return "template", true + } + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// The import half of TestExport_TemplateSpellingIsNoLongerReserved, and the +// reason the reservation could be deleted rather than merely moved. +// +// The reservation existed because two chains read the same field: the stored +// key came from the VOCABULARY, while the kind derivation and the +// /template_for gate ran through the document's own chain alone (Validate has +// no vocabulary, §12, so the importer had to agree with it). The two could +// disagree — a Template smartblock whose ObjectTypeKeys do not contain +// `template`, invisible to every downstream template check, since they all +// test lo.Contains(ObjectTypeKeys, TypeKeyTemplate). +// +// `kind` answers both questions off a field NO chain touches. So a vocabulary +// may now move the spelling as freely as it moves any other: there is only +// one resolution of `type` left, and nothing for it to contradict. +func TestImport_TheVocabularyMayMoveTheTemplateSpelling(t *testing.T) { + opts := func() Options { return Options{GenerateId: seqIds("g"), Keys: templateMovingVocab{}} } + + t.Run("the kind is the kind whatever the vocabulary answers", func(t *testing.T) { + doc := `{"version": 2, "kind": "template", "type": "template", "template_for": "task"}` + var warned []Issue + o := opts() + o.OnWarning = func(i Issue) { warned = append(warned, i) } + + sbType, snap, err := Unmarshal([]byte(doc), o) + + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Template, sbType, + "the kind is read off `kind`, not off what the vocabulary made of the type") + assert.Equal(t, []string{"ot-" + customTypeKey, "ot-task"}, snap.ObjectTypes, + "and the type is whatever the vocabulary resolved, with no reservation second-guessing it") + assert.Empty(t, warned, "there is no longer a refusal to report") + + // and it re-exports whole: the target slot is keyed off the kind, so + // a moved spelling cannot cost the target type any more + out, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err) + assert.Equal(t, "Task", decodeEnvelope(t, out).TemplateFor) + }) + + t.Run("a spelling the vocabulary binds onto the template key is just a type", func(t *testing.T) { + doc := `{"version": 2, "type": "tpl"}` + + sbType, snap, err := Unmarshal([]byte(doc), opts()) + + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Page, sbType, + "nothing about the type term makes a document a template") + assert.Equal(t, []string{"ot-template"}, snap.ObjectTypes, + "while the vocabulary's answer is taken at face value — this is a page whose type IS the template type") + }) +} + +// The one property-namespace rule the type namespace deliberately does NOT +// carry: `/properties` refuses two spellings that bind one stored key, because +// two members collapse into one details field and one of the two values is +// lost with nothing to say which. Two type slots collapse into nothing — +// ObjectTypes is an ordered list, and a repeated entry displaces no value — so +// the document is accepted. This is a decision, not an oversight: it is pinned +// here so that adding the refusal has to argue with §3 first. +func TestTypeNamespaceHasNoDuplicateBindingRefusal(t *testing.T) { + doc := `{"version": 2, "kind": "template", "type": "a", "template_for": "b", + "type_internal_keys": {"a": "template", "b": "template"}}` + + require.NoError(t, Validate([]byte(doc))) + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Template, sbType) + assert.Equal(t, []string{"ot-template", "ot-template"}, snap.ObjectTypes, + "a repeated entry is a repeated entry; nothing was displaced") +} diff --git a/pkg/lib/anyblockjson/typeproperties.go b/pkg/lib/anyblockjson/typeproperties.go new file mode 100644 index 0000000000..84581772fa --- /dev/null +++ b/pkg/lib/anyblockjson/typeproperties.go @@ -0,0 +1,585 @@ +package anyblockjson + +// typeproperties.go maps a type document's typeProperties array (§2a) to and +// from the four recommended-relation id lists on the snapshot's details. + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/gogo/protobuf/types" + "golang.org/x/text/unicode/norm" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// PropertyDefinition describes a property (relation) object — the Go side of +// the schema's one `$defs/propertyDefinition` shape, which every surface that +// describes a property references (§2a, §2d; §15 #14). Import hands the WHOLE +// decoded definition to the resolver's create path, so a member the wiring +// can store is never silently shed at the codec seam; on an existing property +// every member is inert, the same rule the §2a table states for name and +// format. +type PropertyDefinition struct { + Key domain.RelationKey + // KeyIsInternal records that the document STATED this key as its + // `internal_key`, rather than the key being a spelling (`property`, or + // one derived from `name`). + // + // A consumer that CREATES the property needs the difference and cannot + // recover it from the value: a stated internal key must be reproduced + // exactly, so a bundle re-imported elsewhere yields the same stored key, + // while a spelling must get a FRESH minted key the way the app mints one + // when a user creates a property. Judging by shape — "24 hex characters + // means minted" — gets the common case right and a hand-written 24-hex + // spelling wrong, silently. + KeyIsInternal bool + Name string + Format model.RelationFormat + // Options is the declared vocabulary of a select/multiSelect property, + // in display order (§2a). Options are otherwise only discovered from + // values that happen to be used, so a vocabulary entry no record carries + // would never exist, and minted options carry no orderId and fall back to + // sorting by name. Empty means "whatever usage produces", the pre-options + // behaviour. + Options []OptionDefinition + // ObjectTypes restricts which types an objects/files property may point + // at, in priority order, given as **type keys** — the STORED spelling on + // this struct; the document spells the display name, and the codec + // translates at the boundary like every other key slot (§3). Empty means any + // object, which is also what an untargeted property accepts — a task + // could be assigned to a random page. Listing the built-in `participant` + // alongside a bundle's own people type is what makes the current-user + // filter value available on the property (§6.2) while still allowing the + // seeded people as values. + ObjectTypes []string + // Description is the property's own description (the stored + // `description` detail on its relation object). + Description string + // IncludeTime says whether a date property's values carry a time of day + // (stored `relationFormatIncludeTime`). A pointer because absent and + // false differ: absent says nothing, false clears the flag. + IncludeTime *bool + // MaxCount bounds how many values the property holds (stored + // `relationMaxCount`); 0 is unlimited, the stored default. + MaxCount int64 + // Readonly marks the property's value as not user-writable (stored + // `relationReadonlyValue`). + Readonly bool + // DefaultValue is the value a new object receives for this property + // (stored `relationDefaultValue`), as decoded JSON — the wiring converts + // it to its store form. + DefaultValue any +} + +// OptionDefinition is one entry of a declared select vocabulary (§2a). Color +// is an Anytype option color name (util/constant.OptionColors); empty leaves +// the choice to the import wiring, which is why the canonical JSON form of a +// colorless option is the bare name rather than an object. +// +// The color belongs to the option rather than to a parallel array on the +// property so that inserting or reordering an option cannot shift it — the +// silent-failure class SPEC goal 2 exists to avoid. +type OptionDefinition struct { + Name string `json:"name"` + Color string `json:"color"` + // InternalKey is the option's stored key. It is minted, so an author + // never writes one and export writes it only where it exists. + // + // It is the only thing about an option that is derivable from nothing. + // The name and colour say what the option MEANS; the array position says + // where it sits (§2f); and the option's api key is regenerated from the + // name by the app's own rule — measured over a 77-space export, all 514 + // real option api keys are reproduced by that rule (470 by the slug, 44 + // by the transliterate fallback for names like `$$` that slug to + // nothing), so not one of them needs to travel. + InternalKey string `json:"internal_key"` +} + +// UnmarshalJSON accepts both §2a forms: a bare name, or an object carrying a +// color. Same shape as jsonCell (§6.1), minus the null and array arms. +func (o *OptionDefinition) UnmarshalJSON(data []byte) error { + if strings.HasPrefix(strings.TrimSpace(string(data)), `"`) { + return jsonUnmarshal(data, &o.Name) + } + type plain OptionDefinition // shed this method, or it recurses + return jsonUnmarshal(data, (*plain)(o)) +} + +// optionsToAny renders a declared vocabulary for export: the bare name when +// the option carries no color, an object otherwise. The string form is +// canonical whenever it qualifies, as for table cells (§6.1). +func optionsToAny(opts []OptionDefinition) []any { + var out []any + for _, o := range opts { + if o.Name == "" { + continue + } + if o.Color == "" && o.InternalKey == "" { + out = append(out, o.Name) + continue + } + m := &omap{} + m.set("name", o.Name) + m.setNonEmpty("color", o.Color) + m.setNonEmpty("internal_key", o.InternalKey) + out = append(out, m) + } + return out +} + +// optionEntryName reads the name out of either §2a option form. The semantic +// checks (§12) run on the raw document, before it decodes into +// OptionDefinition, so they need this rather than the struct. +func optionEntryName(entry any) string { + switch e := entry.(type) { + case string: + return e + case map[string]any: + name, _ := e["name"].(string) + return name + } + return "" +} + +// PropertyResolver maps property object ids to definitions on export and +// definitions back to ids on import. Creating missing properties is the +// import wiring's job (the OptionResolver contract, §3): PropertyId receives +// the full definition so the wiring can create-and-return in one step. +type PropertyResolver interface { + PropertyById(id string) (PropertyDefinition, bool) + PropertyId(def PropertyDefinition) (string, bool) +} + +// recommendedListKeys are the four detail keys typeProperties replaces, in +// the §2a canonical section order. The empty section is the regular +// (sidebar) list. +var recommendedListKeys = []struct { + detailKey string + section string +}{ + {"recommendedFeaturedRelations", "featured"}, + {"recommendedRelations", ""}, + {"recommendedFileRelations", "file"}, + {"recommendedHiddenRelations", "hidden"}, +} + +// typePropsActive reports whether this export rewrites the recommended lists +// into typeProperties: only type documents, and only with a resolver — ids +// are space-local, so without one the lists pass through in properties as +// raw ids (the same degradation as options without an option resolver). +func (e *exporter) typePropsActive() bool { + return e.isTypeDoc() && e.opts.ResolveProperties != nil +} + +// typePropDetailKeys returns the detail keys hidden from properties (and from +// the §9a legend) because typeProperties carries them, or nil when inactive. +func (e *exporter) typePropDetailKeys() map[string]bool { + if !e.typePropsActive() { + return nil + } + skip := make(map[string]bool, len(recommendedListKeys)) + for _, l := range recommendedListKeys { + skip[l.detailKey] = true + } + return skip +} + +// buildTypeProperties renders the §2a array: sections in canonical order, +// source order preserved within each list, unresolvable ids dropped. The +// array is emitted even when empty — its presence tells import to rebuild +// the four lists (as explicit empty lists) rather than leave them absent. +func (e *exporter) buildTypeProperties() []any { + if !e.typePropsActive() { + return nil + } + out := []any{} + for _, l := range recommendedListKeys { + for _, id := range valueStringList(e.detail(l.detailKey)) { + def, ok := e.resolveTypeProperty(id) + if !ok { + continue + } + // An entry whose stored key is not writable is dropped, and the + // drop is reported: the empty key a vocabulary bug once resolved + // onto (which names nothing and is invisible in every UI), and + // one past the legend bound alike. The import seam refuses such a + // key (§2a), so emitting one hands back an archive its own + // Unmarshal rejects — I1, the failure nobody sees until the + // archive is needed. Nothing can rescue it on the way out either: + // the slot carries the term verbatim when no vocabulary spells + // it, and writableSlug backs a spelling off to the stored key + // precisely when that key is unwritable. + // + // The path is the ARRAY, not an index in it: the entry is + // dropped, so it has no index, and the index the next SURVIVOR + // takes would address a healthy entry as the fault (§13). The + // message names the key, which is what identifies the drop; this + // is the same shape the property namespace reports a dropped key + // with (`/properties`). + if !writableTypePropertyKey(def) { + e.warn(typePropertyDefinitionsPath, + "property %q is dropped: %s", def.Key, + unwritableKeyReason("property key", string(def.Key))) + continue + } + m := &omap{} + // the spelling and the stored key travel side by side (§2e): + // `property` is the document-facing spelling every other key slot + // writes, `internal_key` the stored id the app minted — export + // states both, an author needs neither (identity may be a `name` + // alone) + m.set(memberProperty, e.propertySlug(string(def.Key))) + m.set(memberInternalKey, string(def.Key)) + m.setNonEmpty("name", def.Name) + m.setNonEmpty("format", formatName(def.Format)) + m.setNonEmpty("options", optionsToAny(def.Options)) + // object_types is a TYPE key slot (§3) — it names types, so it + // speaks the same vocabulary the envelope `type` does, claims its + // spellings through the same term ledger, and owes the same + // type_internal_keys legend (§3) + m.setNonEmpty("object_types", stringsToAny(e.typeSlugs(def.ObjectTypes))) + m.setNonEmpty("section", l.section) + out = append(out, m) + } + } + return out +} + +// writableTypePropertyKey reports whether buildTypeProperties will emit this +// resolved definition — the question the type-key census (seedTypeTermLedger) +// has to ask too, or it reserves the target types of an entry no slot writes +// and export stops being a fixpoint (see modelledTypeKeys). +func writableTypePropertyKey(def PropertyDefinition) bool { + return isWritablePropertyKey(string(def.Key)) +} + +// resolveTypeProperty resolves one recommended-list entry. Entries are +// normally property object ids, but legacy type objects store bare property +// KEYS (e.g. "creator") in these lists — those resolve via the reverse +// lookup or, for system properties, the bundle. +func (e *exporter) resolveTypeProperty(id string) (PropertyDefinition, bool) { + r := e.opts.ResolveProperties + if def, ok := r.PropertyById(id); ok { + return def, true + } + key := domain.RelationKey(id) + if rid, ok := r.PropertyId(PropertyDefinition{Key: key}); ok { + if def, ok := r.PropertyById(rid); ok { + return def, true + } + return PropertyDefinition{Key: key}, true + } + if rel, err := bundle.GetRelation(key); err == nil { + return PropertyDefinition{Key: key, Name: rel.Name, Format: rel.Format}, true + } + return PropertyDefinition{}, false +} + +// TypeProperty is one §2a typeProperties entry in its JSON shape — exported +// so API wiring can accept typeProperties outside a full document (the +// PATCH type surface). It is the schema's one propertyDefinition shape plus +// `section`; the five members after ObjectTypes decode so a document that +// states them reaches the resolver's create path with the whole definition +// rather than losing them at the seam. +type TypeProperty struct { + // Property is the entry's document-facing SPELLING ("Due date") — a key + // slot like any other, inverted through the legend and the vocabulary + // (§3). It is deliberately NOT called a key: the word `key` used to mean + // both this spelling and the stored id, and the split gave each its own + // name (§2e). + Property string `json:"property"` + // InternalKey is the STORED internal key, written by export for fidelity + // (the app-minted bson id of a custom property, the camelCase key of a + // bundled one). An author never needs to state one — identity is + // `property`, or `internal_key`, or a `name` the spelling derives from — + // and when none is stated for a custom property the import wiring mints a + // fresh internal key, exactly as the app does when a user creates one. + InternalKey string `json:"internal_key"` + Name string `json:"name"` + Format string `json:"format"` + Options []OptionDefinition `json:"options"` + ObjectTypes []string `json:"object_types"` + Description string `json:"description"` + IncludeTime *bool `json:"include_time"` + // json.Number for the schema-integer reason every integer field in this + // package decodes that way: 3.0 and 3e0 are integers to JSON Schema, so + // Validate accepts them, and a typed int would then fail to decode a + // document Validate declared valid. + MaxCount json.Number `json:"max_count"` + Readonly bool `json:"readonly"` + DefaultValue any `json:"default_value"` + Section string `json:"section"` +} + +// authoredKey is the identity this entry states, rewired for the +// key/spelling split: its `property` spelling, else its `internal_key`, else +// the spelling its NAME derives. The second return says which kind of term +// came back — a spelling runs through the §3 resolution chain like any +// other key slot, while an `internal_key` IS the stored key and resolves +// verbatim: a stored id is always its own address (§3), and re-entering the +// name tables could rebind it (the bundled fold takes `due_date` to +// `dueDate`, which is exactly wrong for a member whose whole meaning is +// "this exact stored key"). +// +// An identifying member used to be required in both homes of this shape, and +// that was a trap for the population the format most wants to serve. Every +// exported example is full of space-minted bson ids +// (`6a83296f61fab2265263ae34`), because export writes the keys a real space +// actually holds; an author generating a use case has no space to draw one +// from, so a required stored key asks them to INVENT an identifier whose only +// correct forms they cannot produce. What they write instead is the spelling — +// which is right, and which the name already supplies. +// +// So a name is enough, and there is nothing to DERIVE: the name IS the +// spelling. `{"name": "Cooking Time", "format": "number"}` declares a +// property spelled `Cooking Time`, and that term runs through the same +// resolution chain as a written `property`, so `{"name": "Due Date"}` +// lands on the bundled `dueDate` rather than minting a lookalike beside it. +// +// It used to run the api-slug derivation — strcase plus a transliterating +// sanitizer — and that was the one place a derived identifier survived in a +// format that has none. It did not merely rename: it TRANSLITERATED, and +// then truncated. "Cooking Time" became `cooking_time`, which no longer +// matches the name a resolver holds; "Тоггл" became `toggl`; "作業内容" +// became `zuo_ye_nei_rong`; "C++" became `c`; "☕" and "#" became the empty +// string, which the callers below then refused as an unwritable key. Every +// one of those is a legal spelling now, and each is its own address. +// +// NFC and otherwise verbatim, the same normalization every other key slot +// applies. No length bound is imposed here: the bound belongs to the +// SPELLING, and both callers already refuse a resolved key that is not +// writable (empty, over the key bound, or carrying a control character) — +// with the slot's own JSON pointer, which is a better report than a +// silently truncated term. The old derivation bounded at the object-ref +// length, 255, while a property spelling is bounded at 128; there is now +// one bound, asked in one place. +// +// `internal_key` ranks below `property` deliberately: export writes both +// from one stored key, so on its own output the two agree, and the spelling +// is the member the document's own legend speaks for. +// +// Export writes `property` and `internal_key` on every entry, so this changes +// nothing about what this package produces (§11 I1). +func (tp TypeProperty) authoredKey() (term string, isInternalKey bool) { + if tp.Property != "" { + return tp.Property, false + } + if tp.InternalKey != "" { + return tp.InternalKey, true + } + if tp.Name == "" { + return "", false + } + return norm.NFC.String(tp.Name), false +} + +// definition assembles the shared PropertyDefinition this entry declares, +// with the key slots already resolved by the caller — one builder for both +// doors the array arrives through (applyTypeProperties and +// BuildRecommendedLists), so the two cannot disagree about which members +// travel. +func (tp TypeProperty) definition(key string, format model.RelationFormat, targets []string) PropertyDefinition { + term, stated := tp.authoredKey() + return PropertyDefinition{ + Key: domain.RelationKey(key), + // authoritative when the entry STATED the key, and equally when + // resolution moved it: a legend binding a spelling to a stored key + // (§9a) is the document telling the reader which property it means, + // exactly as `internal_key` does. Only a key nothing bound — a bare + // spelling that resolved to itself — is a name awaiting a real key. + KeyIsInternal: stated || key != term, + Name: tp.Name, + Format: format, + Options: tp.Options, + ObjectTypes: targets, + Description: tp.Description, + IncludeTime: tp.IncludeTime, + MaxCount: maxCountValue(tp.MaxCount), + Readonly: tp.Readonly, + DefaultValue: tp.DefaultValue, + } +} + +// maxCountValue reads the schema-integer max_count. The schema bounds it to +// [0, 2^31-1], so the float conversion cannot truncate a valid document; an +// absent member is the zero, which is the stored default (unlimited). +func maxCountValue(n json.Number) int64 { + if n == "" { + return 0 + } + f, err := n.Float64() + if err != nil { + return 0 + } + return int64(f) +} + +type jsonTypeProperty = TypeProperty + +// RecommendedList is one of the four §2a recommended-relation lists in +// detail-key form, produced by BuildRecommendedLists. +type RecommendedList struct { + DetailKey string + Ids []string +} + +// BuildRecommendedLists resolves a typeProperties array into the four +// recommended-relation id lists (§2a), in canonical section order. All four +// lists are always present — type objects store empty sections as explicit +// empty lists. Keys the resolver cannot (or, on a dry run, will not) resolve +// pass through in place of ids, the same degradation as import (§2a). It +// carries the declared vocabulary and target types through to the resolver, +// so a property minted here is created with the same shape import gives it. +// +// It takes the full Options rather than a bare resolver because a +// typeProperties array carries KEY SLOTS — the entry identity and `objectTypes` — and this +// is the PATCH channel for the same array `applyTypeProperties` reads out of a +// document. Both must invert through the same vocabulary, or the two ways of +// writing one type's property list disagree about what a key means. +// +// The §3 chain runs from step 1, not from the caller's vocabulary: there is no +// document here, so the legend arrives through **Options.Legend** — the same +// three maps the enclosing document's envelope carries. A caller that lifted +// these spellings out of a document hands over that document's legend; a +// caller that composed them itself leaves the field zero and the chain starts +// at the vocabulary, which is what this entry point did unconditionally +// before, and is why a spelling lifted from a legend-carrying document used +// to land on whichever relation the READER'S table gave it. +// +// It refuses what applyTypeProperties refuses, on the same resolved keys and +// with the same JSON pointers, because it is the SAME array arriving through +// the other door — the API's PATCH-type channel. A vocabulary answering "" for +// a spelling is a vocabulary bug (a stale name index, a hand-rolled +// KeyVocabulary), and an unrefused one wrote the empty key straight into a +// type's recommended lists: `recommendedRelations: [""]` and +// `ObjectTypes: ["", "page"]`, both of which name nothing, are invisible in +// every UI, and re-export as a shorter list than they went in as. The document +// path has refused exactly this since the seam was written; the two doors owed +// the same answer, or the format's guarantees hold only for whichever door the +// caller happened to use. +func BuildRecommendedLists(props []TypeProperty, opts Options) ([]RecommendedList, error) { + bySection := map[string][]string{} + for i, tp := range props { + key, isInternal := tp.authoredKey() + if !isInternal { + key = opts.legendPropertyKey(key) + } + if !isWritablePropertyKey(key) { + return nil, &ValidationError{Issues: []Issue{{ + Path: fmt.Sprintf(typePropertyDefinitionsPath+"/%d/"+memberProperty, i), + Message: unwritableKeyReason("resolved property key", key), + }}} + } + // object_types is a TYPE key slot, inverted entry by entry through the + // same chain as the key above: Options.Legend's type half first — a + // PATCH caller states what its spellings mean the way a document does + // with type_internal_keys (§13.1) — then the caller's vocabulary. Resolved (and + // refused) OUTSIDE the + // resolver branch, so the verdict on a given input does not depend on + // whether the caller happened to wire a resolver — applyTypeProperties + // refuses unconditionally, and this is the same array. + var targets []string + for j, slug := range tp.ObjectTypes { + resolved := opts.legendTypeKey(slug) + if resolved == "" { + return nil, &ValidationError{Issues: []Issue{{ + Path: fmt.Sprintf(typePropertyDefinitionsPath+"/%d/object_types/%d", i, j), + Message: unwritableKeyReason("resolved type key", resolved), + }}} + } + targets = append(targets, resolved) + } + id := key + if opts.ResolveProperties != nil { + def := tp.definition(key, declaredFormatWith(opts, key, tp.Format), targets) + if resolved, ok := opts.ResolveProperties.PropertyId(def); ok { + id = resolved + } + } + bySection[tp.Section] = append(bySection[tp.Section], id) + } + out := make([]RecommendedList, 0, len(recommendedListKeys)) + for _, l := range recommendedListKeys { + ids := bySection[l.section] + if ids == nil { + ids = []string{} + } + out = append(out, RecommendedList{DetailKey: l.detailKey, Ids: ids}) + } + return out, nil +} + +// applyTypeProperties rebuilds the four recommended-relation lists from the +// document's typeProperties (§2a). Definitions resolve to property ids via +// the resolver; without one — or on a miss the wiring chose not to create — +// the key passes through in place of an id for the wiring to reconcile. The +// field's presence (even as an empty array) is the trigger: absent means the +// document does not carry the lists at all. +func (imp *importer) applyTypeProperties(details *types.Struct) error { + ts := imp.doc.TypeSettings + if ts == nil || ts.TypeProps == nil { + return nil + } + lists := map[string][]*types.Value{} + for i, tp := range *ts.TypeProps { + // the entry identity is a PROPERTY key slot, and the seam admits only keys export + // could write (§3) — the same refusal /properties makes one file over. + // The schema bounds the SPELLING (minLength 1), but a wider vocabulary + // resolves past it: PropertyKey("assignee") answering ("", true) landed + // the empty key in the type's recommended list, where it names nothing + // and disappears on re-export. Only the resolved key can be judged, + // which is why the schema cannot own this. + slot := fmt.Sprintf(typePropertyDefinitionsPath+"/%d/"+memberProperty, i) + key, isInternal := tp.authoredKey() + if !isInternal { + key = imp.propertyKeyIn(key, slot) + } + if !isWritablePropertyKey(key) { + return &ValidationError{Issues: []Issue{{ + Path: slot, + Message: unwritableKeyReason("resolved property key", key), + }}} + } + // object_types is a TYPE key slot (§2a): the document's own legend + // first, then the vocabulary — and the seam refuses a resolution + // onto the empty key, which has no written form (§3) + var targets []string + for j, slug := range tp.ObjectTypes { + slotPath := fmt.Sprintf(typePropertyDefinitionsPath+"/%d/object_types/%d", i, j) + resolved := imp.typeKey(slug, slotPath) + if resolved == "" { + return &ValidationError{Issues: []Issue{{ + Path: slotPath, + Message: unwritableKeyReason("resolved type key", resolved), + }}} + } + targets = append(targets, resolved) + } + def := tp.definition(key, imp.declaredFormat(key, tp.Format), targets) + id := key + if imp.opts.ResolveProperties != nil { + if resolved, ok := imp.opts.ResolveProperties.PropertyId(def); ok { + id = resolved + } + } + lists[tp.Section] = append(lists[tp.Section], + &types.Value{Kind: &types.Value_StringValue{StringValue: id}}) + } + // all four lists are written, empty ones included: type objects carry + // them as explicit empty lists, and leaving a key absent would break + // export∘import byte-stability for empty sections + for _, l := range recommendedListKeys { + vals := lists[l.section] + if vals == nil { + vals = []*types.Value{} + } + details.Fields[l.detailKey] = &types.Value{ + Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: vals}}, + } + } + return nil +} diff --git a/pkg/lib/anyblockjson/typeproperties_test.go b/pkg/lib/anyblockjson/typeproperties_test.go new file mode 100644 index 0000000000..c140b0d20d --- /dev/null +++ b/pkg/lib/anyblockjson/typeproperties_test.go @@ -0,0 +1,598 @@ +package anyblockjson + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +type testPropertyResolver struct { + byId map[string]PropertyDefinition + byKey map[domain.RelationKey]string +} + +func (r *testPropertyResolver) PropertyById(id string) (PropertyDefinition, bool) { + def, ok := r.byId[id] + return def, ok +} + +func (r *testPropertyResolver) PropertyId(def PropertyDefinition) (string, bool) { + id, ok := r.byKey[def.Key] + return id, ok +} + +func newTestPropertyResolver() *testPropertyResolver { + r := &testPropertyResolver{ + byId: map[string]PropertyDefinition{ + "relid-dueDate": {Key: "dueDate", Name: "Due date", Format: model.RelationFormat_date}, + "relid-assignee": {Key: "assignee", Name: "Assignee", Format: model.RelationFormat_object}, + "relid-status": {Key: "status", Name: "Status", Format: model.RelationFormat_status}, + "relid-origin": {Key: "origin", Name: "Origin", Format: model.RelationFormat_longtext}, + "relid-fileExt": {Key: "fileExt", Name: "File extension", Format: model.RelationFormat_shorttext}, + }, + byKey: map[domain.RelationKey]string{}, + } + for id, def := range r.byId { + r.byKey[def.Key] = id + } + return r +} + +func typeSnapshot() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Key: "task", + Details: fields(map[string]*types.Value{ + "id": str("typeObjectId"), + "name": str("Task"), + "recommendedFeaturedRelations": strList("relid-dueDate", "relid-assignee"), + "recommendedRelations": strList("relid-status"), + "recommendedFileRelations": strList("relid-fileExt"), + "recommendedHiddenRelations": strList("relid-origin"), + }), + Blocks: []*model.Block{ + {Id: "typeObjectId", ChildrenIds: nil, Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}, + }, + } +} + +func TestTypePropertiesExport(t *testing.T) { + t.Run("recommended lists become property_definitions in section order", func(t *testing.T) { + // given + opts := Options{ResolveProperties: newTestPropertyResolver()} + want := `"type_settings": { + "property_definitions": [ + { + "property": "Due date", + "internal_key": "dueDate", + "name": "Due date", + "format": "date", + "section": "featured" + }, + { + "property": "Assignee", + "internal_key": "assignee", + "name": "Assignee", + "format": "objects", + "section": "featured" + }, + { + "property": "Status", + "internal_key": "status", + "name": "Status", + "format": "select" + }, + { + "property": "File extension", + "internal_key": "fileExt", + "name": "File extension", + "format": "text", + "section": "file" + }, + { + "property": "Origin", + "internal_key": "origin", + "name": "Origin", + "format": "text", + "section": "hidden" + } + ] + }` + + // when + data, err := Marshal(model.SmartBlockType_STType, typeSnapshot(), opts) + + // then + require.NoError(t, err) + assert.Contains(t, string(data), want) + assert.NotContains(t, string(data), "recommendedRelations") + assert.NotContains(t, string(data), "relid-") + }) + + t.Run("unresolvable ids are dropped", func(t *testing.T) { + // given + snapshot := typeSnapshot() + snapshot.Details.Fields["recommendedRelations"] = strList("relid-status", "relid-gone") + + // when + data, err := Marshal(model.SmartBlockType_STType, snapshot, Options{ResolveProperties: newTestPropertyResolver()}) + + // then + require.NoError(t, err) + assert.Contains(t, string(data), `"property": "Status"`) + assert.NotContains(t, string(data), "relid-gone") + }) + + t.Run("no resolver keeps raw id lists in properties", func(t *testing.T) { + // when + data, err := Marshal(model.SmartBlockType_STType, typeSnapshot(), Options{}) + + // then + require.NoError(t, err) + assert.NotContains(t, string(data), "type_settings") + // the raw list keys spell their bundled display names, which say + // "properties" — the word the format uses everywhere + assert.Contains(t, string(data), "Recommended featured properties") + assert.Contains(t, string(data), "relid-dueDate") + }) + + t.Run("non-type documents never emit type_settings", func(t *testing.T) { + // given + snapshot := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{ + "id": str("pageId"), + "recommendedRelations": strList("relid-status"), + }), + } + + // when + data, err := Marshal(model.SmartBlockType_Page, snapshot, Options{ResolveProperties: newTestPropertyResolver()}) + + // then + require.NoError(t, err) + assert.NotContains(t, string(data), "type_settings") + }) + + t.Run("all-empty lists emit an explicit empty array", func(t *testing.T) { + // given: a type whose four recommended lists are all empty + snapshot := typeSnapshot() + for _, key := range []string{"recommendedFeaturedRelations", "recommendedRelations", + "recommendedFileRelations", "recommendedHiddenRelations"} { + snapshot.Details.Fields[key] = strList() + } + + // when + data, err := Marshal(model.SmartBlockType_STType, snapshot, Options{ResolveProperties: newTestPropertyResolver()}) + + // then: presence of the (empty) array is what lets import rebuild + // the four lists as explicit empty lists + require.NoError(t, err) + assert.Contains(t, string(data), `"property_definitions": []`) + }) + + t.Run("bare bundle key entries resolve via the bundle fallback", func(t *testing.T) { + // given: legacy type objects store property KEYS in the lists + snapshot := typeSnapshot() + snapshot.Details.Fields["recommendedHiddenRelations"] = strList("creator", "createdDate") + + // when + data, err := Marshal(model.SmartBlockType_STType, snapshot, Options{ResolveProperties: newTestPropertyResolver()}) + + // then + require.NoError(t, err) + assert.Contains(t, string(data), `"property": "Created by"`) + assert.Contains(t, string(data), `"property": "Creation date"`) + }) + + t.Run("a lifted recommended id is spelled out, not labelled", func(t *testing.T) { + // given: an id long enough that the old refs labeller would have + // compacted it, had it collected the lifted lists at all + snapshot := typeSnapshot() + snapshot.Details.Fields["recommendedRelations"] = strList("relid-status") + + // when + data, err := Marshal(model.SmartBlockType_STType, snapshot, Options{ + ResolveProperties: newTestPropertyResolver(), + CompactIds: true, + }) + + // then — a POSITIVE statement about what is there: an absence + // assertion on a legend that no longer exists would hold no matter + // what the export did + require.NoError(t, err) + assert.Contains(t, string(data), `"property": "Status"`, + "the lifted list resolves to a type_properties entry") + assert.NotContains(t, string(data), `"relid-status"`, + "the raw id is consumed by the lift, not carried as a label") + }) +} + +func TestTypePropertiesImport(t *testing.T) { + docJSON := `{ + "version": 2, + "kind": "object_type", + "internal_key": "task", + "properties": { "name": "Task" }, + "type_settings": {"property_definitions": [ + { "property": "due_date", "name": "Due date", "format": "date", "section": "featured" }, + { "property": "status", "name": "Status", "format": "select" }, + { "property": "origin", "section": "hidden" } + ]} +}` + + t.Run("lists rebuilt through the resolver", func(t *testing.T) { + // given + opts := Options{ResolveProperties: newTestPropertyResolver(), GenerateId: seqIds("id")} + + // when + sbType, snapshot, err := Unmarshal([]byte(docJSON), opts) + + // then + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_STType, sbType) + assert.Equal(t, strList("relid-dueDate"), snapshot.Details.Fields["recommendedFeaturedRelations"]) + assert.Equal(t, strList("relid-status"), snapshot.Details.Fields["recommendedRelations"]) + assert.Equal(t, strList("relid-origin"), snapshot.Details.Fields["recommendedHiddenRelations"]) + // empty sections come back as explicit empty lists, not absent keys + assert.Equal(t, strList(), snapshot.Details.Fields["recommendedFileRelations"]) + }) + + t.Run("unresolved keys pass through for the wiring", func(t *testing.T) { + // given + resolver := newTestPropertyResolver() + delete(resolver.byKey, "status") + opts := Options{ResolveProperties: resolver, GenerateId: seqIds("id")} + + // when + _, snapshot, err := Unmarshal([]byte(docJSON), opts) + + // then + require.NoError(t, err) + assert.Equal(t, strList("status"), snapshot.Details.Fields["recommendedRelations"]) + }) + + t.Run("empty array rebuilds all four lists as empty", func(t *testing.T) { + // given + doc := `{"version": 2, "kind": "object_type", "internal_key": "task", "type_settings": {"property_definitions": []}}` + + // when + _, snapshot, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("id")}) + + // then + require.NoError(t, err) + for _, key := range []string{"recommendedFeaturedRelations", "recommendedRelations", + "recommendedFileRelations", "recommendedHiddenRelations"} { + assert.Equal(t, strList(), snapshot.Details.Fields[key], key) + } + }) + + t.Run("absent type_properties leaves the lists absent", func(t *testing.T) { + // given + doc := `{"version": 2, "kind": "object_type", "internal_key": "task"}` + + // when + _, snapshot, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("id")}) + + // then + require.NoError(t, err) + assert.Nil(t, snapshot.Details.Fields["recommendedRelations"]) + }) + + t.Run("no resolver passes every key through", func(t *testing.T) { + // when + _, snapshot, err := Unmarshal([]byte(docJSON), Options{GenerateId: seqIds("id")}) + + // then + require.NoError(t, err) + assert.Equal(t, strList("dueDate"), snapshot.Details.Fields["recommendedFeaturedRelations"]) + assert.Equal(t, strList("status"), snapshot.Details.Fields["recommendedRelations"]) + }) +} + +func TestTypePropertiesRoundTrip(t *testing.T) { + t.Run("export import export is byte-stable", func(t *testing.T) { + // given + opts := Options{ResolveProperties: newTestPropertyResolver(), GenerateId: seqIds("id")} + first, err := Marshal(model.SmartBlockType_STType, typeSnapshot(), opts) + require.NoError(t, err) + + // when + sbType, snapshot, err := Unmarshal(first, opts) + require.NoError(t, err) + second, err := Marshal(sbType, snapshot, opts) + + // then + require.NoError(t, err) + assert.Equal(t, string(first), string(second)) + }) + + t.Run("all-empty lists are byte-stable", func(t *testing.T) { + // given: the regression case — a type with zero recommended entries + opts := Options{ResolveProperties: newTestPropertyResolver(), GenerateId: seqIds("id")} + snapshot := typeSnapshot() + for _, key := range []string{"recommendedFeaturedRelations", "recommendedRelations", + "recommendedFileRelations", "recommendedHiddenRelations"} { + snapshot.Details.Fields[key] = strList() + } + first, err := Marshal(model.SmartBlockType_STType, snapshot, opts) + require.NoError(t, err) + + // when + sbType, reimported, err := Unmarshal(first, opts) + require.NoError(t, err) + second, err := Marshal(sbType, reimported, opts) + + // then + require.NoError(t, err) + assert.Equal(t, string(first), string(second)) + assert.Equal(t, strList(), reimported.Details.Fields["recommendedRelations"]) + }) +} + +func TestTypePropertiesValidation(t *testing.T) { + t.Run("rejected outside type documents", func(t *testing.T) { + // given + doc := `{"version": 2, "type_settings": {"property_definitions": [{"property": "due_date"}]}}` + + // when + err := Validate([]byte(doc)) + + // then + require.Error(t, err) + assert.Contains(t, err.Error(), `kind "object_type"`) + }) + + t.Run("rejected alongside raw recommended lists", func(t *testing.T) { + // given + doc := `{ + "version": 2, + "kind": "object_type", + "properties": { "recommendedRelations": ["relid-status"] }, + "type_settings": {"property_definitions": [{"property": "due_date"}]} +}` + + // when + err := Validate([]byte(doc)) + + // then + require.Error(t, err) + assert.Contains(t, err.Error(), "conflicts with type_settings.property_definitions") + }) + + t.Run("unknown section and no identity at all rejected by schema", func(t *testing.T) { + for _, doc := range []string{ + `{"version": 2, "kind": "object_type", "type_settings": {"property_definitions": [{"property": "a", "section": "sidebar"}]}}`, + `{"version": 2, "kind": "object_type", "type_settings": {"property_definitions": [{"format": "number"}]}}`, + `{"version": 2, "kind": "object_type", "type_settings": {"property_definitions": [{"property": "a", "format": "status"}]}}`, + } { + assert.Error(t, Validate([]byte(doc)), strings.ReplaceAll(doc, "\n", " ")) + } + }) + + // A NAME is identity enough: an author has no way to mint the key a real + // space would, so requiring one asked for an invented identifier. + t.Run("a name alone declares a property", func(t *testing.T) { + require.NoError(t, Validate([]byte( + `{"version": 2, "kind": "object_type", "internal_key": "recipe", "type_settings": {`+ + `"property_definitions": [{"name": "Cooking Time", "format": "number"}]}}`))) + }) + + t.Run("valid type document passes", func(t *testing.T) { + doc := `{ + "version": 2, + "kind": "object_type", + "internal_key": "task", + "type_settings": {"property_definitions": [ + { "property": "due_date", "name": "Due date", "format": "date", "section": "featured" } + ]} +}` + assert.NoError(t, Validate([]byte(doc))) + }) +} + +// BuildRecommendedLists is the PATCH-types channel for the same §2a array a +// type document carries, so it must refuse what the document path refuses — +// on the same resolved keys, with the same JSON pointers. It refused nothing: +// a vocabulary answering "" for a spelling (a stale slug index, a hand-rolled +// KeyVocabulary — Options.Keys is a public interface) wrote the empty key +// straight into a type's recommended lists, where it names nothing, is +// invisible in every UI and disappears on the next export. The document path +// has refused the type half of exactly this since the seam was written; the +// property half was unrefused on BOTH paths. +func TestBuildRecommendedListsRefusesUnwritableResolvedKeys(t *testing.T) { + t.Run("a property key the vocabulary resolves onto nothing", func(t *testing.T) { + // given — PropertyKey("blank") answers ("", true) + props := []TypeProperty{{Property: "blank", Name: "Blank", Section: "featured"}} + + // when + lists, err := BuildRecommendedLists(props, Options{Keys: blankKeyVocab{}}) + + // then + require.Error(t, err, `recommendedFeaturedRelations: [""] names nothing`) + assert.Nil(t, lists) + var ve *ValidationError + require.ErrorAs(t, err, &ve, "the refusal is path-addressed, as on the document path") + assert.Contains(t, err.Error(), "/type_settings/property_definitions/0/property") + }) + + t.Run("an object_types entry the vocabulary resolves onto nothing", func(t *testing.T) { + // given — TypeKey("blanktype") answers ("", true); the good sibling + // stands first so a fix that merely skipped the bad entry is caught + props := []TypeProperty{{ + Property: "assignee", + ObjectTypes: []string{"page", "blanktype"}, + Section: "featured", + }} + + // when + lists, err := BuildRecommendedLists(props, Options{ + Keys: blankTypeVocab{}, + ResolveProperties: &recordingPropertyResolver{}, + }) + + // then + require.Error(t, err) + assert.Nil(t, lists) + assert.Contains(t, err.Error(), "/type_settings/property_definitions/0/object_types/1") + }) + + t.Run("the refusal does not depend on a resolver being wired", func(t *testing.T) { + // object_types used to be resolved only inside the resolver branch, so + // the same input got two different verdicts depending on the caller's + // wiring — while applyTypeProperties refuses unconditionally + props := []TypeProperty{{Property: "assignee", ObjectTypes: []string{"blanktype"}}} + + _, err := BuildRecommendedLists(props, Options{Keys: blankTypeVocab{}}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "/type_settings/property_definitions/0/object_types/0") + }) + + t.Run("a resolvable array still builds", func(t *testing.T) { + props := []TypeProperty{{Property: "due_date", ObjectTypes: []string{"page"}, Section: "featured"}} + + lists, err := BuildRecommendedLists(props, Options{Keys: blankTypeVocab{}}) + + require.NoError(t, err) + require.NotEmpty(t, lists) + assert.Equal(t, []string{"dueDate"}, lists[0].Ids) + }) +} + +// The document path's own property half, which had the same gap: the schema +// bounds the SPELLING (type_properties[].key is minLength 1), but only the +// RESOLVED key can be judged, and a wider vocabulary resolves past the schema. +// The type half of this entry has been refused since the seam was written +// (TestImport_SeamRefusesAnEmptyResolvedTypeKey), three lines below. +func TestImport_TypePropertyKeyRefusesAnUnwritableResolvedKey(t *testing.T) { + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [{"property": "blank", "format": "text"}]}}` + require.NoError(t, Validate([]byte(doc)), + "the document's own chain resolves blank verbatim — Validate cannot see the vocabulary") + + _, _, err := Unmarshal([]byte(doc), + Options{GenerateId: seqIds("g"), Keys: blankKeyVocab{}, ResolveProperties: &recordingPropertyResolver{}}) + + require.Error(t, err) + var ve *ValidationError + require.ErrorAs(t, err, &ve, "the refusal is path-addressed") + assert.Contains(t, err.Error(), "/type_settings/property_definitions/0/property") +} + +// The two doors into the §2a array must also agree on what a DECLARED FORMAT +// means. `text` is one name for two stored formats, and §3 makes it resolve +// per key: a key already known to be shorttext — every bundled `name`, +// `icon_emoji`, `cover_id`, and whatever the wiring's ResolveFormat +// recognizes — keeps that format, and only an unknown key becomes longtext. +// The document door ran that rule (declaredFormat); the PATCH door read the +// name literally, so the same array minted the bundled `name` property as +// longtext through one endpoint and left it shorttext through the other. +func TestTypePropertyFormatIsTheSameThroughBothDoors(t *testing.T) { + viaDocument := func(t *testing.T, opts Options, tp TypeProperty) []PropertyDefinition { + t.Helper() + r := &recordingPropertyResolver{} + opts.ResolveProperties = r + opts.GenerateId = seqIds("g") + entry := &omap{} + entry.set("property", tp.Property) + entry.setNonEmpty("name", tp.Name) + entry.setNonEmpty("format", tp.Format) + entry.setNonEmpty("section", tp.Section) + raw, err := json.Marshal(entry) + require.NoError(t, err) + doc := `{"version": 2, "kind": "object_type", "id": "t1", "internal_key": "k", + "type_settings": {"property_definitions": [` + string(raw) + `]}}` + _, _, err = Unmarshal([]byte(doc), opts) + require.NoError(t, err) + return r.defs + } + viaPatch := func(t *testing.T, opts Options, tp TypeProperty) []PropertyDefinition { + t.Helper() + r := &recordingPropertyResolver{} + opts.ResolveProperties = r + _, err := BuildRecommendedLists([]TypeProperty{tp}, opts) + require.NoError(t, err) + return r.defs + } + + cases := map[string]struct { + opts Options + tp TypeProperty + want model.RelationFormat + }{ + "a bundled short-text key keeps its stored format": { + tp: TypeProperty{Property: "name", Name: "Name", Format: "text", Section: "featured"}, + want: model.RelationFormat_shorttext, + }, + "a key the wiring resolves as short text keeps it too": { + opts: Options{ResolveFormat: func(key domain.RelationKey) (model.RelationFormat, bool) { + return model.RelationFormat_shorttext, key == "headline" + }}, + tp: TypeProperty{Property: "headline", Name: "Headline", Format: "text"}, + want: model.RelationFormat_shorttext, + }, + "a new key declared as text is long text": { + tp: TypeProperty{Property: "summary", Name: "Summary", Format: "text"}, + want: model.RelationFormat_longtext, + }, + "any other name is taken literally": { + tp: TypeProperty{Property: "name", Name: "Name", Format: "number"}, + want: model.RelationFormat_number, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + // when + doorA := viaDocument(t, tc.opts, tc.tp) + doorB := viaPatch(t, tc.opts, tc.tp) + + // then + require.Len(t, doorA, 1) + require.Len(t, doorB, 1) + assert.Equal(t, tc.want, doorA[0].Format, "the document door") + assert.Equal(t, tc.want, doorB[0].Format, "the PATCH door") + assert.Equal(t, doorA[0], doorB[0], "one array, one meaning") + }) + } +} + +// §13: a path addresses the fault. A dropped §2a entry has no index in the +// document — it is not there — and the warning used to carry +// `/type_settings/property_definitions/`, which is the index the next SURVIVING entry +// takes. So the diagnostic for the broken entry pointed at a healthy one, and +// a caller that trusted the pointer read the wrong property. +func TestExport_ADroppedTypePropertyIsReportedAtTheArray(t *testing.T) { + // given: k1's definition has no key at all (a vocabulary bug's residue, + // the shape real type objects hold), k2's is healthy + snapshot := &model.SmartBlockSnapshotBase{ + Details: fields(map[string]*types.Value{ + "id": str("t1"), + "recommendedFeaturedRelations": strList("k1", "k2"), + }), + ObjectTypes: []string{"ot-objectType"}, + } + var warned []Issue + + // when + data, err := Marshal(model.SmartBlockType_STType, snapshot, Options{ + ResolveProperties: censusPropResolver{}, + OnWarning: func(i Issue) { warned = append(warned, i) }, + }) + require.NoError(t, err) + + // then + doc := decodeEnvelope(t, data) + require.Len(t, doc.TypeProps(), 1) + assert.Equal(t, "owner", doc.TypeProps()[0].Property, "entry 0 of the document is the HEALTHY one") + require.Len(t, warned, 1) + assert.Equal(t, typePropertyDefinitionsPath, warned[0].Path, + "the array is the fault's address; the dropped entry has no index in it") + assert.Contains(t, warned[0].Message, "is dropped", + "and the message names the key, which is what identifies it") +} diff --git a/pkg/lib/anyblockjson/typesettings.go b/pkg/lib/anyblockjson/typesettings.go new file mode 100644 index 0000000000..42270ee20a --- /dev/null +++ b/pkg/lib/anyblockjson/typesettings.go @@ -0,0 +1,458 @@ +package anyblockjson + +// typesettings.go implements the §2a `type_settings` group: everything that +// defines a TYPE, in one gated subtree — the five settings members lifted +// from `properties`, plus `property_definitions` (the array that lived at +// the root as `type_properties` in an earlier revision). +// +// Nesting is not tidiness. §2d already put one root `allOf` conditional on +// the schema; five more root fields would be five more, and the eval found +// that models put `type_properties` on non-type documents precisely BECAUSE +// the root had no conditionals — while many constrained decoders do not +// implement `if`/`then` at all. One group is one conditional, and a per-kind +// generated schema includes or omits it in one move. +// +// The same change DROPS the type object's own display and provenance from a +// type document's `properties`. Each key passed the §15 #12 admission test +// individually against a 38,061-document corpus (1,760 type documents); +// the verdicts live on typeProvenanceKeys and the keys that FAILED the test +// are recorded there too. + +import ( + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The five stored detail keys the type_settings members carry. Named off the +// bundle so a rename there is a compile error here rather than a silent +// un-lift (the §2b rule). +var ( + detailKeyRecommendedLayout = bundle.RelationKeyRecommendedLayout.String() + detailKeyApiObjectKey = bundle.RelationKeyApiObjectKey.String() + detailKeyPluralName = bundle.RelationKeyPluralName.String() + detailKeyDefaultTemplateId = bundle.RelationKeyDefaultTemplateId.String() + detailKeyDefaultViewType = bundle.RelationKeyDefaultViewType.String() +) + +// typeSettingsLiftedDetailKeys is the §2a settings lift list — the single +// source for both directions, like liftedDetailKeys (§2b) and +// propertySettingsLiftedDetailKeys (§2d): export writes these keys nowhere but the +// group, and on a TYPE document import refuses their flat spellings in +// `properties`. Unlike the §2d list the refusal is KIND-SCOPED, and that is +// measured, not stylistic: `apiObjectKey` is real data on 9,725 relation and +// 525 relation-option documents, where it stays an ordinary property — the +// group exists on type documents only, so only there is a flat spelling a +// second way to write one fact. +func typeSettingsLiftedDetailKeys() map[string]bool { + return map[string]bool{ + detailKeyRecommendedLayout: true, + detailKeyApiObjectKey: true, + detailKeyPluralName: true, + detailKeyDefaultTemplateId: true, + detailKeyDefaultViewType: true, + } +} + +// typeSettingsLiftedKeyRepair names the group member a refused flat spelling +// belongs in — liftedKeyRepair's rule (§2b). +func typeSettingsLiftedKeyRepair(key string) string { + switch key { + case detailKeyRecommendedLayout: + return `"layout": ""` + case detailKeyApiObjectKey: + return `"api_key": ""` + case detailKeyPluralName: + return `"plural_name": ""` + case detailKeyDefaultTemplateId: + return `"default_template": ""` + case detailKeyDefaultViewType: + return `"default_view": ""` + } + return "" +} + +// typeProvenanceKeys are the stored details a TYPE document does not carry: +// the type object's own display and provenance, which describe the install +// rather than the type. Export omits them on type documents and import +// drops them there (stale, not wrong — the transientProperties policy). +// +// Every entry passed the §15 #12 admission test individually, against 1,760 +// corpus type documents; the map value records the verdict. Keys that were +// candidates and FAILED the test — they stay in `properties` because they +// carry something real: +// +// - `isHidden` (626 docs, all true): cannot be proven install-only — an +// integration can hide a type it minted, and §15 #12 requires proof, +// not plausibility. Its EMPTY value is already trimmed (systemtrim.go). +// - `orderId` (343 docs, 130 distinct lexids): the user's own ordering of +// types in the library. User intent, kept. +// - `layoutWidth` / `layoutAlign` (40/38 docs, 3/1 non-zero): the display +// of the type object's OWN page, set by a person where non-zero. Kept. +// - `featuredRelations` (400 docs): what this type OBJECT features — +// which differs from `section: "featured"` (what objects OF this type +// feature) in 361 of 400 corpus cases. Two things, not one. Kept. +// - `headerRelationsLayout` (51 docs): a real per-type editor setting the +// group does not model; kept in `properties` rather than half-lifted. +// - `revision` (1,455 docs): admitted at first on the reasoning that the +// system "re-runs the bundled migrations from zero and restamps it". +// It does re-run, and the re-run is not a no-op. systemobjectreviser +// guards on `bundleRevision <= localObject.GetInt64(revisionKey)`; an +// absent revision reads 0, the guard stops short-circuiting, and +// buildDiffDetails then copies the BUNDLED values over the local ones +// for every key in systemObjectFilterKeys — name, pluralName, +// recommendedLayout, isHidden, relationMaxCount among them. Measured +// on the same corpus: of 1,599 installed bundled type documents, 40 +// carry a local `name` the reviser would overwrite (key `relation` is +// locally "Relation", bundled "Property") and 36 a local plural name. +// Dropping it silently reverts a user's rename on restore. KEPT. +var typeProvenanceKeys = map[string]string{ + // 1,758 of 1,760 docs, ONE distinct value ("object_type"): derivable + // from the kind, information-free + "layout": "one distinct value, derivable from the kind", + // 1,760 of 1,760, ONE distinct value: same verdict + "resolvedLayout": "one distinct value, derivable from the kind", + // 1,608 docs, and every one also carries sourceObject — i.e. it occurs + // only on installed copies of bundled types, where the bundled table + // carries the same value + "smartblockTypes": "install artifact, restated from the bundled table", + // 1,623 docs: the bundled url this type was installed from — derivable + // from the type's own key (`_ot`) + "sourceObject": "install artifact, derivable from the type key", + // 1,693 docs, values builtin(7) 1,310 · usecase(6) 278 · import(3) 90 · + // api(9) 15 — how the INSTALL happened, not what the type is. (An + // earlier draft read enum 3 as `dragAndDrop`; that is 2, and it occurs + // zero times. The verdict stands; the value list did not.) On ordinary + // objects origin is real provenance and stays; the drop is + // type-documents-only. + "origin": "install provenance, not the type's definition", + // 1,627 docs, 1,600 of them the epoch zero (1970-01-01): an install + // timestamp at best, garbage at median + "addedDate": "install timestamp, epoch-zero on 98% of the corpus", + // 1,757 docs, and 1,756 of them hold the document's OWN id — a + // self-reference, not the pointer-to-nothing an earlier measurement + // reported (it compared raw values against bare ids while the corpus + // dump carried `#name` suffixes, so every comparison missed). The drop + // is safe for a different reason than the one first written down: + // objecttype.go:264 re-stamps it with WithForcedDetail from the + // object's own id on every init, so it is a function of the id and + // cannot survive into a new space anyway. On a SET document `setOf` is + // the collection's meaning and stays; the drop is type-documents-only. + "setOf": "the type's own id, re-stamped by WithForcedDetail on every init", +} + +// DroppedTypeProvenanceKey reports a stored detail that export omits on a +// TYPE document because it describes the install rather than the type (§2a). +// It is the exported half of the rule, for the round-trip comparator — the +// predicate is the format's own, not a copy, so the comparator and the +// exporter cannot disagree (the miss that produced 1,344 false failures in +// one sweep). +func DroppedTypeProvenanceKey(sbType model.SmartBlockType, key string) bool { + if !isTypeSmartBlock(sbType) { + return false + } + _, dropped := typeProvenanceKeys[key] + return dropped +} + +// DroppedEmptyTypeSetting reports a stored detail that export omits on a +// TYPE document because it is one of the five lifted settings and its value +// is empty: the group follows the §4 omit-empty canon — a `pluralName` of "" +// (145 corpus docs) and a `defaultTemplateId` of [] (87) say nothing a +// reader could act on, unlike the §2d members, which are the property's +// definition and mirror presence exactly. The comparator consults this +// predicate for the absent-vs-dropped-empty step, like its three siblings. +func DroppedEmptyTypeSetting(sbType model.SmartBlockType, key string, v *types.Value) bool { + if !isTypeSmartBlock(sbType) || !typeSettingsLiftedDetailKeys()[key] { + return false + } + return isEmptySystemValue(v) +} + +// isTypeSmartBlock is the SNAPSHOT-side statement of which kinds are type +// documents, and isTypeKind the DOCUMENT-side one — the same two-halves +// shape as isPropertySmartBlock/isPropertyKind (§2d), with the same rule: +// both halves and the schema's gate must name the same kinds, or one side +// emits what the other refuses. `bundled_object_type` is in the set for the +// §2d side-door reason: 0 of 38,061 corpus documents carry it, but the +// schema's `kind` enum offers it beside `object_type` with nothing marking +// it non-authorable, and a kind nothing emits is exactly the kind nobody +// thought to guard. +func isTypeSmartBlock(sbType model.SmartBlockType) bool { + return sbType == model.SmartBlockType_STType || + sbType == model.SmartBlockType_BundledObjectType +} + +// isTypeKind reports the kinds whose document IS a type. +func isTypeKind(doc map[string]any) bool { + kind, _ := doc["kind"].(string) + return kind == kindNames.name(model.SmartBlockType_STType) || + kind == kindNames.name(model.SmartBlockType_BundledObjectType) +} + +// typeSettingsOf reads the §2a group off a raw document, for the checks that +// run before it decodes — propertySettingsOf's twin. +func typeSettingsOf(doc map[string]any) (map[string]any, bool) { + raw, has := doc["type_settings"] + group, _ := raw.(map[string]any) + return group, has +} + +// typePropertyDefinitionsOf reads the property-definition list off a raw +// document. One reader for every raw-document pass, so none of them can +// keep looking at the legacy root location. +func typePropertyDefinitionsOf(doc map[string]any) ([]any, bool) { + group, _ := typeSettingsOf(doc) + raw, has := group["property_definitions"] + list, _ := raw.([]any) + return list, has +} + +// typePropertyDefinitionsPath is the JSON-pointer prefix of one +// property-definition entry — in one place because the document pass, the +// import seam and the PATCH channel must address the same slot identically. +const typePropertyDefinitionsPath = "/type_settings/property_definitions" + +// +// ---- export ---- +// + +// isTypeDoc reports whether this export carries the §2a type_settings group. +func (e *exporter) isTypeDoc() bool { + return isTypeSmartBlock(e.sbType) +} + +// buildTypeSettings renders the §2a group, or nil off a type document. The +// five settings members follow the §4 omit-empty canon (see +// DroppedEmptyTypeSetting for why the §2d mirror does not apply); +// `property_definitions` is present even when empty exactly as the root +// array was — its presence is what tells import to rebuild the four lists. +// An empty group is omitted whole, which can only happen without a property +// resolver: with one, `property_definitions` is always present. +func (e *exporter) buildTypeSettings() *omap { + if !e.isTypeDoc() { + // off a type document the five stored keys are ORDINARY properties + // and stay in `properties` — measured, not assumed: apiObjectKey is + // real data on 9,725 relation documents. The kind-scoped lift is the + // whole difference from §2d's unconditional one. + return nil + } + g := &omap{} + // both enum members read through the guarded vocabulary adapters (§3): + // the bare int32 casts they used before named NaN and fractions after + // real members — int32(NaN) is 0 on this machine, which spelt a NaN + // layout as the zero's name + g.setNonEmpty("layout", e.typeSettingEnumValue(detailKeyRecommendedLayout, "/type_settings/layout", + layoutVocabulary.has, layoutVocabulary.name)) + g.setNonEmpty("api_key", e.typeSettingString(detailKeyApiObjectKey, "/type_settings/api_key")) + g.setNonEmpty("plural_name", e.typeSettingString(detailKeyPluralName, "/type_settings/plural_name")) + g.setNonEmpty("default_template", e.typeSettingTemplate()) + g.setNonEmpty("default_view", e.typeSettingEnumValue(detailKeyDefaultViewType, "/type_settings/default_view", + viewTypeVocabulary.has, viewTypeVocabulary.name)) + if tp := e.buildTypeProperties(); tp != nil { + g.set("property_definitions", tp) // present even when empty (§2a) + } + return g +} + +// typeSettingString reads a string-valued setting; a value of another shape +// is dropped with a warning — the §2d include_time policy: there is no way +// to write it in this member. +func (e *exporter) typeSettingString(detailKey, path string) string { + v := e.detail(detailKey) + if v == nil { + return "" + } + switch k := v.GetKind().(type) { + case *types.Value_StringValue: + return k.StringValue + case *types.Value_NullValue: + return "" // a stored null carries nothing a string member could + default: + e.warn(path, "%s %v is not a string and is dropped — there is no way to write it", + detailKey, protoValueToJSON(v)) + return "" + } +} + +// typeSettingEnumValue reads a name-over-number setting: a stored number in +// the enum renders as its name, a number outside it passes through raw (the +// layout-key policy `properties` has always applied), a stored string the +// vocabulary knows passes as that name, and anything else drops with a +// warning — an unknown string may not be written, because the member's +// validation refuses unknown names (a typo silently landing on a +// number-format detail is the disease the layout rule exists for) and +// Marshal never emits what Validate rejects (§11 I1). +func (e *exporter) typeSettingEnumValue(detailKey, path string, known func(string) bool, name func(float64) string) any { + v := e.detail(detailKey) + if v == nil { + return nil + } + switch k := v.GetKind().(type) { + case *types.Value_NumberValue: + if n := name(k.NumberValue); n != "" { + return n + } + return k.NumberValue + case *types.Value_StringValue: + if known(k.StringValue) { + return k.StringValue + } + e.warn(path, "%s %q is not a name this member can hold and is dropped — there is no way to write it", + detailKey, k.StringValue) + return nil + case *types.Value_NullValue: + return nil + default: + e.warn(path, "%s %v is neither a name nor a number and is dropped — there is no way to write it", + detailKey, protoValueToJSON(v)) + return nil + } +} + +// typeSettingTemplate reads the stored defaultTemplateId — a LIST in every +// corpus document (142 of 142; 87 empty, 55 with one entry) — as the scalar +// object reference the member holds. A second entry has no written form and +// drops with a warning: 0 of 1,760 corpus type documents carry one, and a +// scalar member is what keeps the group readable as "the default template" +// rather than a list with a phantom order. +func (e *exporter) typeSettingTemplate() string { + v := e.detail(detailKeyDefaultTemplateId) + if v == nil { + return "" + } + entries := valueStringList(v) + if len(entries) == 0 { + return "" + } + if len(entries) > 1 { + e.warn("/type_settings/default_template", + "%s holds %d entries; the member is the ONE default template, so only the first is written", + detailKeyDefaultTemplateId, len(entries)) + } + return e.objectRef(entries[0]) +} + +// +// ---- import ---- +// + +// jsonTypeSettings is the decoded `type_settings` group (§2a). Layout and +// DefaultView decode as `any` because each member is a name over a stored +// number, with a raw number passing through for a value outside the enum — +// the layout-key policy `properties` has always applied. +type jsonTypeSettings struct { + Layout any `json:"layout"` + ApiKey string `json:"api_key"` + PluralName string `json:"plural_name"` + DefaultTemplate string `json:"default_template"` + DefaultView any `json:"default_view"` + TypeProps *[]jsonTypeProperty `json:"property_definitions"` // pointer: [] and absent differ (§2a) +} + +// applyTypeSettings writes the stored keys the §2a group members stand for, +// and hands `property_definitions` to the list machinery. A member the +// document omits writes nothing. +func (imp *importer) applyTypeSettings(details *types.Struct, sbType model.SmartBlockType) error { + ts := imp.doc.TypeSettings + if ts == nil { + return nil + } + if !isTypeSmartBlock(sbType) { + // the schema keeps the group off every other kind (§2a); a caller + // that skipped Validate gets the same silence as §2d + return nil + } + if v := typeSettingDetailValue(ts.Layout, func(name string) (float64, bool) { + if !layoutNames.has(name) { + return 0, false + } + return float64(layoutNames.value(name)), true + }); v != nil { + details.Fields[detailKeyRecommendedLayout] = v + } + if ts.ApiKey != "" { + details.Fields[detailKeyApiObjectKey] = &types.Value{ + Kind: &types.Value_StringValue{StringValue: ts.ApiKey}} + } + if ts.PluralName != "" { + details.Fields[detailKeyPluralName] = &types.Value{ + Kind: &types.Value_StringValue{StringValue: ts.PluralName}} + } + if ts.DefaultTemplate != "" { + // the stored shape is a list; the member is its one entry + details.Fields[detailKeyDefaultTemplateId] = &types.Value{ + Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: []*types.Value{ + {Kind: &types.Value_StringValue{StringValue: imp.objectRef(ts.DefaultTemplate)}}, + }}}} + } + if v := typeSettingDetailValue(ts.DefaultView, func(name string) (float64, bool) { + if !viewTypeNames.has(name) { + return 0, false + } + return float64(viewTypeNames.value(name)), true + }); v != nil { + details.Fields[detailKeyDefaultViewType] = v + } + return imp.applyTypeProperties(details) +} + +// typeSettingDetailValue inverts a name-over-number member: a known name +// becomes its stored number, a number passes through raw, and nil stays +// nothing. An unknown string is unreachable off a validated document — the +// member's semantic check refuses it by name — and passes through verbatim +// only as the backstop for a caller that skipped Validate. +func typeSettingDetailValue(v any, value func(string) (float64, bool)) *types.Value { + switch x := v.(type) { + case string: + if x == "" { + return nil + } + if n, known := value(x); known { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: n}} + } + return &types.Value{Kind: &types.Value_StringValue{StringValue: x}} + case float64: + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: x}} + } + return nil +} + +// definitionIdentityIssue reports a type or relation document that names no +// `internal_key`. Such a document defines something and says nothing about WHAT: the +// key is the stored identity every other document addresses it by, and +// without one the definition lands on nothing an object can be typed by or a +// property value can resolve to. +// +// A WARNING and not a refusal, because §11 I1 forbids emitting what Validate +// rejects and a snapshot's stored key is untrusted — the hostile corpus +// builds a type whose stored key is the empty string precisely because a +// 36,808-object sweep falsified a closed charset over that slot. Export must +// stay able to write whatever a snapshot holds. +// +// It earns its place under §12's first test — it catches something silent. +// Four of four schema-only runs in the small-model authoring eval wrote a +// type document with `"type": "podcast_episode"` and no `key` at all; every +// one validated, imported and round-tripped, and the type came back with no +// identity. Nothing said a word. +func definitionIdentityIssue(doc map[string]any, warn func(path, format string, args ...any)) { + kind, _ := doc["kind"].(string) + var what string + switch { + case isPropertyKind(doc): + what = "property" + case kind == kindNames.name(model.SmartBlockType_STType) || + kind == kindNames.name(model.SmartBlockType_BundledObjectType): + what = "type" + default: + return + } + if key, _ := doc[memberInternalKey].(string); key != "" { + return + } + warn("/"+memberInternalKey, "a %s document defines something and names no `internal_key` — the stored "+ + "identity every other document addresses it by. Without one the definition "+ + "imports with no identity: nothing can be typed by it, and no property value "+ + "resolves to it", what) +} diff --git a/pkg/lib/anyblockjson/typesettings_test.go b/pkg/lib/anyblockjson/typesettings_test.go new file mode 100644 index 0000000000..5b9640d4c5 --- /dev/null +++ b/pkg/lib/anyblockjson/typesettings_test.go @@ -0,0 +1,467 @@ +package anyblockjson + +// typesettings_test.go — the §2a type_settings group: the five settings +// lifted from `properties`, the kind-scoped refusal of their flat spellings, +// the install-provenance drops, and the migration story off the pre-v0.32 +// root `type_properties`. + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// settingsTypeSnapshot is a type snapshot carrying every lifted setting in +// its stored shape, plus the install provenance the document must not carry. +func settingsTypeSnapshot() *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Key: "use_case", + Details: fields(map[string]*types.Value{ + "id": str("t1"), + "name": str("Use Case"), + "recommendedLayout": num(float64(model.ObjectType_basic)), + "apiObjectKey": str("use_case"), + "pluralName": str("Use Cases"), + "defaultTemplateId": strList("bafyreitemplate"), + "defaultViewType": num(float64(model.BlockContentDataviewView_Table)), + // the provenance block (§2a): each admitted to the drop + // individually, see typeProvenanceKeys + "layout": num(float64(model.ObjectType_objectType)), + "resolvedLayout": num(float64(model.ObjectType_objectType)), + "smartblockTypes": strList("16"), + "sourceObject": strList("_otuse_case"), + "origin": num(7), + "addedDate": num(0), + "revision": num(3), + "setOf": strList("bafyreinothing"), + }), + Blocks: []*model.Block{{ + Id: "t1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + }}, + } +} + +// The five stored settings travel in the group, in their §3 spellings, and +// never in `properties` — decision 5's example, rendered. +// +// How this can fail: drop typeSettingsLiftedDetailKeys from +// envelopeLiftedKeys (the flat spellings reappear in properties, and the +// document fails its own validation there), or drop a member from +// buildTypeSettings (the fact vanishes). +func TestTypeSettings_LiftsTheFiveSettings(t *testing.T) { + // when + data, err := Marshal(model.SmartBlockType_STType, settingsTypeSnapshot(), testOptions()) + require.NoError(t, err) + + // then + for _, want := range []string{ + `"layout": "basic"`, + `"api_key": "use_case"`, + `"plural_name": "Use Cases"`, + `"default_template": "bafyreitemplate"`, + `"default_view": "table"`, + } { + assert.Contains(t, string(data), want) + } + var doc struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + for _, flat := range []string{"recommended_layout", "api_object_key", "plural_name", + "default_template_id", "default_view_type"} { + _, has := doc.Properties[flat] + assert.Falsef(t, has, "%s must not survive in properties — the group carries it (§2a)", flat) + } + require.NoError(t, Validate(data), "Marshal never emits what Validate rejects (§11 I1)") +} + +// A type document does not carry its own install provenance: the eight +// admitted keys are omitted on export and dropped on import, silently, the +// transientProperties policy scoped by kind — and on every OTHER kind the +// same keys are ordinary properties and survive. +// +// How this can fail: remove a key from typeProvenanceKeys (it reappears in +// the type document), or scope the drop wider than isTypeSmartBlock (the +// page case loses its origin). +func TestTypeSettings_ProvenanceIsDroppedOnTypeDocumentsOnly(t *testing.T) { + t.Run("omitted on export of a type document", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_STType, settingsTypeSnapshot(), testOptions()) + require.NoError(t, err) + var doc struct { + Properties map[string]any `json:"properties"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + for _, slug := range []string{"layout", "resolved_layout", "smartblock_types", + "source_object", "origin", "added_date", "set_of"} { + _, has := doc.Properties[slug] + assert.Falsef(t, has, "%s describes the install, not the type (§2a)", slug) + } + }) + + t.Run("dropped on import of a type document", func(t *testing.T) { + doc := `{"version":2,"kind":"object_type","id":"t1","internal_key":"k", + "properties":{"name":"T","origin":7,"set_of":["bafyreinothing"],"revision":3}}` + _, snap, err := Unmarshal([]byte(doc), testOptions()) + require.NoError(t, err, "a document carrying install provenance is stale, not wrong") + for _, key := range []string{"origin", "setOf"} { + assert.Nilf(t, snap.Details.Fields[key], "%s is dropped on a type document", key) + } + }) + + t.Run("kept everywhere else", func(t *testing.T) { + snap := trimSnapshot(map[string]*types.Value{ + "name": str("A page"), + "origin": num(1), + "setOf": strList("bafyreitype"), + }) + data, err := Marshal(model.SmartBlockType_Page, snap, testOptions()) + require.NoError(t, err) + assert.Contains(t, string(data), `"Origin"`, "on a page, origin is real provenance") + assert.Contains(t, string(data), `"Set of"`, "on a set, setOf is the collection's meaning") + }) +} + +// The flat spellings of the five settings are refused in `properties` ON +// TYPE DOCUMENTS, by Validate and by Unmarshal (§12 I2), with the group +// repair named — and accepted everywhere else, because the lift is +// kind-scoped: apiObjectKey is real data on 9,725 relation documents. +// +// How this can fail: drop the isTypeSmartBlock/isTypeKind gates and the +// relation case starts refusing real documents; drop the refusal itself and +// a type document gets two legal spellings for one fact. +func TestTypeSettings_FlatSpellingsAreRefusedOnTypeDocumentsOnly(t *testing.T) { + t.Run("refused on a type document", func(t *testing.T) { + doc := `{"version":2,"kind":"object_type","id":"t1","internal_key":"k", + "properties":{"plural_name":"Tasks"}}` + err := Validate([]byte(doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/properties/plural_name") + assert.Contains(t, err.Error(), "type_settings") + + _, _, err = Unmarshal([]byte(doc), testOptions()) + require.Error(t, err, "Unmarshal must refuse what Validate refuses (§12 I2)") + }) + + t.Run("accepted on a relation document", func(t *testing.T) { + doc := `{"version":2,"kind":"property","id":"r1","internal_key":"budget", + "property_settings":{"format":"number"}, + "properties":{"name":"Budget","api_object_key":"budget"}}` + require.NoError(t, Validate([]byte(doc)), + "apiObjectKey is an ordinary property off a type document") + _, snap, err := Unmarshal([]byte(doc), testOptions()) + require.NoError(t, err) + assert.Equal(t, "budget", snap.Details.Fields["apiObjectKey"].GetStringValue()) + }) +} + +// The group is legal only on type kinds, and the older spellings get their +// migration hints: the root `type_properties` names its new home, the way +// the §2d root members name theirs. +// +// How this can fail: remove the type_settings arm from the schema's allOf +// (the page case validates clean); drop the migration hint from the +// root-member special case (the type_properties case degrades to a bare +// "not allowed"). +func TestTypeSettings_GatedByKindWithMigrationHints(t *testing.T) { + for name, tc := range map[string]struct{ doc, want string }{ + "type_settings on a page": { + doc: `{"version":2,"id":"o1","type_settings":{"layout":"basic"}}`, + want: `/type_settings: property "type_settings" is only valid on type documents`, + }, + "type_settings on a relation": { + doc: `{"version":2,"kind":"property","id":"o1","internal_key":"b","property_settings":{"format":"number"},"type_settings":{}}`, + want: `property "type_settings" is only valid on type documents`, + }, + "the pre-v0.32 root type_properties": { + doc: `{"version":2,"kind":"object_type","id":"o1","internal_key":"k","type_properties":[{"key":"due_date"}]}`, + want: `property "type_properties" moved`, + }, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(tc.doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.want) + + _, _, err = Unmarshal([]byte(tc.doc), testOptions()) + assert.Error(t, err, "Unmarshal must refuse what Validate refuses (§12 I2)") + }) + } +} + +// The settings round-trip onto the same stored keys, stored shapes included: +// the scalar default_template comes back as the one-entry LIST the store +// keeps, and the view/layout names come back as their numbers. +// +// How this can fail: write defaultTemplateId back as a scalar (the store's +// list readers see nothing), or map the names through the wrong enum. +func TestTypeSettings_RoundTripsOntoTheStoredKeys(t *testing.T) { + // given + snap := settingsTypeSnapshot() + want := map[string]*types.Value{} + for k := range typeSettingsLiftedDetailKeys() { + want[k] = snap.Details.Fields[k] + } + + // when + data, err := Marshal(model.SmartBlockType_STType, snap, testOptions()) + require.NoError(t, err) + _, got, err := Unmarshal(data, testOptions()) + require.NoError(t, err) + + // then + for k, v := range want { + assert.Equal(t, v, got.Details.Fields[k], "detail %q changed on the way round", k) + } +} + +// Export ∘ Import is byte-stable over a type document with every setting +// present (§11 guarantee 2). +// +// How this can fail: any asymmetry between buildTypeSettings and +// applyTypeSettings — a member written that import drops, or one import +// rewrites into a different value — shows up as a byte diff on the second +// export. +func TestTypeSettings_ExportImportIsByteStable(t *testing.T) { + first, err := Marshal(model.SmartBlockType_STType, settingsTypeSnapshot(), testOptions()) + require.NoError(t, err) + sbType, got, err := Unmarshal(first, testOptions()) + require.NoError(t, err) + second, err := Marshal(sbType, got, testOptions()) + require.NoError(t, err) + assert.Equal(t, string(first), string(second)) +} + +// The five members follow the §4 omit-empty canon — unlike the §2d members, +// which are the property's definition and mirror presence. A pluralName of +// "" (145 corpus docs) and a defaultTemplateId of [] (87) say nothing a +// reader could act on; DroppedEmptyTypeSetting is the comparator's half of +// exactly this rule. +// +// How this can fail: emit the members unconditionally (empty strings appear +// in the group), or narrow DroppedEmptyTypeSetting so the comparator starts +// reporting the documented omission as loss. +func TestTypeSettings_EmptySettingsAreOmitted(t *testing.T) { + // given + snap := settingsTypeSnapshot() + snap.Details.Fields["pluralName"] = str("") + snap.Details.Fields["defaultTemplateId"] = strList() + + // when + data, err := Marshal(model.SmartBlockType_STType, snap, testOptions()) + require.NoError(t, err) + + // then + assert.NotContains(t, string(data), `"plural_name"`) + assert.NotContains(t, string(data), `"default_template"`) + assert.True(t, DroppedEmptyTypeSetting(model.SmartBlockType_STType, "pluralName", str("")), + "the comparator's predicate is the same rule") + assert.False(t, DroppedEmptyTypeSetting(model.SmartBlockType_Page, "pluralName", str("")), + "scoped to type documents, like the lift itself") + assert.False(t, DroppedEmptyTypeSetting(model.SmartBlockType_STType, "pluralName", str("Tasks")), + "a non-empty value is never this rule's business") +} + +// A stored defaultTemplateId with a SECOND entry has no written form: the +// member is the one default template, only the first entry is written, and +// the drop is reported — 0 of 1,760 corpus type documents carry one, so the +// warning is the only trace when the shape does appear. +// +// How this can fail: emit the whole list (the schema's scalar member fails, +// I1), or drop the warning (the second entry vanishes in silence). +func TestTypeSettings_SecondDefaultTemplateWarns(t *testing.T) { + // given + snap := settingsTypeSnapshot() + snap.Details.Fields["defaultTemplateId"] = strList("bafyreione", "bafyreitwo") + var warns []Issue + opts := testOptions() + opts.OnWarning = func(i Issue) { warns = append(warns, i) } + + // when + data, err := Marshal(model.SmartBlockType_STType, snap, opts) + require.NoError(t, err) + + // then + assert.Contains(t, string(data), `"default_template": "bafyreione"`) + assert.NotContains(t, string(data), "bafyreitwo") + require.NotEmpty(t, warns) + found := false + for _, w := range warns { + if w.Path == "/type_settings/default_template" { + found = true + assert.Contains(t, w.Message, "only the first is written") + } + } + assert.True(t, found, "the drop must be reported at the member") + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") +} + +// An unknown default_view NAME is refused like an unknown layout: a typo +// would import as a raw string onto a number-format detail, which every +// consumer reads with an int getter and silently sees as table. A raw +// NUMBER outside the enum still passes — a stored value round-trips as its +// number. +// +// The SCHEMA answers this now — `default_view` $refs the same viewType +// definition a view's own `type` does, so the two cannot drift — and the +// refusal names the vocabulary, which the semantic message did not. +// +// How this can fail: loosen either slot back to a bare string and a +// schema-driven generator emits a name the codec will refuse. +func TestTypeSettings_UnknownViewNameRefusedRawNumberPasses(t *testing.T) { + err := Validate([]byte(`{"version":2,"kind":"object_type","id":"t1","internal_key":"k", + "type_settings":{"default_view":"Table"}}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "/type_settings/default_view") + assert.Contains(t, err.Error(), "'table'", "the refusal names the vocabulary") + + _, snap, err := Unmarshal([]byte(`{"version":2,"kind":"object_type","id":"t1","internal_key":"k", + "type_settings":{"default_view":42}}`), testOptions()) + require.NoError(t, err) + assert.Equal(t, float64(42), snap.Details.Fields["defaultViewType"].GetNumberValue()) +} + +// The snapshot comparator accepts the documented type-document +// normalizations — the provenance drops and the empty-setting omissions — +// and still reports everything else, through the format's own predicates. +// +// How this can fail: teach export a drop without the predicate (every type +// document in a corpus sweep lights up — the 1,344-false-failures miss), or +// scope the predicate wider than the drop (a real loss goes quiet). +func TestTypeSettings_ComparatorPredicatesMatchTheDrops(t *testing.T) { + for key := range typeProvenanceKeys { + assert.Truef(t, DroppedTypeProvenanceKey(model.SmartBlockType_STType, key), + "%s is dropped on type documents and the comparator must know", key) + assert.Falsef(t, DroppedTypeProvenanceKey(model.SmartBlockType_Page, key), + "%s is real data off a type document", key) + } + assert.False(t, DroppedTypeProvenanceKey(model.SmartBlockType_STType, "name"), + "the predicate covers the admitted keys and nothing else") +} + +// `revision` is NOT provenance, and the difference is not cosmetic: it is the +// guard that stops SystemObjectReviser re-applying a bundled definition over +// a user's own. +// +// systemobjectreviser short-circuits on +// `bundleRevision <= localObject.GetInt64(revisionKey)`. An absent revision +// reads 0, so the guard stops firing, and buildDiffDetails then copies the +// BUNDLED values over the local ones for every key in systemObjectFilterKeys +// — name, pluralName, recommendedLayout, isHidden, relationMaxCount. +// +// Measured on 1,599 installed bundled type documents: 40 carry a local +// `name` the reviser would overwrite (key `relation` is locally "Relation", +// bundled "Property") and 36 a local plural name. Dropping revision reverts +// those renames on restore, silently. +// +// How this can fail: put "revision" back into typeProvenanceKeys and a type +// document stops carrying the marker that protects its own name. +func TestTypeSettings_RevisionIsNotProvenance(t *testing.T) { + // given a type document carrying a revision + doc := []byte(`{"version":2,"kind":"object_type","internal_key":"task", + "properties":{"name":"Task","revision":3}, + "type_settings":{"layout":"basic"}}`) + + // when + require.NoError(t, Validate(doc)) + _, snap, err := Unmarshal(doc, Options{}) + require.NoError(t, err) + + // then + assert.Contains(t, snap.GetDetails().GetFields(), "revision", + "revision guards the type's name against the bundled reviser and must survive") + assert.NotContains(t, typeProvenanceKeys, "revision", + "it failed the §15 #12 admission test — the verdict is recorded beside the list") +} + +// A definition that names no `key` has no identity. Four of four schema-only +// runs in the small-model authoring eval wrote exactly this — a type +// document with `"type": "podcast_episode"` and no `key` — and every one +// validated, imported and round-tripped with the type coming back nameless. +// +// It is a WARNING, not a refusal: §11 I1 forbids emitting what Validate +// rejects, and a snapshot's stored key is untrusted (the hostile corpus +// builds a type whose stored key is the empty string on purpose), so export +// must stay able to write one. +// +// How this can fail: drop definitionIdentityIssue and the keyless shape goes +// silent again; widen it past type and relation documents and an ordinary +// page starts warning about a key it never owed. +func TestDefinitionIdentity_AKeylessDefinitionWarns(t *testing.T) { + for name, tc := range map[string]struct { + doc string + want bool + }{ + "type document with no key": { + `{"version":2,"kind":"object_type","properties":{"name":"Podcast Episode"}}`, true}, + "relation document with no key": { + `{"version":2,"kind":"property","property_settings":{"format":"number"},` + + `"properties":{"name":"Episode"}}`, true}, + "type document WITH a key": { + `{"version":2,"kind":"object_type","internal_key":"podcast","properties":{"name":"Podcast"}}`, false}, + "an ordinary page owes no key": { + `{"version":2,"kind":"page","properties":{"name":"A page"}}`, false}, + } { + t.Run(name, func(t *testing.T) { + // when + var warned []Issue + require.NoError(t, ValidateWarn([]byte(tc.doc), func(i Issue) { warned = append(warned, i) })) + + // then + var got bool + for _, w := range warned { + if w.Path == "/internal_key" { + got = true + } + } + assert.Equal(t, tc.want, got, "warnings: %v", warned) + }) + } +} + +// The space's own object holds the space's SETTINGS — its name, icon, +// homepage — not the space itself. `space_settings` says that; `workspace` +// said something the product no longer calls anything. +// +// It is a wire value in the `kind` enum, so it is free today and a version +// bump after the freeze: one per space, 77 in a 77-space corpus, all +// machine-written and never authored. The Go smartblock type is untouched — +// only the spelling moves. +// +// No backward compatibility, per §10: the draft has no external consumers, +// and the retired spelling is refused rather than quietly accepted, so a +// document written against the old vocabulary fails loudly instead of +// importing as something else. +// +// How this can fail: point kindNames back at "workspace" and the export +// spelling changes under a reader that expects the new one; widen the schema +// enum to accept both and the retired spelling stops failing. +func TestKind_TheSpacesOwnObjectSpellsItsSettings(t *testing.T) { + // given the space's own object + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "bafyreispace", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{"id": str("bafyreispace"), "name": str("My space")}), + } + + // when + data, err := Marshal(model.SmartBlockType_Workspace, snap, testOptions()) + require.NoError(t, err) + + // then it spells the new name, and reads back as the same smartblock type + require.NoError(t, Validate(data), "I1: Marshal never emits what its own Validate rejects") + assert.Contains(t, string(data), `"kind": "space_settings"`) + sbType, _, err := Unmarshal(data, testOptions()) + require.NoError(t, err) + assert.Equal(t, model.SmartBlockType_Workspace, sbType, + "only the spelling moves; the smartblock type is unchanged") + + // and the retired spelling is refused, not silently reinterpreted + assert.Error(t, Validate([]byte(`{"version":2,"kind":"workspace","properties":{"name":"x"}}`)), + "no backward compatibility while the format is a draft — fail loudly") +} diff --git a/pkg/lib/anyblockjson/validate.go b/pkg/lib/anyblockjson/validate.go new file mode 100644 index 0000000000..2b9703d26f --- /dev/null +++ b/pkg/lib/anyblockjson/validate.go @@ -0,0 +1,2707 @@ +package anyblockjson + +// validate.go implements §12: schema validation against the embedded JSON +// Schema (draft 2020-12) plus the semantic checks the schema cannot express, +// all reported as path-addressed issues. + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" + + "github.com/santhosh-tekuri/jsonschema/v6" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/santhosh-tekuri/jsonschema/v6/kind" + "golang.org/x/text/language" + "golang.org/x/text/message" +) + +//go:embed schema/object.schema.json +var schemaJSON []byte + +// SchemaJSON returns the embedded published JSON Schema (§12). Callers must +// not mutate the returned slice; discovery surfaces (API v2 §5) serve it +// verbatim. +func SchemaJSON() []byte { + return schemaJSON +} + +const ( + // FormatVersion is the AnyBlock JSON format version this package reads + // and writes (§10). It is a single integer with no minor axis: every + // format change bumps it, and a reader rejects anything newer than its + // own while migrating anything older — with the one carve-out + // preFreezeVersion names. + FormatVersion = 2 + + // preFreezeVersion is the integer every draft carried while the grammar + // was still moving, and the one version this reader refuses outright + // (§10, §15 #9). It is NOT "older than FormatVersion": every revision of + // the pre-release grammar — the three legends that replaced `refs`, the + // relation lift, the `relation`→`property` rename — shipped under this + // same integer, so a document declaring it names no single grammar to + // migrate from. 2 is the first frozen grammar and the first this reader + // will ever migrate FROM, which is why the refusal is spelled as an + // equality and never as `v < FormatVersion`. + preFreezeVersion = 1 + + // schemaBaseURL is where the published schemas live, one directory per + // format version. + schemaBaseURL = "https://schemas.anytype.io/anyblock/" + + // maxBlockIndent is the F4 resource bound on nesting depth, mirrored by + // the schema's indent maximum. Export enforces it too — Marshal must + // never emit output its own Validate rejects. + maxBlockIndent = 32 +) + +// SchemaURL, IndexSchemaURL and PropertiesSchemaURL are the published schema +// locations written into exported documents. All are derived from +// FormatVersion so a version bump carries them along and they cannot drift +// out of sync with it; the +// $id inside each embedded schema file is checked against them by +// TestVersionIdentity, which is the one copy the compiler cannot keep honest. +var ( + SchemaURL = schemaBaseURL + strconv.Itoa(FormatVersion) + "/object.schema.json" + IndexSchemaURL = schemaBaseURL + strconv.Itoa(FormatVersion) + "/index.schema.json" + PropertiesSchemaURL = schemaBaseURL + strconv.Itoa(FormatVersion) + "/properties.schema.json" +) + +// Issue is a single path-addressed validation problem. +type Issue struct { + Path string // JSON pointer into the document, "" for the root + Message string +} + +func (i Issue) String() string { + if i.Path == "" { + return i.Message + } + return i.Path + ": " + i.Message +} + +// ValidationError aggregates every issue found in a document (§12). +type ValidationError struct { + Issues []Issue + // NewerFormat is set when the document declares a format version newer + // than this package reads, which a reader always rejects outright (§10). + NewerFormat bool +} + +func (e *ValidationError) Error() string { + var b strings.Builder + if e.NewerFormat { + b.WriteString("document was produced by a newer version of the AnyBlock format; ") + } + b.WriteString("validation failed") + for _, i := range e.Issues { + b.WriteString("\n ") + b.WriteString(i.String()) + } + return b.String() +} + +var compileSchema = sync.OnceValues(func() (*jsonschema.Schema, error) { + doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaJSON)) + if err != nil { + return nil, fmt.Errorf("decode embedded schema: %w", err) + } + c := jsonschema.NewCompiler() + if err := c.AddResource(SchemaURL, doc); err != nil { + return nil, fmt.Errorf("add schema resource: %w", err) + } + sch, err := c.Compile(SchemaURL) + if err != nil { + return nil, fmt.Errorf("compile schema: %w", err) + } + return sch, nil +}) + +// DetectFormat reports the version and $schema markers of a document without +// validating or importing it — the cheap dispatch probe for import wiring +// (§13). ok is false when data is not a JSON object carrying an integer +// version. +func DetectFormat(data []byte) (version int, schemaURL string, ok bool) { + var probe struct { + Schema string `json:"$schema"` + Version json.Number `json:"version"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return 0, "", false + } + v, ok := jsonIntValue(probe.Version) + if !ok { + return 0, "", false + } + return int(v), probe.Schema, true +} + +// Validate checks data against the embedded schema and the semantic rules +// without building a snapshot (§12). Validate is always strict; the lenient +// indent mode exists only on Unmarshal (Options.NormalizeIndent). +func Validate(data []byte) error { + _, err := validateToDoc(data, false, nil) + return err +} + +// ValidateWarn is Validate with a sink for warning-grade issues: things that +// do not make a document invalid but do mean part of it is dead weight — a +// groupBy on a view type that cannot group (§6.2), for instance. Validate +// discards them, so a tool that wants to show them must call this. +func ValidateWarn(data []byte, onWarning func(Issue)) error { + _, err := validateToDoc(data, false, onWarning) + return err +} + +// validateToDoc runs the full validation pipeline and returns the decoded +// document for the importer to consume. With lenient set, over-deep indents +// are clamped instead of rejected, each clamp reported through warn (§4). +func validateToDoc(data []byte, lenient bool, warn func(Issue)) (map[string]any, error) { + raw, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + if err != nil { + return nil, &ValidationError{Issues: []Issue{{Message: fmt.Sprintf("invalid JSON: %v", err)}}} + } + doc, ok := raw.(map[string]any) + if !ok { + return nil, &ValidationError{Issues: []Issue{{Message: "document must be a JSON object"}}} + } + if err := checkVersion(doc); err != nil { + return nil, err + } + // a bundle index or a property dictionary is a different grammar, and + // walking one through this grammar produces errors about the very members + // that make it what it is (§2c, §2f). After the version gate, which every + // grammar shares. + if issues := misroutedIssues(data, KindObject); len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + // MIGRATION SEAM: an older version is migrated forward here, between the + // version gate and schema validation. The schema pins the version to a + // const, so it doubles as the assertion that migration ran (§10). + sch, err := compileSchema() + if err != nil { + return nil, fmt.Errorf("embedded schema: %w", err) + } + // the key slots first: the schema states their rule but cannot say which + // member broke it, so this pass owns the wording and schemaIssues stays + // quiet about whatever it spoke for (see propertyNameIssues). + spoken := propertyNameIssues(doc) + // the typed envelope fields' discriminator, for the same reason: the + // schema can say `format` is missing but not that it is a CHOICE, and + // naming the alternatives at the moment the author is wrong is the whole + // reason those fields are typed rather than flat (§2b) + iconFormatIssues(doc, &spoken) + // a relation document's required `format` (§2d), same trade again: the + // schema's `required` verdict cannot list the names, and the author most + // likely to be missing it — one holding a legacy document that spelled + // `relation_format` in properties — needs the vocabulary, not the bound + propertyFormatSlotIssue(doc, &spoken) + if err := sch.Validate(doc); err != nil { + return nil, &ValidationError{Issues: append(spoken.issues, schemaIssues(err, spoken)...)} + } + if len(spoken.issues) > 0 { + // unreachable while the two statements of the rule agree; a + // divergence must still refuse the document rather than pass it + return nil, &ValidationError{Issues: spoken.issues} + } + + if issues := semanticIssues(doc, lenient, warn); len(issues) > 0 { + return nil, &ValidationError{Issues: issues} + } + return doc, nil +} + +// checkVersion rejects unsupported versions with a dedicated error naming +// both versions (§10), before schema validation gets a chance to produce a +// generic constraint failure. Two versions are unsupported for different +// reasons: anything NEWER than this reader (no forward compatibility, and the +// caller is told so through NewerFormat), and preFreezeVersion, which is +// refused as a draft rather than migrated. +func checkVersion(doc map[string]any) error { + raw, ok := doc["version"] + if !ok { + return &ValidationError{Issues: []Issue{{Path: "/version", Message: "version is required"}}} + } + num, ok := raw.(json.Number) + if !ok { + return &ValidationError{Issues: []Issue{{Path: "/version", Message: "version must be an integer"}}} + } + v, ok := jsonIntValue(num) + if !ok { + return &ValidationError{Issues: []Issue{{Path: "/version", Message: "version must be an integer"}}} + } + if v > FormatVersion { + return &ValidationError{ + NewerFormat: true, + Issues: []Issue{{ + Path: "/version", + Message: fmt.Sprintf("document version %d is newer than the supported version %d", v, FormatVersion), + }}, + } + } + if v == preFreezeVersion { + return &ValidationError{Issues: []Issue{{ + Path: "/version", + Message: fmt.Sprintf( + "version %d is the pre-freeze draft format and cannot be read: the grammar changed "+ + "more than once while %d was current, so there is no single format to migrate from. "+ + "Re-export the document to get version %d", + preFreezeVersion, preFreezeVersion, FormatVersion), + }}} + } + if v < 1 { + return &ValidationError{Issues: []Issue{{Path: "/version", Message: fmt.Sprintf("unknown version %d", v)}}} + } + return nil +} + +// jsonPath renders a schema error's instance location as the JSON pointer §12 +// promises. The library hands back RAW tokens — a member name exactly as the +// document spells it — so each is escaped before it is joined (RFC 6901), the +// same way the restated key-slot checks build theirs (propertyNameIssues). +// Joining them verbatim addressed the wrong place for any key carrying `/` or +// `~`, and cost §12's one fault, one issue on top: the schema's verdict is +// suppressed for the members those checks spoke for, and that ledger is keyed +// by pointer — so an unescaped pointer missed the escaped entry and one empty +// legend value came back three times, once as `/property_internal_keys/a~1b` and twice +// more at `/property_internal_keys/a/b`, a location the document does not have. +func jsonPath(tokens []string) string { + if len(tokens) == 0 { + return "" + } + escaped := make([]string, len(tokens)) + for i, token := range tokens { + escaped[i] = escapeJSONPointer(token) + } + return "/" + strings.Join(escaped, "/") +} + +// schemaIssues turns a jsonschema error tree into the flat, path-addressed +// issue list §12 promises. Flattening the tree verbatim does not produce that +// list: it produces the tree's own bookkeeping, in which two mechanics report +// problems the document does not have. +// +// - `unevaluatedProperties: false` (the closed-set check on blocks) only +// sees the properties that *successfully* evaluated subschemas annotated. +// When a block's type-specific subschema fails — a bad `type`, one field +// of the wrong shape — its annotations are discarded and every property of +// that block is reported unevaluated, i.e. "not allowed". So a document +// whose only fault is `"type": "bulleted_list_item"` is also told to +// remove `type` and `text`. +// - an `anyOf` reports every branch it tried. A table cell written as an +// object collects the three "wrong shape" verdicts from the string, null +// and array branches alongside the one real complaint. +// +// Both are confidently wrong rather than merely noisy, and the format's +// purpose is the generate → validate → feed-back loop: an agent told +// `property "type" is not allowed` deletes `type` and its next attempt is +// worse. So the noise is pruned here rather than explained in the spec. +func schemaIssues(err error, spoken keySlotReport) []Issue { + verr, ok := err.(*jsonschema.ValidationError) + if !ok { + return []Issue{{Message: err.Error()}} + } + printer := message.NewPrinter(language.English) + leaves := collectSchemaLeaves(verr, printer, spoken) + + // a leaf that is not an unevaluated-property verdict is a real fault, and + // it makes the closed-set verdict on its enclosing objects unreliable + realAt := map[string]bool{} + markReal := func(path string) { + for p := path; ; p = parentPath(p) { + realAt[p] = true + if p == "" { + break + } + } + } + // a fault ANOTHER pass spoke for is still a fault at that location, and + // the closed-set verdicts around it are just as unreliable. Suppressing + // the schema's own leaf without recording this made a callout whose icon + // lacks its `format` report the icon (once, well) and then also demand + // `text` and `type` be deleted — the exact confidently-wrong advice this + // pruning exists to remove. + for path := range spoken.values { + markReal(path) + } + for _, l := range leaves { + if l.unevaluated { + continue + } + markReal(l.path) + } + vocabulary := schemaPropertyNames() + out := make([]Issue, 0, len(leaves)) + for _, l := range leaves { + // "not allowed" is dropped only where it is unreliable: a name the + // schema knows somewhere, inside an object that failed for another + // reason. A name the schema never mentions is inadmissible under + // every reading, so that verdict stands and the author gets both + // facts in one round. + if l.unevaluated && vocabulary[l.property] && realAt[parentPath(l.path)] { + continue + } + out = append(out, Issue{Path: l.path, Message: l.message}) + } + return out +} + +// schemaLeaf is one rendered schema complaint plus what the pruning needs to +// know about where it came from. +type schemaLeaf struct { + path string + message string + unevaluated bool // reported by unevaluatedProperties, not by a rule + property string // the property name, for an unevaluated verdict +} + +func collectSchemaLeaves(e *jsonschema.ValidationError, printer *message.Printer, spoken keySlotReport) []schemaLeaf { + // a member propertyNameIssues already named: its verdict there carries a + // pointer and this one does not, so reporting both says the same thing + // twice, once unusably + if k, isName := e.ErrorKind.(*kind.PropertyNames); isName && spoken.names[k.Property] { + return nil + } + // `additionalProperties: false` — the closed-set check on the ENVELOPE and + // on the fixed-shape definitions — reports every unknown member of one + // object in a single verdict carried at the OBJECT's location. Flattened + // verbatim that is `additional properties 'refs' not allowed` at path "" + // for a document whose fault is one named member, which is the pathless + // verdict §12 rules out ("an issue names the member it is about"): inside a + // block the same fault comes back correctly addressed, because blocks close + // with `unevaluatedProperties`, which the library reports per member. So + // the verdict is split into one leaf per member here, each at its own + // pointer, and the names are sorted because the library collects them by + // ranging over the instance's map — two unknown members otherwise came back + // in a different order run to run. + // + // Unlike an unevaluated-property verdict these are never pruned, and that + // is not an oversight: `additionalProperties` consults `properties` and + // `patternProperties` of the SAME schema object, which always evaluate, so + // its verdict does not depend on a sibling subschema having succeeded — the + // unreliability the pruning exists for cannot arise here. + if k, isAdditional := e.ErrorKind.(*kind.AdditionalProperties); isAdditional { + at := jsonPath(e.InstanceLocation) + props := append([]string(nil), k.Properties...) + sort.Strings(props) + out := make([]schemaLeaf, 0, len(props)) + for _, prop := range props { + msg := unknownPropertyMessage(prop) + // the legacy relation-definition spellings, at the ROOT only + // — anywhere else (a view, a sort) the same names are ordinary + // unknown members and the hint would mislead. Same reasoning as + // `refs` (§10): told only "not allowed", the obvious wrong + // repair is to delete the definition rather than regroup it. + if at == "" { + switch prop { + case "format", "include_time", "object_types": + msg = fmt.Sprintf("property %q moved off the root: a property document "+ + "states its definition in the \"property_settings\" group — "+ + "move it (and its two siblings, if present) in there", prop) + case "type_properties": + msg = `property "type_properties" moved: a type document states its ` + + `definitions in "type_settings" and this array is its ` + + `"property_definitions" member — move it in there` + } + } + out = append(out, schemaLeaf{ + path: at + "/" + escapeJSONPointer(prop), + message: msg, + }) + } + return out + } + if len(e.Causes) == 0 { + l := schemaLeaf{path: jsonPath(e.InstanceLocation), message: schemaIssueMessage(e, printer)} + if spoken.values[l.path] { + return nil // same value, already reported by name and by rule + } + if strings.Contains(e.SchemaURL, "/unevaluatedProperties") { + l.unevaluated = true + if toks := e.InstanceLocation; len(toks) > 0 { + l.property = toks[len(toks)-1] + } + } + return []schemaLeaf{l} + } + switch e.ErrorKind.(type) { + case *kind.AnyOf, *kind.OneOf: + return branchLeaves(e, printer, spoken) + } + var out []schemaLeaf + for _, c := range e.Causes { + out = append(out, collectSchemaLeaves(c, printer, spoken)...) + } + return out +} + +// branchLeaves reports the alternatives of an anyOf/oneOf. A branch whose only +// complaint is the instance's own type never applied — the author did not write +// a string where this branch wanted a string — so reporting it says nothing +// about the document. When some branch did apply, only those are reported; +// when none did, the shape is wrong and the alternatives merge into one issue +// naming all of them, which is the whole content of a failed anyOf. +func branchLeaves(e *jsonschema.ValidationError, printer *message.Printer, spoken keySlotReport) []schemaLeaf { + at := jsonPath(e.InstanceLocation) + var applied []schemaLeaf + var inapplicable []*kind.Type + for _, c := range e.Causes { + leaves := collectSchemaLeaves(c, printer, spoken) + // a branch that failed only on the instance's own type is a branch + // the instance was never a candidate for + types := branchTypeErrors(c) + if len(types) == len(leaves) && allAt(leaves, at) { + inapplicable = append(inapplicable, types...) + continue + } + applied = append(applied, leaves...) + } + if len(applied) > 0 { + return applied + } + if len(inapplicable) == 0 { + // nothing to merge and nothing applied: report the tree verbatim + // rather than swallow the failure into an error with no issues + var out []schemaLeaf + for _, c := range e.Causes { + out = append(out, collectSchemaLeaves(c, printer, spoken)...) + } + return out + } + want := make([]string, 0, len(inapplicable)) + for _, t := range inapplicable { + want = append(want, t.Want...) + } + return []schemaLeaf{{ + path: at, + message: fmt.Sprintf("got %s, want %s", inapplicable[0].Got, strings.Join(dedupe(want), ", ")), + }} +} + +func allAt(leaves []schemaLeaf, path string) bool { + for _, l := range leaves { + if l.path != path { + return false + } + } + return true +} + +// branchTypeErrors returns the type mismatches of one anyOf branch, and +// nothing when the branch failed for any other reason. +func branchTypeErrors(e *jsonschema.ValidationError) []*kind.Type { + if t, isType := e.ErrorKind.(*kind.Type); isType { + return []*kind.Type{t} + } + var out []*kind.Type + for _, c := range e.Causes { + out = append(out, branchTypeErrors(c)...) + } + return out +} + +func dedupe(in []string) []string { + out := make([]string, 0, len(in)) + seen := map[string]bool{} + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} + +// parentPath returns the JSON pointer of the container holding path. +func parentPath(path string) string { + if i := strings.LastIndex(path, "/"); i >= 0 { + return path[:i] + } + return "" +} + +// schemaPropertyNames is every property name the embedded schema mentions +// anywhere. It answers one question: could this name have been admitted under +// some reading of the schema? A name that is absent could not, whatever else +// failed — which is what makes the "not allowed" verdict on it trustworthy. +var schemaPropertyNames = sync.OnceValue(func() map[string]bool { + names := map[string]bool{} + var doc any + if err := json.Unmarshal(schemaJSON, &doc); err != nil { + return names + } + var walk func(node any) + walk = func(node any) { + switch n := node.(type) { + case map[string]any: + for key, v := range n { + if key == "properties" { + if props, isMap := v.(map[string]any); isMap { + for name, sub := range props { + names[name] = true + walk(sub) + } + continue + } + } + walk(v) + } + case []any: + for _, v := range n { + walk(v) + } + } + } + walk(doc) + return names +}) + +// schemaFormatEnum is the list of variants a typed field's `format` member +// admits, read out of the published schema by definition name. Reading it +// rather than restating it is what makes the extension seam (§2b) free: a +// layer that appends a variant to the schema gets it named in the reader's +// own diagnostics without touching this package. +// +// Every `enum` the definition carries at a `properties/format` position is +// intersected, so a definition that narrows another one by $ref plus a +// second enum (plainIcon) answers with the narrowed set. +func schemaFormatEnum(def string) []string { + return schemaFormatEnums()[def] +} + +var schemaFormatEnums = sync.OnceValue(func() map[string][]string { + out := map[string][]string{} + var doc struct { + Defs map[string]any `json:"$defs"` + } + if err := json.Unmarshal(schemaJSON, &doc); err != nil { + return out + } + for name, def := range doc.Defs { + var found [][]string + var walk func(node any, inFormat bool) + walk = func(node any, inFormat bool) { + switch n := node.(type) { + case map[string]any: + for key, v := range n { + switch { + case inFormat && key == "enum": + if list, isList := v.([]any); isList { + names := make([]string, 0, len(list)) + for _, e := range list { + if s, isStr := e.(string); isStr { + names = append(names, s) + } + } + if len(names) > 0 { + found = append(found, names) + } + } + case key == "properties": + if props, isMap := v.(map[string]any); isMap { + for prop, sub := range props { + walk(sub, prop == "format") + } + continue + } + walk(v, false) + default: + walk(v, false) + } + } + case []any: + for _, v := range n { + walk(v, inFormat) + } + } + } + walk(def, false) + if len(found) == 0 { + continue + } + // the intersection, in the order of the first list found. Map + // iteration decides which that is when a definition carries two, so + // the lists are sorted by length first: the narrowest is the answer, + // and narrowing is the only reason a second one exists. + sort.SliceStable(found, func(i, j int) bool { return len(found[i]) < len(found[j]) }) + keep := map[string]int{} + for _, list := range found { + for _, n := range list { + keep[n]++ + } + } + var names []string + for _, n := range found[0] { + if keep[n] == len(found) { + names = append(names, n) + } + } + out[name] = names + } + return out +}) + +// propertyFormatEnum is the §3 format-name vocabulary as the published +// schema states it ($defs/propertyFormat) — read rather than restated, the +// schemaFormatEnums rule: the schema is the one statement an external +// validator also sees, so the reader's diagnostics quote it instead of a +// second list that can drift. +var propertyFormatEnum = sync.OnceValue(func() []string { + var doc struct { + Defs map[string]struct { + Enum []any `json:"enum"` + } `json:"$defs"` + } + if err := json.Unmarshal(schemaJSON, &doc); err != nil { + return nil + } + var names []string + for _, e := range doc.Defs["propertyFormat"].Enum { + if s, isStr := e.(string); isStr { + names = append(names, s) + } + } + return names +}) + +// schemaIssueMessage renders one schema error. Unknown properties fail +// against a `false` schema (unevaluatedProperties / removed keys), whose +// stock text "false schema" names neither the rule nor the key — rewrite it +// to name the property, through the same renderer the envelope's closed-set +// verdict uses, so a removed key reads the same wherever it is written. +func schemaIssueMessage(e *jsonschema.ValidationError, printer *message.Printer) string { + if _, isFalse := e.ErrorKind.(*kind.FalseSchema); isFalse { + if toks := e.InstanceLocation; len(toks) > 0 { + prop := toks[len(toks)-1] + if _, err := strconv.Atoi(prop); err != nil { // numeric = array index, not a property + // the §2d group is declared at the root and gated by kind, so + // its false schema fires only OFF a relation document — and + // only at the root (len == 1), where the member name is + // unambiguous. "not allowed" would send the author toward + // deleting the group; the actual repair is the kind. + if len(toks) == 1 && prop == memberPropertySettings { + return fmt.Sprintf("property %q is only valid on property documents "+ + `(kind "property")`, prop) + } + if len(toks) == 1 && prop == "type_settings" { + return fmt.Sprintf("property %q is only valid on type documents "+ + `(kind "object_type")`, prop) + } + // inside the group, the refused members each have a home + // already (§2d): telling the author only "not allowed" sends + // them toward deleting the fact instead of moving it. + if len(toks) == 2 && toks[0] == memberPropertySettings { + if home, owned := propertySettingsMemberHomes[prop]; owned { + return fmt.Sprintf("property_settings does not carry %q — %s", prop, home) + } + } + return unknownPropertyMessage(prop) + } + } + } + return e.ErrorKind.LocalizedString(printer) +} + +// unknownPropertyMessage names a member no reading of the schema admits, and +// carries a migration hint for the three names a document written against an +// older grammar brings. The hints exist because the bare verdict sends the +// reader the wrong way, and the format's purpose is the generate → validate → +// feed-back loop (§13): +// +// - `key` is the legacy spelling of the property-naming slot in a +// dataview's `properties[]` and the `property` block — 95,842 slots of a +// 28,599-document export spell it, so an agent prompted on old exports +// WILL write it. Told only "not allowed", the obvious wrong repair is to +// delete the member, which costs the block the one thing it says. +// - `children` is what every nested-era generator writes; told only that it +// is not allowed, the obvious repair is to drop the subtree rather than to +// flatten it into `indent` (§4). +// - `refs` is the object-reference legend this format used to carry (§9a). +// Told only that it is not allowed, the obvious repair is to delete the +// legend — which leaves behind exactly the short labels the legend was the +// only means of inverting, now addressing nothing. The version integer +// cannot say this either: the grammar changed under a version this reader +// still accepts, so `refs` is the one marker a legacy document carries, +// and the message is where the reader is told what happened. +// +// propertySettingsMemberHomes names, for each propertyDefinition member the +// §2d group refuses, where the fact it spells already lives — the repair the +// bare "not allowed" cannot point at. +var propertySettingsMemberHomes = map[string]string{ + "internal_key": "the envelope `internal_key` is the property's stored key", + "property": "a property document is addressed by its envelope `internal_key`; its spelling is its display name, which the `name` property already carries", + "name": "the property's name is the `name` property", + "description": "the property's description is the `description` property", + "options": "a property's options are property_option documents of their own", + "max_count": "it still travels in `properties` as \"Max values\"", + "readonly": "it still travels in `properties` as \"Property value is readonly\"", + "default_value": "it still travels in `properties` as \"Default value\"", +} + +func unknownPropertyMessage(prop string) string { + switch prop { + case "key": + return `property "key" is not allowed — the member that names a property is spelled "property" in every structure: a dataview's properties[] entry and the property block spelled it "key" earlier, twelve lines from view columns, sorts and filters that spelled "property". Rename the member and keep its value` + case "children": + return `property "children" is not allowed — the flat format has no children; nest with indent instead` + case "refs": + return `property "refs" is not allowed — the object-reference legend was removed: every object id is now written in full, on every shape, with no legend. This document was written by an older exporter; replace each short label it uses with the id "refs" maps that label to, then drop "refs". Dropping it alone leaves labels that address nothing` + } + return fmt.Sprintf("property %q is not allowed", prop) +} + +// textBearing reports whether the block type's text is parsed for inline +// markup; code/embed text is literal (§8.4). +func textBearing(typ string) bool { + switch typ { + case "paragraph", "heading_1", "heading_2", "heading_3", "heading_4", "header_4", + "quote", "checkbox", "bulleted_list_item", "numbered_list_item", "toggle", + "callout", "toggle_heading_1", "toggle_heading_2", "toggle_heading_3", + "title", "description": + return true + } + return false +} + +// leafBlockTypes are the block types that cannot be parents (V2) — the same +// list as the export side's withChildren = false sites and the editor's +// leaf blocks, plus the equation input alias. +var leafBlockTypes = map[string]bool{ + "embed": true, "equation": true, "bookmark": true, "link": true, + "divider": true, "table": true, "property": true, "dataview": true, + "icon": true, "table_of_contents": true, "featured_properties": true, + "chat": true, +} + +// LeafBlockType reports whether typ cannot be a parent (§5 leaf types, the +// V2 containment check). Exported for wiring that pre-checks edits before a +// full document validation (API v2 Phase 3). +func LeafBlockType(typ string) bool { + return leafBlockTypes[typ] +} + +// TextBlockType reports whether typ carries a `text` prop — the §5 +// text-bearing styles plus the literal-text blocks (`code`, `embed` and its +// `equation` alias, §8.4). Exported for the same wiring as LeafBlockType. +func TextBlockType(typ string) bool { + switch typ { + case "code", "embed", "equation": + return true + } + return textBearing(typ) +} + +// clampIndents applies the §4 lenient rule in place: an indent more than one +// deeper than its predecessor clamps to predecessor+1 (CommonMark's "a level +// that hasn't been established cannot be opened"); the first entry's +// predecessor is base. onClamp, when non-nil, receives each clamp. +func clampIndents(indents []int, base int, onClamp func(i, from, to int)) { + prev := base + for i, k := range indents { + if k > prev+1 { + if onClamp != nil { + onClamp(i, k, prev+1) + } + k = prev + 1 + indents[i] = k + } + prev = k + } +} + +// jsonIntValue reads a json.Number as an integer, accepting integer-valued +// floats like 2.0 and 1e0 — JSON Schema numeric equality treats them as +// integers, so every reader of a schema-integer field must too. +func jsonIntValue(num json.Number) (int64, bool) { + v, err := num.Int64() + if err == nil { + return v, true + } + f, ferr := num.Float64() + if ferr != nil || f != math.Trunc(f) { + return 0, false + } + return int64(f), true +} + +// jsonInt64 and jsonInt32 read a schema-integer field into the stored type. +// They are the decode-side half of the agreement rule: the schema admits +// integer-valued floats and bounds each field to its stored type's range, so +// these accept exactly what it accepts, and an absent field (the zero +// json.Number) reads as 0. +// +// The clamp is unreachable while the schema carries the bounds — Unmarshal +// always validates first — and is here so that a bound removed from the schema +// costs a wrong pixel width rather than a wrapped negative one. +func jsonInt64(num json.Number) int64 { + v, _ := jsonIntValue(num) + return v +} + +func jsonInt32(num json.Number) int32 { + v, ok := jsonIntValue(num) + if !ok { + return 0 + } + if v > math.MaxInt32 { + return math.MaxInt32 + } + if v < math.MinInt32 { + return math.MinInt32 + } + return int32(v) +} + +// indentOf reads a block's indent; absent means 0. The schema guarantees an +// integer in [0, 32] (V4) when present, which includes integer-valued +// floats — jsonIntValue keeps this reader in agreement with the schema and +// with Unmarshal. +func indentOf(block map[string]any) int { + raw, ok := block["indent"] + if !ok { + return 0 + } + num, ok := raw.(json.Number) + if !ok { + return 0 + } + v, ok := jsonIntValue(num) + if !ok { + return 0 + } + return int(v) +} + +// semanticIssues runs the checks the schema cannot express: envelope +// combinations, indent monotonicity and containment over the flat blocks +// array (V1–V3), id uniqueness over the flattened tree including derived +// table cell ids, table arity, language-vs-fields.lang conflicts, and inline +// markup grammar (§12). With lenient set, V1 violations clamp (reported via +// warn) instead of erroring; V2/V3 are evaluated on the clamped indents and +// stay errors. +func semanticIssues(doc map[string]any, lenient bool, warn func(Issue)) []Issue { + var issues []Issue + addIssue := func(path, format string, args ...any) { + issues = append(issues, Issue{Path: path, Message: fmt.Sprintf(format, args...)}) + } + warnIssue := func(path, format string, args ...any) { + if warn != nil { + warn(Issue{Path: path, Message: fmt.Sprintf(format, args...)}) + } + } + + // Every number in the document has to land in a float64: the loose surfaces + // (§3 properties, block fields, store, filter values) decode into a proto + // Struct, whose numbers are doubles, and the schema cannot bound them + // without closing surfaces the format deliberately leaves open. Left + // unchecked, Validate accepted 1e400 and Unmarshal then failed with a bare + // Go decode error carrying no JSON pointer — the divergence §12 rules out. + checkNumbers(doc, "", addIssue) + + // Key spellings are display names, carried exactly as the space holds + // them — and a name can hold what nobody can see: edge whitespace or a + // default-ignorable code point (8 of 767 measured production names do). + // Warned, never refused or trimmed: the value is legal and the document + // is honest about the space's own state, but an invisible byte is all it + // takes for another writer's exact match to miss, and the place to clean + // it up is where the property is named, not at this seam. + warnKeySpellingHygiene(doc, warnIssue) + + // Two member names that are one name in two Unicode normal forms render + // identically and resolve — under §3's NFC rule — through one canonical + // form. Warned here, refused only at the import seam and only when they + // land on ONE stored key (the duplicate-binding refusal): export + // legitimately writes both byte forms when the space holds both as + // stored keys, each its own verbatim address, so a hard refusal here + // would make Marshal emit what Validate rejects (§11, I1). + warnNFCTwinSpellings(doc, warnIssue) + + // The template gate reads `kind`, and nothing else (§2). It used to + // resolve the `type` spelling through the document's own chain — legend, + // bundled table, verbatim — a private copy of §3 written so that Validate, + // which has no vocabulary (§13), reached the same verdict as the + // importer's kind derivation (§12). Both sides now read a field no chain + // touches, so the copy is gone and so is the class of disagreement it + // managed. + // + // The special case that lived here refused the legacy spelling of a + // template — `{"type": "template"}` with no `kind` — because that one + // shape was well-formed under both the old reading and the new, and so + // would have imported as an ordinary page with nothing anywhere saying + // so. It existed only for that ambiguity, and the freeze ended it: every + // document written under the old reading declares version 1, which + // checkVersion refuses before this pass runs (§10, §15 #9). So + // `template_for` is now gated on `kind` alone, with no exemption — a + // document with no `kind` reaches the first case below and is told + // template_for needs a template, which for a version-2 document is the + // whole truth. + kind, _ := doc["kind"].(string) + typeTerm, _ := doc["type"].(string) + if _, ok := doc["template_for"]; ok { + switch { + case kind != kindNames.name(model.SmartBlockType_Template): + addIssue("/template_for", `template_for is only valid on templates (kind "template")`) + case typeTerm == "": + // the target type is object_types[1] and there is no [1] without a + // [0]: import reads template_for only inside the branch that read + // `type`, so without one the field is silently discarded. The old + // gate refused this shape as a side effect of resolving `type`; + // reading `kind` instead, it has to be said outright. + addIssue("/template_for", + `template_for names the type this template is FOR, and needs the template's own type beside it: add "type"`) + } + } + + // Every per-key check below runs on the STORED key a document spelling + // resolves to — not on the raw spelling. The canonical document spells the + // display name (§3) — and spelled the derived slug when this rule was + // written — so checks keyed off the raw spelling were dead + // for exactly the documents this format produces: "unique_key" walked past + // the deny rule that "uniqueKey" tripped, and the legend could rebind any + // spelling onto any stored key, denied ones included. resolveDocKey is the + // §3 chain with the only vocabulary Validate has: the document's own + // legend first, then the bundled name table and its fold, then the + // spelling verbatim — the + // same resolution importer.propertyKey performs with default Options. A + // caller-supplied vocabulary can resolve further than Validate can see, + // which is why importer.build re-runs admission on ITS resolved key. + legend, _ := doc[memberPropertyInternalKeys].(map[string]any) + resolveDocKey := func(term string) string { + if v, ok := legend[term]; ok { + if key, isStr := v.(string); isStr && key != "" { + return key + } + } + key, _ := BundledKeyVocabulary{}.PropertyKey(term) + return key + } + + // A legend VALUE is a stored key, and admission judges it as one: the + // deny rule runs on the value itself, whether or not any member spells + // the entry. Checked only where a /properties member resolved through it, + // {"sneaky": "uniqueKey"} sat in the legend unchallenged — a laundering + // primitive for any key slot outside /properties, and one export never + // writes (a denied key never takes a spelling — writableSlug). + for _, term := range sortedMapKeys(legend) { + key, isStr := legend[term].(string) + if !isStr { + continue + } + if reason, denied := deniedPropertyKey(key); denied { + addIssue("/"+memberPropertyInternalKeys+"/"+escapeJSONPointer(term), "legend value: %s", reason) + } + } + + // An `option_ids` outer key is a PROPERTY SPELLING (§9a), and one naming a + // property the document never spells qualifies nothing: import indexes the + // legend by the spelling the slot it is resolving wrote, so such an entry + // is unreachable and the values under it resolve by name as if the legend + // were absent. + // + // A warning, not an error, because a legend is allowed to carry more than + // one document needs. But ignored in SILENCE is how a legend filed under + // `priorty` validates clean and then quietly loses the identity it was + // written to carry, which is the degradation this format reports + // everywhere else. + // + // A KEY-SET COMPARISON, not a parse. The flat spelling this replaced had + // to split each key at its last separator before it could ask the census + // anything, and that split was defined only for keys the shape rule + // admitted; here the property spelling IS the key, so the census answers + // it directly. + if legend, _ := doc["option_ids"].(map[string]any); len(legend) > 0 { + spellings := rawPropertySpellings(doc) + for _, slug := range sortedMapKeys(legend) { + if spellings[slug] { + continue + } + warnIssue("/option_ids/"+escapeJSONPointer(slug), + "no property in this document spells %q — this legend entry can "+ + "never be consulted, and the option names under it resolve by "+ + "name", slug) + } + } + + // The loop below is the MIRROR of the importer's details seam + // (importer.build), refusal for refusal — denied resolved key, unwritable + // resolved key, two spellings binding one key — in the same sorted order, + // so with default Options the two verdicts cannot differ (§12, I2). The + // duplicate-binding refusal used to live in the seam alone, and a + // hand-written {"iconEmoji": …, "icon_emoji": …} validated clean and then + // failed to import. + if props, _ := doc["properties"].(map[string]any); props != nil { + boundBy := make(map[string]string, len(props)) + for _, term := range sortedMapKeys(props) { + v := props[term] + path := "/properties/" + escapeJSONPointer(term) + // the importer lifts these two spellings into the envelope before + // any resolution runs (§2), so the legend cannot re-purpose them + key := term + if term != detailKeyId && term != detailKeyType { + key = resolveDocKey(term) + } + if reason, denied := deniedPropertyKey(key); denied { + addIssue(path, "%s", reason) + continue + } + // the §2a type-settings lift, kind-scoped like the import seam it + // mirrors (typesettings.go): on a TYPE document the five stored + // keys live in the group, and the flat spelling is refused with + // the repair named; on every other kind the same keys are + // ordinary properties + if isTypeKind(doc) && typeSettingsLiftedDetailKeys()[key] { + addIssue(path, "%q is written on a type document as %s in type_settings, "+ + "not as a property", key, typeSettingsLiftedKeyRepair(key)) + continue + } + // the document's own chain can hardly resolve a shape-checked + // term onto an unwritable key — legend values and spellings were + // vetted before this runs — but the seam refuses one however it + // arrives, and this pass mirrors the seam, not an argument about + // reachability + if !isWritablePropertyKey(key) { + addIssue(path, "%s", unwritableKeyReason("resolved property key", key)) + continue + } + if first, dup := boundBy[key]; dup { + addIssue(path, "%q and %q both address property %q — keep one", first, term, key) + continue + } + boundBy[key] = term + // name-over-number properties are named, not numbered (§3). A + // typo would otherwise import as a raw string onto a + // number-format property: no error anywhere, and every consumer + // reads it with an int getter and silently sees the enum's zero. + // The refusal states the vocabulary, because no schema slot can: + // a property SPELLING is not fixed to its stored key (a legend + // may rebind it), so this semantic pass — which runs on the + // RESOLVED key — is the vocabulary's only enforceable statement. + if vocab, named := namedEnumProperty(key); named { + if s, isStr := v.(string); isStr { + if !vocab.has(s) { + addIssue(path, "unknown %s %q — one of %s; a raw stored number is also accepted", + vocab.what, s, vocab.quotedNames()) + } + continue // a known name, or a raw number: both accepted (§3) + } + } + if reason, wrong := wrongShapeForFormat(key, v); wrong { + warnIssue(path, "%s", reason) + } + } + } + + // the §2d relation-definition fields: a meaningful value against a format + // that cannot use it is a WARNING, never an error — the reasoning lives + // with the check (relationformat.go), beside the export surface it must + // not contradict (I1) + propertySettingsIssues(doc, warnIssue) + // a definition with no identity stays a WARNING, and the reason is I1 + // rather than judgement: Marshal can emit a keyless type document — a + // snapshot whose type carries no unique key produces one — so refusing it + // here would make this package reject its own output. Real data never + // does it (0 of 2,975 corpus definitions), but the invariant is stated + // over every snapshot, not the likely ones. The cost is measured and + // real: 8 of 36 type documents authored against the schema shipped with + // no key and were told they were fine (§15). + definitionIdentityIssue(doc, warnIssue) + + // the type_settings name-over-number members carry the layout rule (§2a, + // §3): a typo'd NAME is refused — it would import as a raw string onto a + // number-format detail, which every consumer reads with an int getter and + // silently sees as the zero — while a raw number still passes, because a + // stored value outside the vocabulary round-trips as its number. + if group, _ := typeSettingsOf(doc); group != nil { + if s, isStr := group["layout"].(string); isStr && !layoutNames.has(s) { + addIssue("/type_settings/layout", "unknown layout %q", s) + } + if s, isStr := group["default_view"].(string); isStr && !viewTypeNames.has(s) { + addIssue("/type_settings/default_view", "unknown view type %q", s) + } + } + + if defs, hasDefs := typePropertyDefinitionsOf(doc); hasDefs { + // property_definitions replaces the recommended-relation lists (§2a): + // a document carrying both is ambiguous. The lists are named by + // whatever spelling resolves onto them — recommendedListKeys holds + // stored keys, and a document can reach one through any spelling the + // chain resolves: the display name ("Recommended properties"), the + // stored key verbatim, or a legend binding — so the check runs on + // the resolved key + if props, _ := doc["properties"].(map[string]any); props != nil { + listKeys := make(map[string]bool, len(recommendedListKeys)) + for _, l := range recommendedListKeys { + listKeys[l.detailKey] = true + } + for _, term := range sortedMapKeys(props) { + if listKeys[resolveDocKey(term)] { + addIssue("/properties/"+escapeJSONPointer(term), + "conflicts with type_settings.property_definitions, which replaces this list") + } + } + } + // name is used only when the property has to be created (§2a); an + // existing one keeps its own, so renaming a bundled key here reads as + // working and silently does nothing + if list := defs; list != nil { + for i, raw := range list { + tp, ok := raw.(map[string]any) + if !ok { + continue + } + // the identity an entry states: its `property` spelling + // (resolved through the document's own legend below), else + // its `internal_key`, which IS a stored key and resolves to + // itself (§2e) + key, _ := tp[memberProperty].(string) + resolvedEntryKey := "" + if key != "" { + resolvedEntryKey = resolveDocKey(key) + } else if ik, _ := tp[memberInternalKey].(string); ik != "" { + key, resolvedEntryKey = ik, ik + } + // options declare a select's vocabulary and its display + // order (§2a); on any other format there is nothing to + // declare and the array would be silently dropped + if opts, has := tp["options"].([]any); has && len(opts) > 0 { + if f, _ := tp["format"].(string); f != "select" && f != "multi_select" { + shown := f + if shown == "" { + shown = "text" + } + addIssue(fmt.Sprintf(typePropertyDefinitionsPath+"/%d/options", i), + "options is only meaningful on select/multi_select, not %q", shown) + } + // an option is a bare name or an object carrying a color + // (§2a), and the two forms name the same vocabulary: the + // duplicate check has to read across both + seen := map[string]bool{} + for j, o := range opts { + n := optionEntryName(o) + if seen[n] { + addIssue(fmt.Sprintf(typePropertyDefinitionsPath+"/%d/options/%d", i, j), + "duplicate option %q", n) + } + seen[n] = true + } + } + // objectTypes restricts what an object reference may point + // at; on any other format there is nothing to restrict and + // the array would be silently dropped + if ots, has := tp["object_types"].([]any); has && len(ots) > 0 { + if f, _ := tp["format"].(string); f != "objects" && f != "files" { + shown := f + if shown == "" { + shown = "text" + } + addIssue(fmt.Sprintf(typePropertyDefinitionsPath+"/%d/object_types", i), + "object_types is only meaningful on objects/files, not %q", shown) + } + } + // a bundled property is used as-is: only the wiring's + // create path reads these, and it never runs for a key that + // already exists (§2a). The key slot spells the display name + // like every other (§3), so the lookup runs on the resolved key + if key != "" { + if rel, err := bundle.GetRelation(domain.RelationKey(resolvedEntryKey)); err == nil && rel != nil { + if name, _ := tp["name"].(string); name != "" && name != rel.Name { + warnIssue(fmt.Sprintf(typePropertyDefinitionsPath+"/%d/name", i), + "%q is a bundled property named %q — this name is ignored; mint a custom key if the label matters", + key, rel.Name) + } + if ots, has := tp["object_types"].([]any); has && len(ots) > 0 && + !restatesBundledTargets(ots, rel) { + warnIssue(fmt.Sprintf(typePropertyDefinitionsPath+"/%d/object_types", i), + "%q is a bundled property; its target types are fixed by the bundle and this list is ignored — mint a custom key to target different types", + key) + } + } + } + } + } + } + + seenIds := map[string]string{} // id -> path of first occurrence + claimId := func(id, path string) { + if id == "" { + return + } + if first, dup := seenIds[id]; dup { + addIssue(path, "duplicate id %q (first used at %s)", id, first) + } else { + seenIds[id] = path + } + } + + // checkInline parses one text string for grammar errors, and reports the + // tag-shaped sequences the grammar does not recognize: those stay literal + // (§10), but canonical export escapes them (§8.2), so an unescaped one + // says the text did not come from this version's export. + checkInline := func(text, path string) { + _, _, notes, err := parseInlineNotes(text) + if err != nil { + addIssue(path, "inline markup: %v", err) + return + } + for _, name := range notes.unknownTags { + warnIssue(path, "tag-shaped %q is not markup this version recognizes — "+ + "kept as literal text; canonical output escapes the \"<\"", "<"+name) + } + } + + checkText := func(block map[string]any, path string) { + typ, _ := block["type"].(string) + if !textBearing(typ) { + return + } + text, _ := block["text"].(string) + if text == "" { + return + } + checkInline(text, path+"/text") + } + + var checkFlatRun func(blocks []any, basePath string, inCell bool) + var walkBlock func(block map[string]any, path string) + walkBlock = func(block map[string]any, path string) { + typ, _ := block["type"].(string) + if id, _ := block["id"].(string); id != "" { + claimId(id, path+"/id") + } + checkText(block, path) + if typ == "code" && codeLangConflict(block) { + addIssue(path, "language and fields.lang are both set") + } + if typ == "table" { + walkTable(block, path, claimId, addIssue, checkInline, walkBlock, checkFlatRun) + } + if typ == "dataview" { + checkDataviewViews(block, path, resolveDocKey, addIssue, warnIssue) + } + } + + // checkFlatRun validates one flat pre-order run (the document's blocks + // array, or a table cell's array form): V1 monotonicity, V2 leaf + // containment, V3 row→column, then the per-block checks. inCell bans an + // id on the first element (cell ids are derived, §6.1). + checkFlatRun = func(blocks []any, basePath string, inCell bool) { + type frame struct { + indent int + typ string + } + prev := -1 + var stack []frame + for i, raw := range blocks { + block, ok := raw.(map[string]any) + if !ok { + continue + } + path := fmt.Sprintf("%s/%d", basePath, i) + typ, _ := block["type"].(string) + if inCell && i == 0 { + if _, has := block["id"]; has { + addIssue(path+"/id", "cell blocks cannot carry an id — cell ids are derived") + } + if transparentBlockTypes[typ] { + // §7a: everywhere else a container is lifted and its + // children take its place. A cell is a position, not a + // run — there is nowhere to lift to — so this is the one + // spelling of a container the format cannot read back. + addIssue(path+"/type", "a cell block cannot be a %s: a transparent container contributes no block of its own, and a cell is a position rather than a run", typ) + } + } + k := indentOf(block) + if k > prev+1 { + // V1: continue with the clamped value either way so one bad + // indent does not cascade into follow-on errors + switch { + case lenient && prev < 0: + warnIssue(path, "indent %d on the first block — clamped to 0", k) + case lenient: + warnIssue(path, "indent %d follows indent %d — clamped to %d", k, prev, prev+1) + case prev < 0: + addIssue(path, "indent %d on the first block — the first block must be at indent 0", k) + default: + addIssue(path, "indent %d follows indent %d — a block can be at most one level deeper than its predecessor", k, prev) + } + k = prev + 1 + } + for len(stack) > 0 && stack[len(stack)-1].indent >= k { + stack = stack[:len(stack)-1] + } + // §7a: containment is judged against the LIFTED tree, because + // that is the tree import builds — `row > group > column` says + // `row > column`, which is exactly what it becomes. So the + // effective parent is the nearest ancestor that survives the + // lift, and a container is itself exempt: it becomes nothing, so + // there is nothing to place. The message names the effective + // parent AND the container between, or it reads as wrong to + // whoever wrote the group. + if !transparentBlockTypes[typ] { + j := len(stack) - 1 + for j >= 0 && transparentBlockTypes[stack[j].typ] { + j-- + } + if j >= 0 { + parent := stack[j] + viaGroup := j != len(stack)-1 + switch { + case leafBlockTypes[parent.typ] && viaGroup: + addIssue(path, "nested under a group inside a %s block — %s blocks cannot have children", parent.typ, parent.typ) + case leafBlockTypes[parent.typ]: + addIssue(path, "nested under a %s block — %s blocks cannot have children", parent.typ, parent.typ) + case parent.typ == "row" && typ != "column" && viaGroup: + addIssue(path, "nested under a group inside a row — a row block can only contain column blocks, got %s", typ) + case parent.typ == "row" && typ != "column": + addIssue(path, "a row block can only contain column blocks, got %s", typ) + } + } + } + stack = append(stack, frame{k, typ}) + prev = k + walkBlock(block, path) + } + } + + if blocks, _ := doc["blocks"].([]any); blocks != nil { + checkFlatRun(blocks, "/blocks", false) + } + return issues +} + +// neverWritableProperties are the keys import must refuse even though they are +// not in bundle.LocalAndDerivedRelationKeys, so the derived half of +// strippedDetailKeys does not know about them. (They ARE bundled relations — +// bundle.HasRelation is true for both — they are just +// not on the local/derived list.) They are the importer's own resolution +// vectors: existingobject.go picks which existing object in the space a +// snapshot merges into using oldAnytypeID, uniqueKey and sourceFilePath, so a +// document that sets them aims itself at an object it did not create. +var neverWritableProperties = map[string]string{ + "oldAnytypeID": "oldAnytypeID selects which existing object a document merges into and cannot be set by a document", + "sourceFilePath": "sourceFilePath selects which existing object a document merges into and cannot be set by a document", +} + +// transientProperties are stored details that describe a MOMENT rather than +// the object: state the app keeps for its own use, which means nothing once +// the object is out of the space it was written in. Export drops them and +// import ignores them, silently and in both directions — they are noise, not +// input, so a document carrying one is not wrong, merely stale. +// +// This is deliberately NOT neverWritableProperties. Those are refused with an +// error because setting one aims a document at an object it did not create; +// setting one of these achieves nothing at all, and refusing it would turn a +// stale export into an unimportable file for no gain. +// +// Nor is it the local/derived list: a transient key can be an ordinary stored +// detail that survives a restart. What puts it here is that its MEANING does +// not survive the trip. +// +// The list is expected to grow. Each entry needs the same two answers as +// internalFlags: what it means in the app, and why nothing downstream of an +// import can act on it. +// +// - internalFlags — editor UI state (editorDeleteEmpty, editorSelectType, +// editorSelectTemplate: "this object was just created, offer the type +// picker"). Measured across 36,967 real objects it is the single largest +// source of exported noise, present on 18,647 of them and EMPTY on all +// of those; a restored object is never mid-creation, so the flags have +// nothing to say. +var transientProperties = map[string]string{ + "internalFlags": "editor state for an object being created, which a restored object never is", + // The client's ANALYTICS context, persisted onto the object instead of + // only being sent as an event. `route` is anytype-ts's analytics-route + // concept (`analytics.route.shortcut`, `.header`, `.menuSystem`), and + // `SettingsSpace` names the screen the "create type" click came from. + // 35 type objects across 7 spaces carry the identical triple + // — data {"route":"SettingsSpace"}, isNew true, layoutFormat 0 — on + // ordinary user types (News, Bug report, Meeting, Issue). + // + // None of the three is a relation: not bundled, and no relation document + // defines them anywhere in 38,061 documents. So they are orphan details + // that no reader can name, give a format to, or act on — and the + // exhaustive legend rule dutifully pins all three, spending three + // entries to preserve the identity of something that describes nothing. + // + // `isNew` is `internalFlags`' idea exactly: a flag saying the object was + // just created, which a restored object never was. + "data": "the client's analytics route context, recorded on the object rather than sent as an event", + "isNew": "a just-created flag, true of the moment and never of the object", + "layoutFormat": "client layout state written beside the analytics context, defined by no relation", + + // The SOURCE SPACE'S LIVE SESSION — its invite credentials, its invite + // state and its analytics identity. Every one of these describes the + // space a bundle was exported FROM, and a space restored from that + // bundle regenerates all of them; not one is a fact about any object in + // it. + // + // Three of them are secrets. `spaceInviteFileKey` and + // `spaceInviteGuestFileKey` are, in the bundled table's own words, the + // "encoded encryption key of invite file" — and a bundle is a SHAREABLE + // artifact: a use case, a template, a backup someone sends on. Measured + // before this rule: 74 of 77 exported spaces carried at least one of + // these, 35 carried the invite key, and `analyticsSpaceId` — a stable + // per-space tracking identifier — travelled in 50. + // + // All ten occur on the space's own document and nowhere else in 38,070 + // corpus documents, so stripping them reaches nothing that wanted them. + "spaceInviteFileKey": "the invite file's ENCRYPTION KEY; a bundle is shareable and a restored space mints its own", + "spaceInviteGuestFileKey": "the guest invite file's ENCRYPTION KEY; same", + "oneToOneRequestMetadataKey": "a participant's request-metadata KEY; belongs to the source space's session", + "spaceInviteFileCid": "addresses the invite file the key above opens; useless and unwanted once the key is gone", + "spaceInviteGuestFileCid": "same, for the guest invite", + "spaceInvitePermissions": "the source space's live invite configuration, remade with the new space's own invite", + "spaceInviteType": "same", + "spaceInviteHeldByOwner": "same", + "oneToOneInboxSentStatus": "the source space's inbox session state", + "analyticsSpaceId": "an anonymous per-space TRACKING id; it identifies the space it left, not the one being made", + + // DEPRECATED space details. `spaceDashboardId` is `homepage`'s + // predecessor and its `object` format never told the truth — 46 of the + // 54 documents carrying both disagree, and its values are the sentinels + // `chat` and `lastOpened` rather than object ids at all. `homepage` + // (longtext, "could handle either object id or a sentinel") is the live + // one and index.json already carries it. + "spaceDashboardId": "deprecated: homepage's predecessor, and its `object` format holds sentinels, not ids", + "spaceUxType": "deprecated", + "hasChat": "deprecated", + + // DEPRECATED, and the clearest case of the three: an object's own + // featured list. The TYPE owns which properties an instance features — + // `section: "featured"` in its property_definitions — and the clients + // read it from there, ignoring whatever the object stores. + // + // heart is actively migrating the stored ones away. layout/syncer.go + // rewrites an object's list to empty, keeping `description` if it was + // there, and the corpus is that migration caught in flight: of 16,927 + // values across 12,603 documents, 6,135 are exactly `["description"]` + // and 1,285 are exactly `[]` — 59% carrying the syncer's own signature. + // There is no UI that sets a per-object featured list, so the remaining + // 41% are not user intent either; they are objects the syncer has not + // reached, still holding the defaults their type had at creation. + // + // Dropping it also retires a lie the format could not otherwise fix: the + // key is declared `format: "objects"` — an array of object ids — in the + // bundled table and in all 77 dictionaries, while holding zero object ids + // in all 16,927 real values. It holds property spellings, camelCase + // stored keys and bson keys, sometimes mixed inside one array. Nothing + // resolves it as declared, and writing a real object id there validated, + // imported and round-tripped with no warning at all. + "featuredRelations": "deprecated: the type's `section: \"featured\"` owns this, and the clients read it from there", + + // A FILE's variant machinery, and the first of them is a SECRET: the + // per-variant encryption keys. This package's own API layer already + // refuses to emit all seven, in its words "so a future change to either + // the bundle or the cache subscription cannot accidentally leak file keys + // / CIDs" (core/api/service/property.go) — and the export was shipping + // every one of them in a bundle built to be shared. + // + // Nothing needs them. They are read by `core/files/queries.go` and the + // file editor, which run in a space that already HOLDS the file; no + // import path reads any of them, and neither does this format or its + // tools. A bundle carries the file itself: imported into another space + // the content matches an existing file and is reused, and imported into + // another ACCOUNT it becomes a new file with a new encryption key and is + // uploaded afresh. The old key describes a blob the new account cannot + // and should not open. + // + // They were also 93% of the format's entire warning channel — 71,736 + // warnings, six keys declared `text` and one `number` while every stored + // value is a list. Not travelling is a better answer than not warning. + "fileVariantKeys": "a secret: the per-variant file ENCRYPTION keys, which a shared bundle must not carry", + "fileVariantIds": "file variant machinery: regenerated when the file is indexed, and never read on import", + "fileVariantChecksums": "file variant machinery: regenerated when the file is indexed, and never read on import", + "fileVariantMills": "file variant machinery: regenerated when the file is indexed, and never read on import", + "fileVariantOptions": "file variant machinery: regenerated when the file is indexed, and never read on import", + "fileVariantPaths": "file variant machinery: regenerated when the file is indexed, and never read on import", + "fileVariantWidths": "file variant machinery: regenerated when the file is indexed, and never read on import", + + // the file's own content addresses, and the last two members of the API's + // refusal list. `fileId` is the cid of the file's content and + // `fileSourceChecksum` its source hash; neither is read from an incoming + // document by any import path, and fileobject/service.go SETS fileId + // itself when it creates the object — so a restored file gets its own. + // + // `fileExt` and `fileMimeType` deliberately stay: they describe the file + // to a reader rather than address it in a store, and the API does not + // refuse them. + "fileId": "the file's content address: the importing space mints its own when it creates the file object", + "fileSourceChecksum": "the file's source hash: recomputed on the way in, and part of the API's file keys / CIDs refusal", + + // THE FILE MACHINERY'S per-device answers, stamped on every file object + // and meaning nothing off the device that stamped them. Their sibling + // `fileSyncStatus` is in bundle.LocalAndDerivedRelationKeys and has never + // exported; these two say the same kind of thing and leaked only because + // the bundle lists them as ordinary relations. + // + // `fileBackupStatus` is filesyncstatus.Status — which node-sync state THIS + // device last observed for the file (core/files/filesync writes it, the + // reconciler and the sync-status updater read it). All 10,248 file objects + // in a 28,604-document corpus carry it: 10,246 say Synced(4), 2 say + // Queued(5). A restored file's backup status is the destination's filesync + // machinery's to determine — importing "Synced" claims the new space's + // filenode holds blocks it has never seen. The enum's own comment says + // even the stored history is untrustworthy: SyncedLegacy exists because a + // migration "accidentally set FileBackupStatus to Synced for all files, + // even not synced". + // + // `fileIndexingStatus` is the sharpest entry on this list: ONE distinct + // value — Indexed(1) — across all 10,248 occurrences, which is the + // definition of saying nothing. And on import it is worse than nothing: + // the file indexer's queue query selects file objects whose + // fileIndexingStatus != Indexed (core/files/fileobject/fileindex.go), so + // an imported "Indexed" tells the destination's indexer the restored file + // needs no indexing — the one thing downstream that could act on the + // value acts on it wrongly. + "fileBackupStatus": "this device's file-sync answer; the destination's filesync machinery determines its own", + "fileIndexingStatus": "one distinct value in all real data, and importing it suppresses the destination's file indexer", +} + +// isTransientProperty reports whether a stored key describes a moment rather +// than the object. Export skips it; import drops it. +func isTransientProperty(key string) bool { + _, ok := transientProperties[key] + return ok +} + +// derivedAttributionProperties are the two properties that say WHO — the +// member who created the object and the member who last changed it. They are +// dropped on import for the same reason a transient key is (nothing +// downstream can act on the value), and they are a separate list because +// export treats them differently: a transient key is not written at all, +// while these are written as `#` — the folded participant id with +// the member's name as the informative suffix (§3, §9, buildProperties). +// +// Why nothing downstream can act on the value, which is the entry price for +// this list: both are `source: derived, maxCount: 1, readonly: true` +// (bundle/relations.json). Their value is not stored input — it is recovered +// from the object tree root's cryptographic signature on every rebuild +// (`treeSource.GetCreationInfo` → `NewParticipantId(spaceId, identity)`), and +// four independent seams already discard whatever a document supplies: +// `state.StructCutKeys(details, LocalAndDerivedRelationKeys)`, the pb +// importer's preserve-list (which names neither), `changeBlockDetailsSet`, +// and the API's "cannot be set directly". A document that carries one is +// telling a reader who wrote the object; it is not, and never was, setting +// anything. +// +// The asymmetry this closes: `creator` used to be ACCEPTED on import (it sat +// in propertiesKeptOnExport, so the deny rule never saw it) while +// `lastModifiedBy` — an identical relation definition, one word apart in the +// bundle — was REFUSED. Neither had any effect. 71 documents in a +// 36,966-object corpus carry `lastModifiedBy`, and every one of them was +// unimportable for it. +var derivedAttributionProperties = map[string]string{ + "creator": "the member who created the object, recovered from the tree root's signature on every rebuild", + "lastModifiedBy": "the member who last changed the object, derived the same way as creator", +} + +// isDroppedOnImport reports whether a stored key is ignored rather than +// refused when a document carries it: the transient keys, whose meaning does +// not survive the trip, and the derived attribution keys, whose value is +// re-derived from the tree and which no write path could honour anyway. Both families +// are stripped from a document's own VALUES like every other internal key — +// what they share is that a stale or hand-written document carrying one is +// still importable (§3). +func isDroppedOnImport(key string) bool { + if isTransientProperty(key) { + return true + } + _, ok := derivedAttributionProperties[key] + return ok +} + +// maxPropertyKeyLen mirrors the schema's propertyNames maxLength (§3). +const maxPropertyKeyLen = 128 + +// isWritablePropertyKey reports whether a key can be a property name at all, +// mirroring the schema's propertyNames rule: non-empty, no control characters, +// and inside the length bound. Both directions consult it — validation through +// the schema, export directly — because a stored detail key is not guaranteed +// to be one: an empty key and a key holding a newline both exist in real data, +// and neither survives as a JSON property name that means anything. +// +// It carries ONE rule the schema does not, and cannot: the key must be valid +// UTF-8. A parsed JSON document always holds valid UTF-8 — the decoder has +// already replaced anything else — so the rule is unstatable there and only +// export can break it. Breaking it was worse than a lost byte, because the +// spelling of a property is now its display NAME, and names come from a store +// that does not police its bytes: +// +// - the writer maps every invalid byte to U+FFFD, while the collision plan +// compares the raw Go strings. Two distinct names differing only in their +// invalid bytes therefore look distinct to the plan, get no suffix, and +// then render as ONE member name. A JSON object cannot hold a member +// twice: one value silently replaces the other, and Validate passes, +// because by the time it reads the document the collision has already +// happened. +// - folding it into the plan instead was considered and does not work. The +// plan would have to compare the RENDERED forms, which are equal, so +// there is no spelling it could hand either claimant; the honest answer +// is that a name whose bytes cannot be written is not a spelling. +// +// The retired normalization grammar dropped U+FFFD as a matter of course, so +// this exposure arrived with raw names. Zero occurrences in the 77-space +// corpus: this is hardening, and the key falls back to being written under +// its stored key like every other unwritable spelling. +func isWritablePropertyKey(key string) bool { + if key == "" || utf8.RuneCountInString(key) > maxPropertyKeyLen { + return false + } + if !utf8.ValidString(key) { + return false + } + for _, r := range key { + if r <= 0x1f || r == 0x7f { + return false + } + } + return true +} + +// keySlotReport is what propertyNameIssues found, plus what it spoke for so +// the schema does not say it again: names holds the member NAMES it rejected +// (the schema's propertyNames verdict on those is redundant and pathless), +// values holds the pointers whose VALUE it rejected (the schema addresses +// those correctly but says nothing about what is wrong with the string). +type keySlotReport struct { + issues []Issue + names map[string]bool + values map[string]bool +} + +// rejectValueAt records an issue at a pointer the schema addresses correctly +// but words badly, and silences the schema's own verdict there. It is the +// same trade propertyNameIssues makes for a key slot: one fault, one issue +// (§12), worded by whichever pass can say what is actually wrong. +func (r *keySlotReport) rejectValueAt(path, message string) { + r.issues = append(r.issues, Issue{Path: path, Message: message}) + r.values[path] = true +} + +// propertyNameIssues states, where the key is in hand, every rule the schema +// carries as `propertyNames`: the `properties` map and the `property_internal_keys` / +// `type_internal_keys` legends take a writable key (§3), and `option_ids` takes one at +// its OUTER level with a merely non-empty option name at its inner level +// (§9a). A legend VALUE rides along because it is a stored key under the same +// rule and the schema's verdict on it names the bound, not the string — and so +// does a property-definition entry's `property` (and its `internal_key`), key +// slots the schema can only reach as ordinary string values. +// +// The rule stays in the published schema — an external validator runs that and +// nothing else (§12) — and is restated here because `propertyNames` cannot +// produce the issue §12 promises. The library validates each name as a +// standalone string instance, so its verdict carries neither the enclosing +// object's location nor, for a length bound, the name: a 200-character +// property key was reported as `maxLength: got 200, want 128` at the document +// ROOT, and an agent running the generate → validate → feed-back loop (§13) +// cannot tell from that which property to fix. The predicate is the export +// side's own (isWritablePropertyKey), so the two directions cannot drift into +// Marshal emitting what Validate rejects (§11, I1). +func propertyNameIssues(doc map[string]any) keySlotReport { + r := keySlotReport{names: map[string]bool{}, values: map[string]bool{}} + rejectName := func(path, name, reason string) { + r.issues = append(r.issues, Issue{Path: path, Message: reason}) + r.names[name] = true + } + rejectValue := func(path, reason string) { + r.issues = append(r.issues, Issue{Path: path, Message: reason}) + r.values[path] = true + } + + // `option_ids` carries a `propertyNames` at BOTH levels (§9a), and each + // needs its own case here or the schema's unaddressable root-level verdict + // comes back for it. The two rules differ on purpose: an outer key is a + // property spelling like any other, an inner key is an option NAME bounded + // only by being non-empty — it is the same string the value slot already + // holds, so any charset rule on it would refuse a legend entry for a value + // the document itself carries. + if legend, _ := doc["option_ids"].(map[string]any); legend != nil { + for _, slug := range sortedMapKeys(legend) { + path := "/option_ids/" + escapeJSONPointer(slug) + if !isWritablePropertyKey(slug) { + rejectName(path, slug, unwritableKeyReason("option_ids property spelling", slug)) + } + names, isObject := legend[slug].(map[string]any) + if !isObject { + continue // the schema types the level; this pass only shapes it + } + for _, name := range sortedMapKeys(names) { + if name != "" { + continue + } + rejectName(path+"/", name, + "option name is empty — an option_ids entry has to name an "+ + "option the document spells") + } + } + } + if props, _ := doc["properties"].(map[string]any); props != nil { + for _, term := range sortedMapKeys(props) { + if !isWritablePropertyKey(term) { + rejectName("/properties/"+escapeJSONPointer(term), term, + unwritableKeyReason("property key", term)) + } + } + } + // a property definition's `property` is a property key slot too (§2a), and + // the only one that is a JSON string VALUE rather than a member name: the + // schema carries the rule, but `propertyNames` never sees this slot, so + // its verdict names a bound or prints a regex instead of saying what is + // wrong with the string. Same rule, same wording as the members above — + // and the same reason it is a rule at all: the import seam refuses a key + // export could not write back, so a document carrying one validated clean + // and then failed to import (I2). + if list, _ := typePropertyDefinitionsOf(doc); list != nil { + for i, raw := range list { + tp, _ := raw.(map[string]any) + if key, isString := tp[memberProperty].(string); isString && !isWritablePropertyKey(key) { + rejectValue(fmt.Sprintf(typePropertyDefinitionsPath+"/%d/"+memberProperty, i), + unwritableKeyReason("property key", key)) + } + // internal_key is a stored key under the same writable rule the + // legend VALUES carry (§3) — the import seam refuses a key export + // could not write back, whichever member states it + if key, isString := tp[memberInternalKey].(string); isString && !isWritablePropertyKey(key) { + rejectValue(fmt.Sprintf(typePropertyDefinitionsPath+"/%d/"+memberInternalKey, i), + unwritableKeyReason("property internal key", key)) + } + } + } + // A dataview FILTER has to name a property, like the sort and the column + // beside it (§6). The schema says so — `required: ["property"]` on the + // leaf branch — but it says it through a `oneOf`, so a filter missing the + // member collects the OTHER branch's whole verdict as well: "missing + // properties 'operator', 'filters'" plus one "not allowed" per member it + // does carry. That is four confidently wrong instructions for one fault, + // which is the failure mode §12's one fault, one issue rule exists for. + // So this pass owns the wording and mutes the branch's noise. + // + // The rule is not decorative. A filter with no property filters on + // nothing: import stored it with an empty relation key, the view silently + // stopped meaning what it said, and export re-emitted the same nameless + // node forever. + // + // The sibling block key slots ride the same walk under the same + // writable-key rule the schema bounds them with (§3): the schema's + // verdicts are path-correct here, but this pass owns the wording + // (unwritableKeyReason names which half of the rule broke where the + // schema names a bound), and it carries the one clause the pattern + // cannot — DEL, which sits above the pattern's control-character class — + // so Validate and the import seam cannot disagree about a spelling. + checkBlockKeySlots(doc, rejectValue) + for _, field := range []string{memberPropertyInternalKeys, memberTypeInternalKeys} { + legend, _ := doc[field].(map[string]any) + for _, term := range sortedMapKeys(legend) { + path := "/" + field + "/" + escapeJSONPointer(term) + if !isWritablePropertyKey(term) { + rejectName(path, term, unwritableKeyReason("legend spelling", term)) + } + key, isString := legend[term].(string) + if !isString { + continue // the schema types the value; this pass only shapes it + } + if !isWritablePropertyKey(key) { + rejectValue(path, unwritableKeyReason("legend stored key", key)) + } + } + } + return r +} + +// checkBlockKeySlots restates the writable-key rule (§3) at every property +// key slot a block can carry: the property block's `property`, a link +// block's `properties[]`, a dataview's `properties[].property`, and a view's +// `group_by`/`cover_property`/`end_property`/`columns[]`/`sorts[]`/ +// `filters[]`. One predicate — isWritablePropertyKey, the export side's own +// — at every slot, so the two directions cannot drift into Marshal emitting +// what Validate rejects (§11, I1). +func checkBlockKeySlots(doc map[string]any, reject func(string, string)) { + rejectKey := func(path, key string) { + if !isWritablePropertyKey(key) { + reject(path, unwritableKeyReason("property key", key)) + } + } + for i, raw := range blocksOf(doc) { + block, _ := raw.(map[string]any) + if block == nil { + continue + } + base := fmt.Sprintf("/blocks/%d", i) + blockType, _ := block["type"].(string) + switch blockType { + case "property": + if key, isString := block[memberProperty].(string); isString { + rejectKey(base+"/"+memberProperty, key) + } + case "link": + list, _ := block["properties"].([]any) + for j, item := range list { + if key, isString := item.(string); isString { + rejectKey(fmt.Sprintf("%s/properties/%d", base, j), key) + } + } + case "dataview": + list, _ := block["properties"].([]any) + for j, item := range list { + entry, _ := item.(map[string]any) + if entry == nil { + continue + } + if key, isString := entry[memberProperty].(string); isString { + rejectKey(fmt.Sprintf("%s/properties/%d/%s", base, j, memberProperty), key) + } + } + views, _ := block["views"].([]any) + for j, rawView := range views { + view, _ := rawView.(map[string]any) + if view == nil { + continue + } + vBase := fmt.Sprintf("%s/views/%d", base, j) + for _, member := range []string{"group_by", "cover_property", "end_property"} { + if key, isString := view[member].(string); isString { + rejectKey(vBase+"/"+member, key) + } + } + for _, list := range []string{"columns", "sorts"} { + entries, _ := view[list].([]any) + for k, rawEntry := range entries { + entry, _ := rawEntry.(map[string]any) + if entry == nil { + continue + } + if key, isString := entry[memberProperty].(string); isString { + rejectKey(fmt.Sprintf("%s/%s/%d/%s", vBase, list, k, memberProperty), key) + } + } + } + nodes, _ := view["filters"].([]any) + checkFilterProperties(nodes, vBase+"/filters", reject) + } + } + } +} + +// checkFilterProperties walks a view's filter tree and reports every LEAF +// node that names no property. A group node (`operator` + `filters`) names +// none by design, so the walk descends rather than judging it. +func checkFilterProperties(nodes []any, path string, reject func(string, string)) { + for i, raw := range nodes { + node, _ := raw.(map[string]any) + if node == nil { + continue + } + nPath := fmt.Sprintf("%s/%d", path, i) + if sub, isGroup := node["filters"].([]any); isGroup { + checkFilterProperties(sub, nPath+"/filters", reject) + continue + } + raw, named := node[memberProperty] + if !named { + reject(nPath, "a filter has to name the property it filters on") + continue + } + if prop, isString := raw.(string); isString && !isWritablePropertyKey(prop) { + reject(nPath+"/property", unwritableKeyReason("property key", prop)) + } + } +} + +// blocksOf is the document's flat block list, or nothing when it has none or +// the member is not a list — the schema types it; this pass only shapes it. +func blocksOf(doc map[string]any) []any { + blocks, _ := doc["blocks"].([]any) + return blocks +} + +// unwritableKeyReason names the string that broke the writable-key rule and +// which half of it broke. Naming it is not redundant with the pointer: a +// legend VALUE has no pointer of its own, and an over-long key makes a pointer +// no one reads. +func unwritableKeyReason(what, key string) string { + switch n := utf8.RuneCountInString(key); { + case key == "": + return what + " is empty — a key slot has to name something" + case n > maxPropertyKeyLen: + return fmt.Sprintf("%s %q is %d characters; the bound is %d", + what, key, n, maxPropertyKeyLen) + case !utf8.ValidString(key): + return fmt.Sprintf("%s %q is not valid UTF-8; every byte a document writes "+ + "has to survive being written, and an invalid one is replaced on the way out — "+ + "two keys differing only there would collapse onto one member name", what, key) + default: + return fmt.Sprintf("%s %q carries a control character", what, key) + } +} + +// deniedPropertyKey reports whether a property key may be written at all, and +// why not. The rule is a single sentence — **import refuses exactly what export +// strips** (§3, §4a) — and it is derived from the export side's own list rather +// than restated, because a restated list is how the two surfaces drifted apart +// in the first place: import used to accept isArchived, spaceId, restrictions, +// uniqueKey and the empty key, all of which export removes. +func deniedPropertyKey(key string) (string, bool) { + if reason, never := neverWritableProperties[key]; never { + return reason, true + } + if key == detailKeyId || key == detailKeyType { + return fmt.Sprintf("%q belongs in the envelope, not in properties", key), true + } + if isDroppedOnImport(key) { + // its VALUE is stripped on export like the rest, but the key is + // DROPPED on import rather than refused: a document carrying transient + // state or an attribution line is stale, not wrong, and an old export + // should still import (§3) + return "", false + } + if strippedDetailKeys()[key] { + return fmt.Sprintf("%q is internal: export strips it, so import does not accept it", key), true + } + // the icon/cover lift (§2b). Unlike the internal keys these DO have a + // written form, so the refusal names it: the same fact — this key is not + // where the value lives any more — is worth twice as much said as a + // repair. The set is the export side's own, never a restatement. + if liftedDetailKeys()[key] { + return fmt.Sprintf("%q is written as %s, not as a property", key, liftedKeyRepair(key)), true + } + // the relation-definition lift (§2d), same rule and same derivation. This + // arm is also the whole of the legacy-input decision (§10): a document + // written earlier spells `relation_format` here, and it is REFUSED + // with the repair named rather than read with a warning — the format is a + // pre-release draft with no external consumers, and a second legal + // spelling for a relation's format is exactly the ambiguity the lift + // deletes. + if propertySettingsLiftedDetailKeys()[key] { + return fmt.Sprintf("%q is written on a property document's envelope as %s in property_settings, "+ + "not as a property", key, propertySettingsLiftedKeyRepair(key)), true + } + return "", false +} + +// wrongShapeForFormat reports a property value whose JSON shape its property's +// format cannot hold — "next Friday" on a date, "yes" on a checkbox — which is +// stored verbatim and then read as the format's zero value forever, with +// nothing to show that anything went wrong. +// +// Only bundled properties can be checked: Validate takes no resolver, so a +// custom key's format is unknown here. And it is a **warning**, not an error, +// for a reason worth writing down: the same check on the export path would make +// one already-corrupt stored value enough to make an object unexportable, and +// "Marshal never emits what Validate rejects" (§11) is the stronger promise. +// Reporting it costs nothing and catches the authoring case, which is the one +// that can still be fixed. +// listValuedDespiteDeclaration are the keys whose bundled declaration +// disagrees with every value the store has ever held, so the shape warning +// below is about the TABLE rather than the document. +// +// All seven describe a file's variants, all seven sit on every file object, +// and all seven hold a list: six are declared `text` and one (`widths`) +// `number`. In a 77-space export that is 71,736 warnings — 93% of the entire +// warning channel, against 371 warnings that tell a reader something. A +// channel that is 99% noise is a channel nobody reads, which costs the format +// the warnings it actually needs: the unguarded date filter that silently +// widens a view, the group_by a view type cannot honour. +// +// The right repair is in the bundled table, which is not this package's to +// change (§15). Until it happens, the format declines to report a mismatch +// that is universal, expected, and about a declaration no document chose. +var listValuedDespiteDeclaration = map[string]bool{ + "fileVariantChecksums": true, + "fileVariantIds": true, + "fileVariantKeys": true, + "fileVariantMills": true, + "fileVariantOptions": true, + "fileVariantPaths": true, + "fileVariantWidths": true, +} + +// restatesBundledTargets reports an `object_types` list that says exactly +// what the bundled table already says for this property. +// +// The warning beside it exists to tell an author their list is IGNORED. That +// is worth saying when the list asks for something the bundle will not +// honour; it is worth nothing when the list is the bundle's own answer +// written out again — and export writes it out again on every bundled +// property that has targets, which was 5,336 warnings in a 77-space export, +// 93% of what remained of the channel. `type` restating `object_type`, +// `creator` restating `participant`, `picture` restating `image`. +// +// A list that DIFFERS still warns, because then something really is being +// discarded. +func restatesBundledTargets(stated []any, rel *model.Relation) bool { + bundled := map[string]bool{} + for _, u := range rel.GetObjectTypes() { + if k, err := bundle.TypeKeyFromUrl(u); err == nil { + bundled[string(k)] = true + bundled[TypeKeySpelling(string(k))] = true + } + } + if len(bundled) == 0 { + return false + } + for _, raw := range stated { + t, _ := raw.(string) + if !bundled[t] { + return false + } + } + return true +} + +func wrongShapeForFormat(key string, v any) (string, bool) { + if v == nil { + return "", false // an explicit null is a value: the key was set (§3) + } + if listValuedDespiteDeclaration[key] { + return "", false + } + rel, err := bundle.GetRelation(domain.RelationKey(key)) + if err != nil || rel == nil { + return "", false + } + switch rel.Format { + case model.RelationFormat_date: + // a number is unix seconds — including the raw number export writes for + // a date with no RFC 3339 form (§3) + if _, isNum := v.(json.Number); isNum { + return "", false + } + if s, isStr := v.(string); isStr { + if _, ok := parseDate(s); ok { + return "", false + } + } + return fmt.Sprintf("%q is a date property: a value that is neither unix seconds nor an "+ + "RFC 3339 string is stored as written and reads as no date at all", key), true + case model.RelationFormat_checkbox: + if _, isBool := v.(bool); !isBool { + return fmt.Sprintf("%q is a checkbox property: anything but true/false reads as false", key), true + } + case model.RelationFormat_number: + if _, isNum := v.(json.Number); !isNum { + return fmt.Sprintf("%q is a number property: a non-number reads as 0", key), true + } + case model.RelationFormat_longtext, model.RelationFormat_shorttext, + model.RelationFormat_url, model.RelationFormat_email, + model.RelationFormat_phone, model.RelationFormat_emoji: + if _, isStr := v.(string); !isStr { + return fmt.Sprintf("%q is a text property: a non-string reads as empty", key), true + } + case model.RelationFormat_object, model.RelationFormat_file: + // a reference is an id, optionally followed by `#name` (§9). A value + // that BEGINS at the separator has no id half, so it addresses + // nothing — and the reader will not repair it: splitRefName refuses + // to split at index 0 precisely so import never invents an empty id, + // which means the value is stored exactly as written and dangles + // forever. It is the shape a writer produces copying only the + // readable half of `id#name`. + for _, ref := range stringsOf(v) { + if strings.HasPrefix(ref, refNameSep) { + return fmt.Sprintf("%q is an object property: %q has no id before its %q, "+ + "so it names no object — a reference is an id, optionally followed by %q", + key, ref, refNameSep, refNameSep+"name"), true + } + } + } + return "", false +} + +// stringsOf collects the strings a property value carries, whether it holds +// one or a list of them (§3: a single value and a one-element list are the +// same value). +func stringsOf(v any) []string { + switch x := v.(type) { + case string: + return []string{x} + case []any: + out := make([]string, 0, len(x)) + for _, e := range x { + if s, ok := e.(string); ok { + out = append(out, s) + } + } + return out + } + return nil +} + +// checkNumbers walks every number in the document and reports the ones no +// reader can hold. A JSON number has no range limit; float64 does, and that is +// where every number in this format ends up — so a value outside it is not a +// number this format has, whatever surface it sits on. +func checkNumbers(node any, path string, addIssue func(path, format string, args ...any)) { + switch n := node.(type) { + case map[string]any: + for _, k := range sortedMapKeys(n) { + checkNumbers(n[k], path+"/"+escapeJSONPointer(k), addIssue) + } + case []any: + for i, v := range n { + checkNumbers(v, fmt.Sprintf("%s/%d", path, i), addIssue) + } + case json.Number: + if _, err := n.Float64(); err != nil { + addIssue(path, "number %s is out of range: values must fit a 64-bit float", n.String()) + } + } +} + +// maxDayCount bounds a counting preset's operand — the same bound the compact +// filter grammar puts on `daysAgo(n)` (filterstring.maxDayCount, §6.2.1), and +// for the same reason: past ~100 years the day arithmetic wraps and the range +// stops meaning anything. The two forms of one filter language must admit the +// same filters, so the structured form carries it too. +const maxDayCount = 36500 + +// dayCountFault reports why a counting preset's operand is not a day count, +// or "" when it is one. Numbers arrive as json.Number here (the document is +// decoded with UseNumber), and as float64 through the fragment surfaces. +func dayCountFault(v any) string { + var n float64 + switch num := v.(type) { + case json.Number: + f, err := num.Float64() + if err != nil { + // a number no float64 can hold is checkNumbers' fault to report, + // at this very pointer; saying it twice is the one-fault-one-issue + // rule broken (§12) + return "" + } + n = f + case float64: + n = num + default: + return fmt.Sprintf("%s counts as 0 days, i.e. today — the engine reads the operand as a number or not at all", jsonKindName(v)) + } + if n != math.Trunc(n) || n < 0 || n > maxDayCount { + return fmt.Sprintf("%v is not a whole day count between 0 and %d", n, maxDayCount) + } + return "" +} + +// jsonKindName names a decoded JSON value's kind for a diagnostic, in the +// grammatical form the messages above read it in. +func jsonKindName(v any) string { + switch v.(type) { + case nil: + return "null" + case bool: + return "a boolean" + case string: + return "a string" + case []any: + return "an array" + case map[string]any: + return "an object" + } + return "a non-numeric value" +} + +func sortedMapKeys(m map[string]any) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// escapeJSONPointer escapes the two characters a JSON pointer token cannot +// carry literally (RFC 6901): a property key is author-controlled, and the +// loose surfaces accept any key at all. +func escapeJSONPointer(token string) string { + token = strings.ReplaceAll(token, "~", "~0") + return strings.ReplaceAll(token, "/", "~1") +} + +// codeLangConflict reports a code block carrying both the first-class +// language prop and the internal fields.lang it lifts (§5.1). +func codeLangConflict(block map[string]any) bool { + if _, hasLang := block["language"]; !hasLang { + return false + } + fields, _ := block["fields"].(map[string]any) + if fields == nil { + return false + } + _, conflict := fields[codeLangField] + return conflict +} + +func walkTable(block map[string]any, path string, + claimId func(id, path string), addIssue func(path, format string, args ...any), + checkInline func(text, path string), + walkBlock func(block map[string]any, path string), + checkFlatRun func(blocks []any, basePath string, inCell bool)) { + + columns, _ := block["columns"].([]any) + colIds := make([]string, 0, len(columns)) + for i, c := range columns { + col, _ := c.(map[string]any) + id, _ := col["id"].(string) + colIds = append(colIds, id) + if id != "" { + claimId(id, fmt.Sprintf("%s/columns/%d/id", path, i)) + } + } + rows, _ := block["rows"].([]any) + for i, r := range rows { + row, _ := r.(map[string]any) + rowPath := fmt.Sprintf("%s/rows/%d", path, i) + rowId, _ := row["id"].(string) + if rowId != "" { + claimId(rowId, rowPath+"/id") + } + cells, _ := row["cells"].([]any) + if len(cells) > len(columns) { + addIssue(rowPath+"/cells", "row has %d cells but the table has %d columns", len(cells), len(columns)) + } + // every row×column pair joins the id uniqueness domain (§4), whether + // or not the cell is written: the id belongs to the table either way, + // and the editor materializes the missing cell at exactly that id the + // first time it is filled. Claiming only the written cells left the + // rest of the grid free for a block to take. + for j, colId := range colIds { + if rowId == "" || colId == "" { + continue + } + at := rowPath + if j < len(cells) { + at = fmt.Sprintf("%s/cells/%d", rowPath, j) + } + claimId(rowId+"-"+colId, at) + } + for j, c := range cells { + cellPath := fmt.Sprintf("%s/cells/%d", rowPath, j) + switch cell := c.(type) { + case string: + if cell != "" { + checkInline(cell, cellPath) + } + case map[string]any: + // §7a: the same refusal the array form applies at index 0. + // A cell is a position, not a run, so a container has nowhere + // to lift to and import refuses it (import.go's blockFromJSON) + // — Validate has to refuse it here or the two disagree, which + // is I2. The array form reaches this through checkFlatRun's + // `inCell` branch; the object form has no run to walk, so it + // needs its own. + if typ, _ := cell["type"].(string); transparentBlockTypes[typ] { + addIssue(cellPath+"/type", "a cell block cannot be a %s: a transparent container contributes no block of its own, and a cell is a position rather than a run", typ) + continue + } + // a full walk: the cell joins the id uniqueness domain and + // gets its text checked (tables inside cells are already + // rejected by the schema's cellBlock definition) + walkBlock(cell, cellPath) + case []any: + // array form (§6.1 F10): a flat run — cell block first at + // indent 0, descendants following + checkFlatRun(cell, cellPath, true) + } + } + } +} + +// groupableFormats lists, per view type, the property formats that view can +// group by. Only kanban and calendar group at all: the middleware assigns +// groupRelationKey for exactly these pairs (converter.insertGroupRelationKey, +// whose default branch is a no-op), the kanban service registers groupers for +// exactly these formats (core/kanban.Service.Init), and the client offers the +// same set (Relation.getGroupTypes). Every other view type ignores groupBy. +var groupableFormats = map[string]map[string]struct{}{ + "kanban": {"select": {}, "multi_select": {}, "checkbox": {}}, + "calendar": {"date": {}}, +} + +// checkDataviewViews runs the per-view semantic checks that need the +// dataview's own properties[] to know a key's format: groupBy viability and +// the date-filter empty trap. It also enforces view-id uniqueness. +// +// It reports a groupBy a view cannot honour. An impossible pair on +// a grouping view is an error: it can only come from authoring, and it +// renders as a single empty group. groupBy on a non-grouping view is only a +// warning — switching a kanban to a table in the editor leaves the stale +// groupRelationKey behind, so real exported data legitimately carries it. +// +// VIEW-ID UNIQUENESS is scoped to the dataview BLOCK, not to the document — +// the one id domain in this format that is not document-wide (§4), and +// deliberately so: +// +// - It is the scope in which a duplicate actually breaks something. Every +// consumer resolves a view reference within ONE dataview's views list +// (the API's matchViewRef, the client's view tabs), and a dataview's +// per-view editor state — groupOrders, objectOrders — is keyed by view +// id inside that same block. Two views of one dataview sharing an id +// make the second unaddressable forever; two views in DIFFERENT dataview +// blocks sharing one are each reachable through their own block. +// - Document-wide would reject data the app itself produces. The default +// view of every set, collection and type is minted with the literal id +// "default" (editor/template.MakeDataviewContent), and creating an +// inline set from an existing object copies that object's views verbatim +// into the new block (dataviewservice.CopyDataviewToBlock) — so a page +// with two inline collections legitimately holds two views called +// "default". A format error there would fail on real exports. +// +// Before this, `views[].id` was the one id slot in the document with no +// uniqueness check at all — invalid but unvalidated on every channel, +// create and import included, not just PATCH. +func checkDataviewViews(block map[string]any, path string, resolveKey func(string) string, + addIssue, warnIssue func(string, string, ...any)) { + views, _ := block["views"].([]any) + if len(views) == 0 { + return + } + seenViewIds := map[string]string{} // id -> path of first occurrence + for i, raw := range views { + view, ok := raw.(map[string]any) + if !ok { + continue + } + id, _ := view["id"].(string) + if id == "" { + continue // ids are optional on input (§9); import generates them + } + idPath := fmt.Sprintf("%s/views/%d/id", path, i) + if first, dup := seenViewIds[id]; dup { + addIssue(idPath, "duplicate view id %q in this dataview (first used at %s)", id, first) + continue + } + seenViewIds[id] = idPath + } + formats := map[string]string{} + props, _ := block["properties"].([]any) + for _, raw := range props { + p, ok := raw.(map[string]any) + if !ok { + continue + } + key, _ := p[memberProperty].(string) + if f, isStr := p["format"].(string); isStr && key != "" { + formats[key] = f + } else if key != "" { + formats[key] = "text" // §3: an omitted format is text + } + } + // The date rules read the format the IMPORTER will attach, which is not + // always the one this block declares: impDvFormat rehydrates a filter's + // format from the dataview's properties list first and the bundled table + // second (§6.2), and a hand-written dataview usually carries no properties + // list at all — `due_date` is a date filter there all the same. Checking + // the declaration alone would leave the rules below silent on exactly the + // documents an agent writes. + isDate := func(prop string) bool { + if declared, listed := formats[prop]; listed { + return declared == "date" + } + f, err := bundle.GetRelationFormat(domain.RelationKey(resolveKey(prop))) + return err == nil && FormatName(f) == "date" + } + for i, raw := range views { + view, ok := raw.(map[string]any) + if !ok { + continue + } + checkDateFilters(view, formats, isDate, fmt.Sprintf("%s/views/%d", path, i), addIssue, warnIssue) + groupBy, _ := view["group_by"].(string) + if groupBy == "" { + continue + } + vPath := fmt.Sprintf("%s/views/%d/group_by", path, i) + viewType, _ := view["type"].(string) + if viewType == "" { + viewType = "table" // §6.2: the default view type + } + allowed, groups := groupableFormats[viewType] + if !groups { + warnIssue(vPath, "%q views do not group; group_by is ignored", viewType) + continue + } + // a key absent from properties has no declared format to check + format, declared := formats[groupBy] + if !declared { + continue + } + if _, ok := allowed[format]; !ok { + addIssue(vPath, "%q views cannot group by %q (format %q); expected %s", + viewType, groupBy, format, strings.Join(sortedKeys(allowed), " · ")) + } + } +} + +func sortedKeys(m map[string]struct{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// checkDateFilters warns about `less`/`less_or_equal` on a date property that +// is not guarded by a `not_empty`/`exists` on the same property in an +// enclosing AND. An object with no value for that date matches: the filter's +// value is set and the record's is not, so domain.Value.Compare returns 1, +// which is exactly what Less tests for. A freshness view written the obvious +// way ("verifiedUntil less today") therefore lists every never-verified +// object alongside the genuinely stale ones. It is a warning, not an error — +// including undated objects is a legal thing to want, and real exported data +// contains such filters. +func checkDateFilters(view map[string]any, formats map[string]string, isDate func(string) bool, + path string, addIssue, warnIssue func(string, string, ...any)) { + nodes, _ := view["filters"].([]any) + if len(nodes) == 0 { + return + } + var walk func(nodes []any, path string, and bool, guarded map[string]bool) + walk = func(nodes []any, path string, and bool, guarded map[string]bool) { + // only an AND lets a sibling notEmpty guarantee anything: under an OR + // the comparison can be reached without it + scope := guarded + if and { + scope = map[string]bool{} + for k := range guarded { + scope[k] = true + } + for _, raw := range nodes { + n, ok := raw.(map[string]any) + if !ok { + continue + } + cond, _ := n["condition"].(string) + if prop, _ := n[memberProperty].(string); prop != "" && + (cond == "not_empty" || cond == "exists") { + scope[prop] = true + } + } + } + for i, raw := range nodes { + n, ok := raw.(map[string]any) + if !ok { + continue + } + nPath := fmt.Sprintf("%s/%d", path, i) + if sub, isGroup := n["filters"].([]any); isGroup { + op, _ := n["operator"].(string) + childScope := scope + if op == "or" { + // an `empty` sibling on the same property under an OR is + // intent to INCLUDE the undated objects ("… OR dueDate IS + // EMPTY") — warning that the comparison also matches them + // would contradict the filter's own text + childScope = map[string]bool{} + for k := range scope { + childScope[k] = true + } + for _, subRaw := range sub { + leaf, isLeaf := subRaw.(map[string]any) + if !isLeaf { + continue + } + cond, _ := leaf["condition"].(string) + if prop, _ := leaf[memberProperty].(string); prop != "" && cond == "empty" { + childScope[prop] = true + } + } + } + walk(sub, nPath+"/filters", op != "or", childScope) + continue + } + // the day-count presets read their operand from value; without + // one the count is 0, which quietly means "today" rather than + // "n days ago" (pkg/lib/database.getDateRange) — but only where + // the preset's range is applied at all, which takes BOTH halves + // of transformDateFilter's own gate: a date property (it returns + // a filter of any other format untouched, before any range is + // computed) and one of the six conditions that substitute the + // range (datePresetConditions). A count nothing reads is not + // missing, and rejecting the document for it refused one the app + // runs exactly as written. + if preset, _ := n["date_preset"].(string); preset != "" { + _, counts := countingPresetNames[preset] + leafCond, _ := n["condition"].(string) + _, applies := datePresetConditions[leafCond] + prop, _ := n[memberProperty].(string) + switch { + case !applies: + // the condition in front of us settles it: this preset + // decides nothing, and a view written as "edited in the + // last week" matches on the condition alone. A WARNING + // and not an error, because export must stay lossless — + // stored filters carry these pairs, the app keeps them as + // UI state, and refusing them would make one unexportable + // object out of every one that has one (§11, I1). + // + // The format half of the same gate stays silent on + // purpose: on the document path a filter's format usually + // comes from outside the document, so "not a date" there + // is as often "not known here", and a warning that fires + // on a correct filter is what makes every warning cheaper + // to ignore (§12). + under := "a leaf with no condition" + if leafCond != "" { + under = fmt.Sprintf("condition %q", leafCond) + } + warnIssue(nPath, "date_preset %q is ignored under %s; a preset's range is applied on %s", + preset, under, strings.Join(sortedKeys(datePresetConditions), " · ")) + case counts && isDate(prop): + // the operand has to BE a day count, not merely be + // present: the engine reads it with domain.Value.Int64, + // which answers 0 for a null, a string or anything else + // that is not a number — the very reading this message + // warns about. A presence-only rule refused the missing + // operand and admitted `"value": null`, which means the + // same thing and says it less honestly. + v, has := n["value"] + switch { + case !has: + addIssue(nPath, "date_preset %q needs a day count in \"value\"; without one it means 0 days, i.e. today", preset) + default: + if fault := dayCountFault(v); fault != "" { + addIssue(nPath+"/value", "date_preset %q needs a day count in \"value\": %s", preset, fault) + } + } + } + } + // a dynamic filter token resolves to an object id, so it can + // only match an object/file property; anywhere else it is + // compared as a literal string and matches nothing. A WARNING + // and not an error, for the reason the date-preset gate above + // gives: export must stay lossless — stored filters carry this + // pair (a template token on a text-declared property is real + // stored data), export wrote it with nothing to say, and this + // package's own Validate then refused the document it had just + // emitted (I1, the one invariant break a sustained attack + // found). One stored filter must not make an object + // unexportable. + if prop, _ := n[memberProperty].(string); prop != "" { + if f, declared := formats[prop]; declared && f != "objects" && f != "files" { + for _, tok := range filterTemplateValues(n["value"]) { + warnIssue(nPath+"/value", + "%q resolves to an object id and cannot match %q (format %q); the filter matches nothing until the property is object- or file-valued", tok, prop, f) + } + } + } + cond, _ := n["condition"].(string) + if cond != "less" && cond != "less_or_equal" { + continue + } + prop, _ := n[memberProperty].(string) + if !isDate(prop) || scope[prop] { + continue + } + warnIssue(nPath, "%q on date %q also matches objects with no %s; "+ + "pair it with a %q leaf in an \"and\" group to exclude them", + cond, prop, prop, "not_empty") + } + } + walk(nodes, path+"/filters", true, map[string]bool{}) +} + +// filterTemplateValues returns the dynamic filter tokens (§6.2) inside a +// filter value, which may be a bare string or an array of them. +func filterTemplateValues(v any) []string { + var out []string + switch x := v.(type) { + case string: + if isFilterTemplate(x) { + out = append(out, x) + } + case []any: + for _, e := range x { + if s, ok := e.(string); ok && isFilterTemplate(s) { + out = append(out, s) + } + } + } + return out +} + +// warnKeySpellingHygiene walks the authored key surfaces — the `properties` +// member names, both legends' keys, and `option_ids` outer keys — and warns +// about spellings carrying edge whitespace or invisible (default-ignorable) +// code points. A warning and only a warning: the format spells names +// verbatim and does not trim, so the document is valid — but such a key can +// only be matched by reproducing bytes the eye cannot check, which is worth +// one line to the caller per spelling. +func warnKeySpellingHygiene(doc map[string]any, warn func(path, format string, args ...any)) { + report := func(member, term string) { + reason := keySpellingHygieneIssue(term) + if reason == "" { + return + } + warn("/"+member+"/"+escapeJSONPointer(term), + "the key spelling %q %s — it is carried exactly as written, and an exact "+ + "match must reproduce the invisible bytes; the forgiving fold bridges the "+ + "near-miss, and a cleanup belongs where the property is named", term, reason) + } + for _, member := range []string{"properties", memberPropertyInternalKeys, + memberTypeInternalKeys, "option_ids"} { + if m, _ := doc[member].(map[string]any); m != nil { + for _, term := range sortedMapKeys(m) { + report(member, term) + } + } + } +} + +// warnNFCTwinSpellings reports, per key-spelling map, every pair of member +// names that are one name in two Unicode normal forms (§3: a NAME is spelled +// NFC on the wire). Byte-distinct, so JSON admits both, and rendered +// identically, so no reader can tell them apart — which is exactly how a +// hostile or hand-edited document plants an indistinguishable twin of a real +// property. %+q spells the code points apart where %q would print the same +// glyphs twice. A warning, not a refusal — see the semanticIssues call site. +func warnNFCTwinSpellings(doc map[string]any, warn func(path, format string, args ...any)) { + for _, member := range []string{"properties", memberPropertyInternalKeys, + memberTypeInternalKeys, "option_ids"} { + m, _ := doc[member].(map[string]any) + if m == nil { + continue + } + firstSpelling := map[string]string{} + for _, term := range sortedMapKeys(m) { + canonical := nfcTerm(term) + first, seen := firstSpelling[canonical] + if !seen { + firstSpelling[canonical] = term + continue + } + warn("/"+member+"/"+escapeJSONPointer(term), + "%+q and %+q are one name in two Unicode normal forms — byte-distinct, "+ + "rendered identically; NFC is the canonical spelling (§3), and both "+ + "resolve through it unless a legend or a live stored key binds the "+ + "exact bytes", + first, term) + } + } +} + +// keySpellingHygieneIssue names what is invisibly wrong with a key spelling, +// or "" when nothing is. The two hazard classes are the measured ones: edge +// whitespace ('Email 📧 ') and default-ignorable code points (two production +// names carry a variation selector). +func keySpellingHygieneIssue(term string) string { + if strings.TrimSpace(term) != term { + return "carries edge whitespace" + } + for _, r := range term { + if unicode.Is(unicode.Variation_Selector, r) || + unicode.Is(unicode.Other_Default_Ignorable_Code_Point, r) || + unicode.Is(unicode.Cf, r) { + return fmt.Sprintf("carries the invisible code point U+%04X", r) + } + } + return "" +} diff --git a/pkg/lib/anyblockjson/validate_test.go b/pkg/lib/anyblockjson/validate_test.go new file mode 100644 index 0000000000..66fcad7189 --- /dev/null +++ b/pkg/lib/anyblockjson/validate_test.go @@ -0,0 +1,872 @@ +package anyblockjson + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// The whole reason this format exists is the generate → validate → feed-back +// loop (§12), so a confident wrong issue is worse than a verbose one: an +// agent told `/blocks/0/type: property "type" is not allowed` deletes `type`. +// Two schema mechanics produce those: `unevaluatedProperties: false` reports +// every property of an object whose type-specific subschema failed (its +// annotations are discarded), and an `anyOf` reports every branch it tried. +func TestValidate_ErrorsDoNotCascade(t *testing.T) { + issues := func(t *testing.T, doc string) []Issue { + err := Validate([]byte(doc)) + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + return ve.Issues + } + + t.Run("a bad type is one issue, not three", func(t *testing.T) { + // the camelCase spelling is now the plausible mistake: it is what the + // pre-snake_case draft used, and what a model trained on it emits + got := issues(t, `{"version": 2, "blocks": [ + {"type": "bulletedListItem", "text": "x"}]}`) + require.Len(t, got, 1, "got: %v", got) + assert.Equal(t, "/blocks/0/type", got[0].Path) + assert.Contains(t, got[0].Message, "value must be one of") + }) + + t.Run("a bad field type is one issue, not four", func(t *testing.T) { + got := issues(t, `{"version": 2, "blocks": [ + {"type": "checkbox", "checked": "yes", "text": "x"}]}`) + require.Len(t, got, 1, "got: %v", got) + assert.Equal(t, "/blocks/0/checked", got[0].Path) + assert.Contains(t, got[0].Message, "got string, want boolean") + }) + + t.Run("the anyOf branch the author meant is the one reported", func(t *testing.T) { + got := issues(t, `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], + "rows": [{"id": "r1", "cells": [{"type": "paragraph", "id": "x1", "text": "a"}]}]}]}`) + require.Len(t, got, 1, "got: %v", got) + assert.Equal(t, "/blocks/0/rows/0/cells/0/id", got[0].Path) + }) + + t.Run("a cell of no admissible shape names every shape once", func(t *testing.T) { + got := issues(t, `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], + "rows": [{"id": "r1", "cells": [7]}]}]}`) + require.Len(t, got, 1, "got: %v", got) + assert.Equal(t, "/blocks/0/rows/0/cells/0", got[0].Path) + for _, want := range []string{"number", "string", "null", "object", "array"} { + assert.Contains(t, got[0].Message, want) + } + }) + + t.Run("an unknown key is still reported when it is the only fault", func(t *testing.T) { + got := issues(t, `{"version": 2, "blocks": [ + {"type": "paragraph", "text": "x", "bogus": 1}]}`) + require.Len(t, got, 1, "got: %v", got) + assert.Equal(t, "/blocks/0/bogus", got[0].Path) + assert.Contains(t, got[0].Message, `property "bogus" is not allowed`) + }) + + t.Run("an unknown key survives a sibling error", func(t *testing.T) { + // suppression is aimed at names the schema knows and could not + // evaluate; a hallucinated key is never admissible, so the verdict + // on it stands and the agent gets both facts in one round + got := issues(t, `{"version": 2, "blocks": [ + {"type": "checkbox", "checked": "yes", "bogus": 1}]}`) + require.Len(t, got, 2, "got: %v", got) + paths := []string{got[0].Path, got[1].Path} + assert.Contains(t, paths, "/blocks/0/checked") + assert.Contains(t, paths, "/blocks/0/bogus") + }) + + t.Run("the children migration hint survives", func(t *testing.T) { + got := issues(t, `{"version": 2, "blocks": [ + {"type": "paragraph", "text": "x", "children": []}]}`) + require.Len(t, got, 1, "got: %v", got) + assert.Contains(t, got[0].Message, "nest with indent instead") + }) + + t.Run("a wrong field on the right type is still reported", func(t *testing.T) { + // `checked` belongs to checkbox, and nothing else in this block + // failed, so the closed-set verdict is trustworthy + got := issues(t, `{"version": 2, "blocks": [ + {"type": "paragraph", "checked": true}]}`) + require.Len(t, got, 1, "got: %v", got) + assert.Equal(t, "/blocks/0/checked", got[0].Path) + }) +} + +// A tag-shaped sequence the grammar does not recognize is literal text and +// never an error (§10) — that leniency is what keeps a stored document +// readable across a version that adds a tag. But canonical export escapes +// those bytes (§8.2), so finding them unescaped means the text was +// hand-written or produced by a version that knows the tag, which is worth +// one warning and no more. +func TestValidate_UnknownTagStaysLiteralAndWarns(t *testing.T) { + warningsFor := func(t *testing.T, doc string) []Issue { + var got []Issue + require.NoError(t, ValidateWarn([]byte(doc), func(i Issue) { got = append(got, i) }), + "an unknown tag is not a validation error") + return got + } + + t.Run("unrecognized tag warns once and known tags do not", func(t *testing.T) { + got := warningsFor(t, `{"version": 2, "blocks": [ + {"type": "paragraph", "text": "x and y"}]}`) + require.Len(t, got, 1, "one warning per unrecognized name, not per occurrence") + assert.Equal(t, "/blocks/0/text", got[0].Path) + assert.Contains(t, got[0].Message, `"x\\"}]}`)) + }) + + t.Run("a table cell string is warned about too", func(t *testing.T) { + got := warningsFor(t, `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], + "rows": [{"id": "r1", "cells": ["hi"]}]}]}`) + require.Len(t, got, 1) + assert.Equal(t, "/blocks/0/rows/0/cells/0", got[0].Path) + }) +} + +func TestValidate_Valid(t *testing.T) { + tests := []struct { + name string + doc string + }{ + {"minimal", `{"version": 2}`}, + {"envelope", `{ + "$schema": "https://schemas.anytype.io/anyblock/1.0/object.schema.json", + "version": 2, + "id": "bafyrei123", + "type": "page", + "icon": {"format": "emoji", "emoji": "🔥"}, + "properties": {"name": "Test", "status": ["In progress"], "priority": 3, "done": false}, + "blocks": [ + {"id": "b1", "type": "heading_2", "text": "Goals"}, + {"id": "b2", "type": "paragraph", "text": "Ship the **new export**"}, + {"type": "bulleted_list_item", "text": "item"}, + {"indent": 1, "type": "bulleted_list_item", "text": "nested"}, + {"type": "checkbox", "checked": true, "text": "Draft"}, + {"type": "code", "language": "go", "text": "func main() {}"}, + {"type": "divider", "style": "dots"}, + {"type": "row"}, + {"indent": 1, "type": "column"}, + {"indent": 2, "type": "paragraph", "text": "left"}, + {"indent": 1, "type": "column"}, + {"indent": 2, "type": "paragraph", "text": "right"} + ] + }`}, + {"table", `{"version": 2, "blocks": [ + {"type": "table", + "columns": [{"id": "c1"}, {"id": "c2", "width": 120}], + "rows": [ + {"id": "r1", "is_header": true, "cells": ["Name", "Status"]}, + {"id": "r2", "cells": ["Export", {"type": "checkbox", "checked": true, "text": "done"}]}, + {"id": "r3", "cells": [null]} + ]} + ]}`}, + {"dataview", `{"version": 2, "blocks": [ + {"type": "dataview", "object_id": "bafyset", + "properties": [{"property": "name", "format": "text"}, {"property": "status", "format": "select"}], + "views": [ + {"id": "v1", "type": "kanban", "name": "By status", "group_by": "status", + "sorts": [{"property": "dueDate", "direction": "asc", "empty_placement": "end"}], + "filters": [ + {"property": "dueDate", "condition": "less", "date_preset": "current_week"}, + {"operator": "or", "filters": [ + {"property": "done", "condition": "equal", "value": false}, + {"property": "done", "condition": "empty"} + ]} + ], + "columns": [{"property": "name"}, {"property": "status", "width": 30, "aggregation": "count_distinct", "align": "right"}]} + ]} + ]}`}, + {"template", `{"version": 2, "kind": "template", "type": "template", "template_for": "task"}`}, + {"collection items", `{"version": 2, "type": "collection", "items": ["obj1", "obj2"]}`}, + {"widget", `{"version": 2, "kind": "widget", "blocks": [ + {"type": "widget", "layout": "tree", "limit": 6}, + {"indent": 1, "type": "link", "object_id": "obj1"} + ]}`}, + {"explicit indent 0", `{"version": 2, "blocks": [{"indent": 0, "type": "paragraph", "text": "x"}]}`}, + {"cell array with descendants", `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [[ + {"type": "toggle", "text": "cell"}, + {"indent": 1, "type": "paragraph", "text": "nested"} + ]]}]} + ]}`}, + {"heading_4 alias", `{"version": 2, "blocks": [{"type": "heading_4", "text": "deep"}]}`}, + {"equation alias", `{"version": 2, "blocks": [{"type": "equation", "text": "E=mc^2"}]}`}, + {"option_ids", `{"version": 2, "properties": {"tag": ["High"], "c#_lang": ["C#"]}, + "option_ids": {"tag": {"import issue": "bafyreiabc", "High": "bafyreidef"}, + "c#_lang": {"C#": "bafyreighi"}}}`}, + // view-id uniqueness is scoped to the dataview BLOCK (§6.2): the app + // mints every set/collection/type default view as "default", and + // creating an inline set from one copies its views verbatim, so a + // page with two inline collections legitimately holds two "default"s + {"one view id in two dataviews", `{"version": 2, "blocks": [ + {"type": "dataview", "object_id": "bafyone", "views": [{"id": "default", "name": "A"}]}, + {"type": "dataview", "object_id": "bafytwo", "views": [{"id": "default", "name": "B"}]} + ]}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.NoError(t, Validate([]byte(tc.doc))) + }) + } +} + +func TestValidate_Invalid(t *testing.T) { + tests := []struct { + name string + doc string + wantMsg string // substring expected in the error + }{ + {"not json", `{`, "invalid JSON"}, + {"not object", `[1]`, "must be a JSON object"}, + {"version missing", `{"blocks": []}`, "version is required"}, + {"version newer", `{"version": 3}`, "newer than the supported version 2"}, + {"version zero", `{"version": 0}`, "unknown version"}, + {"unknown envelope field", `{"version": 2, "banana": true}`, "banana"}, + {"unknown kind", `{"version": 2, "kind": "banana"}`, "/kind"}, + {"unknown block type", `{"version": 2, "blocks": [{"type": "banana"}]}`, "/blocks/0"}, + {"block type missing", `{"version": 2, "blocks": [{"text": "x"}]}`, "/blocks/0"}, + {"unknown block prop", `{"version": 2, "blocks": [{"type": "paragraph", "banana": 1}]}`, "banana"}, + {"prop from wrong type", `{"version": 2, "blocks": [{"type": "paragraph", "checked": true}]}`, "checked"}, + {"bad align", `{"version": 2, "blocks": [{"type": "paragraph", "align": "top"}]}`, "align"}, + {"bad block id charset", `{"version": 2, "blocks": [{"type": "paragraph", "id": "a b"}]}`, "/blocks/0/id"}, + {"children removed from the format", `{"version": 2, "blocks": [{"type": "toggle", "children": [{"type": "paragraph"}]}]}`, "children"}, + {"first block indented", `{"version": 2, "blocks": [{"indent": 1, "type": "paragraph", "text": "x"}]}`, "first block must be at indent 0"}, + {"indent jump", `{"version": 2, "blocks": [ + {"type": "paragraph", "text": "a"}, + {"indent": 2, "type": "paragraph", "text": "b"} + ]}`, "indent 2 follows indent 0"}, + {"nested under leaf block", `{"version": 2, "blocks": [ + {"type": "divider"}, + {"indent": 1, "type": "paragraph", "text": "x"} + ]}`, "divider blocks cannot have children"}, + {"row child not column", `{"version": 2, "blocks": [ + {"type": "row"}, + {"indent": 1, "type": "paragraph", "text": "x"} + ]}`, "a row block can only contain column blocks"}, + {"indent above bound", `{"version": 2, "blocks": [{"indent": 33, "type": "paragraph", "text": "x"}]}`, "/blocks/0/indent"}, + {"negative indent", `{"version": 2, "blocks": [{"indent": -1, "type": "paragraph", "text": "x"}]}`, "/blocks/0/indent"}, + {"indent on bare cell block", `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [{"indent": 1, "type": "paragraph", "text": "x"}]}]} + ]}`, "/blocks/0/rows/0/cells/0"}, + {"id on cell array first block", `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [[ + {"id": "x", "type": "toggle", "text": "cell"}, + {"indent": 1, "type": "paragraph", "text": "nested"} + ]]}]} + ]}`, "cell blocks cannot carry an id"}, + {"duplicate ids", `{"version": 2, "blocks": [{"id": "b1", "type": "paragraph"}, {"id": "b1", "type": "quote"}]}`, "duplicate id"}, + {"derived cell id collision", `{"version": 2, "blocks": [ + {"id": "r1-c1", "type": "paragraph"}, + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": ["x"]}]} + ]}`, "duplicate id"}, + {"row with too many cells", `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": ["a", "b"]}]} + ]}`, "1 columns"}, + {"cell with id", `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": [{"id": "x", "type": "paragraph", "text": "a"}]}]} + ]}`, "/blocks/0/rows/0/cells/0"}, + {"table inner id with dash", `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c-1"}], "rows": []} + ]}`, "/blocks/0/columns/0/id"}, + {"template_for without the template kind", `{"version": 2, "type": "page", "template_for": "task"}`, "template_for"}, + // the type spelling has no say here any more: `kind` is the sole + // authority (§2), so a kindless document carrying template_for is + // refused at template_for whatever its type spells + {"template_for on a kindless document that spells the template type", `{"version": 2, "type": "template", "template_for": "task"}`, "/template_for"}, + {"template_for with no type at all", `{"version": 2, "kind": "template", "template_for": "task"}`, "template_for"}, + {"language and fields.lang conflict", `{"version": 2, "blocks": [ + {"type": "code", "language": "go", "fields": {"lang": "go"}} + ]}`, "fields.lang"}, + {"inline markup error", `{"version": 2, "blocks": [{"type": "paragraph", "text": "unclosed"}]}`, "/blocks/0/text"}, + {"inline markup error in cell", `{"version": 2, "blocks": [ + {"type": "table", "columns": [{"id": "c1"}], "rows": [{"id": "r1", "cells": ["x"]}]} + ]}`, "/blocks/0/rows/0/cells/0"}, + {"an option_ids spelling with a control character", + `{"version": 2, "option_ids": {"a\nb": {"High": "bafy1"}}}`, + `/option_ids/a` + "\n" + `b: option_ids property spelling "a\nb" carries a control character`}, + {"an empty option name", + `{"version": 2, "properties": {"tag": ["High"]}, "option_ids": {"tag": {"": "bafy1"}}}`, + `/option_ids/tag/: option name is empty`}, + {"filter mixing group and leaf", `{"version": 2, "blocks": [ + {"type": "dataview", "views": [{"id": "v", "filters": [{"operator": "and", "property": "x", "filters": []}]}]} + ]}`, "/blocks/0/views/0/filters/0"}, + {"reserved compact filter field", `{"version": 2, "blocks": [ + {"type": "dataview", "views": [{"id": "v", "filter": "done = false"}]} + ]}`, "filter"}, + // §6.2: view ids are unique WITHIN a dataview block. Until this, + // views[].id was the one id slot in the document with no uniqueness + // check at all — invalid but unvalidated on every channel, create and + // import included. + {"duplicate view id in one dataview", `{"version": 2, "blocks": [ + {"type": "dataview", "views": [{"id": "v1", "name": "A"}, {"id": "v1", "name": "B"}]} + ]}`, `duplicate view id "v1" in this dataview`}, + {"duplicate view id path", `{"version": 2, "blocks": [ + {"type": "dataview", "views": [{"id": "v1", "name": "A"}, {"id": "v1", "name": "B"}]} + ]}`, "/blocks/0/views/1/id"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := Validate([]byte(tc.doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantMsg) + }) + } +} + +func TestValidate_NewerFormatHint(t *testing.T) { + // the version integer is the sole authority on format identity (§10): a + // document declaring a newer one is rejected outright, named in the error, + // and never reaches schema validation + t.Run("newer version is rejected and named", func(t *testing.T) { + // given + doc := `{"version": 3, "blocks": [{"type": "paragraph", "sparkles": true}]}` + + // when + err := Validate([]byte(doc)) + + // then + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + assert.True(t, ve.NewerFormat) + assert.True(t, strings.Contains(err.Error(), "newer version")) + assert.True(t, strings.Contains(err.Error(), "3")) + // the unknown field never got a chance to produce a constraint failure + assert.False(t, strings.Contains(err.Error(), "sparkles")) + }) + + t.Run("$schema does not affect format identity", func(t *testing.T) { + // a stale or invented $schema is decorative; only "version" gates + // given + doc := `{ + "$schema": "https://schemas.anytype.io/anyblock/9/object.schema.json", + "version": 2, + "blocks": [{"type": "paragraph", "text": "fine"}] + }` + + // when + err := Validate([]byte(doc)) + + // then + require.NoError(t, err) + }) +} + +// The version gate is the sole authority on format identity (§10), and it has +// three verdicts, not two. Version 2 is the frozen grammar; anything newer is +// refused outright with NewerFormat set; and version 1 — the integer every +// draft carried while the grammar was still moving — is refused as a +// pre-freeze draft rather than migrated, because the revisions it spans (the +// three legends that replaced `refs`, the relation lift, the +// `relation`→`property` rename) are several grammars under one number, so +// there is nothing to migrate FROM (§15 #9). +// +// How this can fail: spell the pre-freeze refusal as `v < FormatVersion` and +// it is silently right today and silently wrong at version 3, when it would +// refuse the first documents this reader is supposed to migrate. +func TestValidate_VersionGate(t *testing.T) { + t.Run("the pre-freeze integer is refused at /version, with the repair named", func(t *testing.T) { + // given a document that is otherwise perfectly well-formed: only the + // version says it predates the freeze + doc := []byte(`{"version": 1, "blocks": [{"type": "paragraph", "text": "fine"}]}`) + + // when + err := Validate(doc) + + // then + require.Error(t, err) + var ve *ValidationError + require.ErrorAs(t, err, &ve) + assert.False(t, ve.NewerFormat, "a draft is not a newer format, and the caller must not be told to upgrade") + require.Len(t, ve.Issues, 1, "the gate runs before the schema, so nothing else gets a verdict") + assert.Equal(t, "/version", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, "pre-freeze") + assert.Contains(t, ve.Issues[0].Message, "Re-export", "the message names the repair") + assert.Contains(t, ve.Issues[0].Message, strconv.Itoa(FormatVersion)) + + _, _, uerr := Unmarshal(doc, Options{}) + require.Error(t, uerr, "Validate and Unmarshal agree (§11 I2)") + }) + + t.Run("the frozen integer is accepted", func(t *testing.T) { + require.NoError(t, Validate([]byte(`{"version": 2}`))) + assert.Equal(t, 2, FormatVersion, "and 2 is what this reader writes") + }) + + t.Run("a newer integer keeps its own verdict", func(t *testing.T) { + err := Validate([]byte(`{"version": 3}`)) + require.Error(t, err) + var ve *ValidationError + require.ErrorAs(t, err, &ve) + assert.True(t, ve.NewerFormat) + assert.Contains(t, err.Error(), "newer than the supported version 2") + assert.NotContains(t, err.Error(), "pre-freeze", + "the two refusals must not be told through one message") + }) + + // §10: `index.json` and the property dictionary share the version number + // and the same rules, and a bundle is versioned as one artifact + t.Run("every grammar shares the gate", func(t *testing.T) { + for name, refuse := range map[string]func([]byte) error{ + "index": func(b []byte) error { _, err := UnmarshalIndex(b); return err }, + "dictionary": func(b []byte) error { + _, err := UnmarshalPropertyDictionary(b) + return err + }, + } { + t.Run(name, func(t *testing.T) { + err := refuse([]byte(`{"version": 1}`)) + require.Error(t, err) + var ve *ValidationError + require.ErrorAs(t, err, &ve) + assert.False(t, ve.NewerFormat) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/version", ve.Issues[0].Path) + assert.Contains(t, ve.Issues[0].Message, "pre-freeze") + }) + } + }) +} + +// TestVersionIdentity pins the one copy of the format version the compiler +// cannot keep honest: the $id and the version const inside each embedded +// schema file. The Go URLs are derived from FormatVersion, so a bump moves +// them automatically — this catches the JSON that a bump must move by hand. +func TestVersionIdentity(t *testing.T) { + // given + want := map[string]struct { + raw []byte + url string + }{ + "object": {raw: schemaJSON, url: SchemaURL}, + "index": {raw: indexSchemaJSON, url: IndexSchemaURL}, + "properties": {raw: propertiesSchemaJSON, url: PropertiesSchemaURL}, + } + + for name, tc := range want { + t.Run(name, func(t *testing.T) { + // when + var got struct { + Id string `json:"$id"` + Version struct { + Const *int `json:"const"` + } `json:"-"` + } + require.NoError(t, json.Unmarshal(tc.raw, &got)) + + var props struct { + Properties struct { + Version struct { + Const *int `json:"const"` + } `json:"version"` + } `json:"properties"` + } + require.NoError(t, json.Unmarshal(tc.raw, &props)) + + // then + assert.Equal(t, tc.url, got.Id, "schema $id must equal the derived URL") + require.NotNil(t, props.Properties.Version.Const, "schema must pin the version") + assert.Equal(t, FormatVersion, *props.Properties.Version.Const) + assert.True(t, strings.HasPrefix(tc.url, schemaBaseURL+strconv.Itoa(FormatVersion)+"/"), + "URL must carry FormatVersion and no minor axis") + }) + } +} + +func TestValidate_PathAddressing(t *testing.T) { + doc := `{"version": 2, "blocks": [ + {"type": "paragraph", "text": "fine"}, + {"type": "toggle", "text": "parent"}, + {"indent": 1, "type": "paragraph", "text": "bad here"} + ]}` + err := Validate([]byte(doc)) + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/blocks/2/text", ve.Issues[0].Path) +} + +// A key slot the schema constrains through `propertyNames` — the `properties` +// map, the `property_internal_keys` legend, both levels of `option_ids` (§3, §9a) — has +// to name the member that broke the rule, like every other issue §12 promises. The +// schema cannot: `propertyNames` validates each name as a standalone string +// instance, so the library's verdict carries neither the enclosing object's +// location nor, for a length bound, the name itself. A 200-character property +// key came back as `maxLength: got 200, want 128` at the document ROOT, which +// tells an agent running the generate → validate → feed-back loop (§13) +// nothing it can act on. The rule stays in the published schema — an external +// validator runs that and nothing else — and is restated where the key is in +// hand, which is the verdict this package reports. +func TestValidate_KeySlotIssuesNameTheOffendingMember(t *testing.T) { + long := strings.Repeat("a", maxPropertyKeyLen+1) + tests := []struct { + name string + doc string + wantPath string + wantIn []string + }{ + { + name: "an over-long property key", + doc: `{"version": 2, "properties": {"` + long + `": "x"}}`, + wantPath: "/properties/" + long, + wantIn: []string{long, "129", "128"}, + }, + { + name: "a property key carrying a control character", + doc: `{"version": 2, "properties": {"a\nb": "x"}}`, + wantPath: "/properties/a\nb", + wantIn: []string{`"a\nb"`, "control character"}, + }, + { + name: "the empty property key", + doc: `{"version": 2, "properties": {"": "x"}}`, + wantPath: "/properties/", + wantIn: []string{"empty"}, + }, + { + name: "an unwritable legend spelling", + doc: `{"version": 2, "property_internal_keys": {"a\nb": "due_date"}}`, + wantPath: "/property_internal_keys/a\nb", + wantIn: []string{`"a\nb"`, "control character"}, + }, + { + name: "an unwritable legend stored key", + doc: `{"version": 2, "property_internal_keys": {"prio": "` + long + `"}}`, + wantPath: "/property_internal_keys/prio", + wantIn: []string{long, "129", "128"}, + }, + { + name: "an empty legend stored key", + doc: `{"version": 2, "property_internal_keys": {"prio": ""}}`, + wantPath: "/property_internal_keys/prio", + wantIn: []string{"empty"}, + }, + { + name: "an option_ids spelling past the bound", + doc: `{"version": 2, "option_ids": {"` + long + `": {"High": "bafyreiabc"}}}`, + wantPath: "/option_ids/" + long, + wantIn: []string{long, "129", "128"}, + }, + { + // the INNER propertyNames, whose only rule is non-empty. Its own + // site in the schema is reported at the document root without + // this case (§12), and the pointer has to reach the level too — + // `/option_ids/tag/` is the empty member of `tag`'s map. + name: "an empty option name", + doc: `{"version": 2, "option_ids": {"tag": {"": "bafyreiabc"}}}`, + wantPath: "/option_ids/tag/", + wantIn: []string{"empty"}, + }, + // A spelling carrying a pointer metacharacter is escaped (RFC 6901), + // and the escape is what keeps the count at one: the schema's own + // verdict on the same value is suppressed through a ledger keyed by + // pointer, so an unescaped location missed it and the one empty value + // was reported three times, twice at a location the document has no + // member at. Both metacharacters are legal in a stored key and in a + // spelling — the writable-key rule bounds length and control + // characters, nothing else (§3). + { + name: "a legend spelling holding a slash", + doc: `{"version": 2, "property_internal_keys": {"a/b": ""}}`, + wantPath: "/property_internal_keys/a~1b", + wantIn: []string{"empty"}, + }, + { + name: "a type legend spelling holding a tilde", + doc: `{"version": 2, "type_internal_keys": {"a~b": ""}}`, + wantPath: "/type_internal_keys/a~0b", + wantIn: []string{"empty"}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := Validate([]byte(tc.doc)) + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + require.Len(t, ve.Issues, 1, "one member, one issue: %v", ve.Issues) + assert.Equal(t, tc.wantPath, ve.Issues[0].Path) + for _, want := range tc.wantIn { + assert.Contains(t, ve.Issues[0].Message, want) + } + }) + } +} + +// propertyNamesSites lists every place in a schema document that constrains +// property names, as JSON pointers. It descends through ARRAYS as well as +// objects, because half of this schema's subschemas hang off array-valued +// keywords — `allOf`, `anyOf`, `oneOf` (the block dispatch, the table cell, +// the filter node) — and a walk that only follows map values would report a +// clean sweep of a schema it had not finished reading. +func propertyNamesSites(node any, at string) []string { + var sites []string + switch n := node.(type) { + case map[string]any: + if _, has := n["propertyNames"]; has { + sites = append(sites, at) + } + for _, k := range sortedMapKeys(n) { + sites = append(sites, propertyNamesSites(n[k], at+"/"+escapeJSONPointer(k))...) + } + case []any: + for i, e := range n { + sites = append(sites, propertyNamesSites(e, fmt.Sprintf("%s/%d", at, i))...) + } + } + return sites +} + +// The restated rule has to cover every `propertyNames` the schema carries, or +// a key slot loses its addressable message the moment one is added — the +// schema's own verdict is still reported for anything this pass does not +// speak for, so the failure would be silent noise rather than a crash. +func TestValidate_EveryPropertyNamesSiteHasAnAddressableMessage(t *testing.T) { + var doc any + require.NoError(t, json.Unmarshal(SchemaJSON(), &doc)) + + sites := propertyNamesSites(doc, "") + sort.Strings(sites) + + assert.Equal(t, []string{ + "/$defs/propertyMap", // the properties map, via $ref from /properties + "/properties/option_ids", + // the option-name level: `option_ids` carries a propertyNames at BOTH + // levels and each owes its own case, which is the easy one to + // under-count + "/properties/option_ids/additionalProperties", + "/properties/property_internal_keys", + "/properties/type_internal_keys", + }, sites, "a new propertyNames site needs a case in propertyNameIssues") +} + +// …and the sweep above is only a guarantee if the walk reaches everywhere a +// site can be. Every site in the schema today is a plain map value, so the +// array descent is unfalsifiable against the schema itself: this fixture is +// what makes it fail when it stops working. The shapes are the ones the +// schema already uses for its subschemas — a block arm under `allOf`, a table +// cell arm under `anyOf`, a filter arm under `oneOf` — plus `prefixItems`, +// which a tuple-shaped slot would use. +func TestPropertyNamesSites_DescendsIntoArrayKeywords(t *testing.T) { + var doc any + require.NoError(t, json.Unmarshal([]byte(`{ + "allOf": [{"then": {"properties": {"legend": {"propertyNames": {"maxLength": 8}}}}}], + "$defs": { + "cell": {"anyOf": [{"type": "string"}, {"propertyNames": {"maxLength": 8}}]}, + "node": {"oneOf": [{"prefixItems": [{"propertyNames": {"maxLength": 8}}]}]} + } + }`), &doc)) + + sites := propertyNamesSites(doc, "") + sort.Strings(sites) + + assert.Equal(t, []string{ + "/$defs/cell/anyOf/1", + "/$defs/node/oneOf/0/prefixItems/0", + "/allOf/0/then/properties/legend", + }, sites) +} + +// TestValidate_IndentErrorMessage: the V1 message is the agent-facing repair +// loop — it must name both indents (§12). +func TestValidate_IndentErrorMessage(t *testing.T) { + doc := `{"version": 2, "blocks": [ + {"type": "paragraph", "text": "a"}, + {"indent": 1, "type": "paragraph", "text": "b"}, + {"indent": 3, "type": "paragraph", "text": "c"} + ]}` + err := Validate([]byte(doc)) + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve)) + require.Len(t, ve.Issues, 1) + assert.Equal(t, "/blocks/2", ve.Issues[0].Path) + assert.Equal(t, "indent 3 follows indent 1 — a block can be at most one level deeper than its predecessor", ve.Issues[0].Message) +} + +// TestNormalizeIndent: lenient mode clamps over-deep indents to the deepest +// establishable level with a path-addressed warning, and the imported state +// equals the equivalent valid document's (§4). +func TestNormalizeIndent(t *testing.T) { + invalid := `{"version": 2, "blocks": [ + {"id": "a", "type": "paragraph", "text": "a"}, + {"indent": 3, "id": "b", "type": "paragraph", "text": "b"} + ]}` + valid := `{"version": 2, "blocks": [ + {"id": "a", "type": "paragraph", "text": "a"}, + {"indent": 1, "id": "b", "type": "paragraph", "text": "b"} + ]}` + + // strict rejects + _, _, err := Unmarshal([]byte(invalid), Options{GenerateId: seqIds("g")}) + require.Error(t, err) + + var warnings []Issue + opts := Options{GenerateId: seqIds("g"), NormalizeIndent: true, OnWarning: func(i Issue) { warnings = append(warnings, i) }} + _, snap, err := Unmarshal([]byte(invalid), opts) + require.NoError(t, err) + require.Len(t, warnings, 1) + assert.Equal(t, "/blocks/1", warnings[0].Path) + assert.Contains(t, warnings[0].Message, "clamped to 1") + + _, want, err := Unmarshal([]byte(valid), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, want.Blocks, snap.Blocks) + + t.Run("first block clamps to 0", func(t *testing.T) { + doc := `{"version": 2, "blocks": [{"indent": 2, "id": "a", "type": "paragraph", "text": "a"}]}` + var w []Issue + o := Options{GenerateId: seqIds("g"), NormalizeIndent: true, OnWarning: func(i Issue) { w = append(w, i) }} + _, snap, err := Unmarshal([]byte(doc), o) + require.NoError(t, err) + require.Len(t, w, 1) + assert.Equal(t, "/blocks/0", w[0].Path) + assert.Contains(t, w[0].Message, "clamped to 0") + root := snap.Blocks[0] + assert.Equal(t, []string{"a"}, root.ChildrenIds) + }) + + t.Run("bounds stay errors in lenient mode", func(t *testing.T) { + doc := `{"version": 2, "blocks": [{"indent": 33, "type": "paragraph", "text": "x"}]}` + o := Options{GenerateId: seqIds("g"), NormalizeIndent: true} + _, _, err := Unmarshal([]byte(doc), o) + require.Error(t, err) + }) +} + +// TestValidate_PrefixProperty: pre-order plus the monotonicity rule makes +// every prefix of an exported blocks array a valid document — the truncation +// guarantee, made testable (§4). +func TestValidate_PrefixProperty(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, richSnapshot(), testOptions()) + require.NoError(t, err) + var doc struct { + Blocks []json.RawMessage `json:"blocks"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + require.NotEmpty(t, doc.Blocks) + for n := 0; n <= len(doc.Blocks); n++ { + parts := make([]string, 0, n) + for _, b := range doc.Blocks[:n] { + parts = append(parts, string(b)) + } + prefix := `{"version": 2, "blocks": [` + strings.Join(parts, ",") + `]}` + require.NoError(t, Validate([]byte(prefix)), "prefix of %d blocks", n) + } +} + +// TestValidate_UnknownEnvelopeMembersAreAddressedOneByOne pins the general +// rule the `refs` diagnostic is one case of: the envelope is closed with +// `additionalProperties: false`, which the library reports as ONE verdict per +// OBJECT — every unknown member named inside its text, at the object's own +// location. Inside a block the same fault comes back per member (blocks close +// with `unevaluatedProperties`), so before this the format's one promise about +// issues — "an issue names the member it is about" (§12) — held everywhere but +// the envelope, and exactly at the envelope is where a document written +// against an older grammar fails. +// +// The fixture carries SIX unknown members and asserts the whole ordered slice +// rather than a set, because the ordering is what a lost sort destroys and a +// two-member fixture catches that only ~1 run in 8 (measured). +func TestValidate_UnknownEnvelopeMembersAreAddressedOneByOne(t *testing.T) { + // given — one legend the format used to carry, plus names it never had. + // SIX of them, deliberately: the library builds its list by ranging over + // the instance's map, so with two members an unsorted reader still answers + // in sorted order by chance about seven runs in eight, and a `-count=1` CI + // run would miss a lost sort almost every time (measured: 23/200). Six + // members put a coincidence at roughly 1 in 720. + doc := `{"version": 2, "refs": {"idxxx": "bafyreitarget"}, + "zzz_unknown": 1, "aaa_unknown": 2, "mmm_unknown": 3, + "bbb_unknown": 4, "qqq_unknown": 5, + "blocks": [{"type": "paragraph", "text": "x"}]}` + want := []string{"/aaa_unknown", "/bbb_unknown", "/mmm_unknown", + "/qqq_unknown", "/refs", "/zzz_unknown"} + + // when + err := Validate([]byte(doc)) + + // then + require.Error(t, err) + var ve *ValidationError + require.True(t, errors.As(err, &ve), "got %v", err) + got := make([]string, 0, len(ve.Issues)) + for _, i := range ve.Issues { + got = append(got, i.Path) + } + assert.Equal(t, want, got, + "each unknown envelope member gets its own pointer, in a stable order") + for _, i := range ve.Issues { + assert.Contains(t, i.Message, "is not allowed") + assert.NotContains(t, i.Message, "zzz_unknown\", \"refs", + "no issue may still carry the merged list the split replaced") + } +} + +// The warning channel is only worth reading if what it says is worth acting +// on. Measured over a 77-space export, 77,446 warnings reached a reader and +// 371 of them told that reader anything: 93% were seven file-variant keys +// whose BUNDLED DECLARATION disagrees with every value the store has ever +// held, and 7% were export restating the bundled table's own target types and +// then reporting that the restatement is ignored. +// +// Both are the format arguing with itself about documents no author wrote. +// Silencing them takes the channel to 379 warnings, 1% of documents, and +// every survivor is a fact about the document: a view that cannot group, a +// date filter that silently widens, a rename that will not apply. +// +// How this can fail: silence the case where a stated target list DIFFERS from +// the bundle and a real discard goes unreported; drop the file-variant +// exemption and 71,736 warnings bury the 371 again. +func TestWarnings_TheChannelReportsTheDocument(t *testing.T) { + warn := func(doc string) []Issue { + var out []Issue + require.NoError(t, ValidateWarn([]byte(doc), func(i Issue) { out = append(out, i) }), doc) + return out + } + + t.Run("a mis-declared file-variant key is not the document's fault", func(t *testing.T) { + assert.Empty(t, warn(`{"version": 2, "id": "f1", "kind": "file_object", + "properties": {"file_variant_paths": ["a", "b"], "file_variant_widths": [100, 200]}}`), + "the bundled table declares these text and number; every stored value is a list") + }) + + t.Run("an ordinary shape mismatch still warns", func(t *testing.T) { + assert.NotEmpty(t, warn(`{"version": 2, "id": "o1", "properties": {"description": ["a list"]}}`), + "description really is a text property and a list really does read as empty") + }) + + t.Run("restating the bundle's own targets says nothing", func(t *testing.T) { + assert.Empty(t, warn(`{"version": 2, "kind": "object_type", "internal_key": "t", + "properties": {"name": "T"}, + "type_settings": {"property_definitions": [ + {"property": "creator", "format": "objects", "object_types": ["participant"]}]}}`), + "participant is exactly what the bundle says creator targets") + }) + + t.Run("but asking for something the bundle will not honour does", func(t *testing.T) { + w := warn(`{"version": 2, "kind": "object_type", "internal_key": "t", + "properties": {"name": "T"}, + "type_settings": {"property_definitions": [ + {"property": "creator", "format": "objects", "object_types": ["page"]}]}}`) + require.NotEmpty(t, w, "creator does not target page, and the list is discarded") + assert.Contains(t, w[0].Message, "ignored") + }) +} diff --git a/pkg/lib/anyblockjson/verbatimfirst_test.go b/pkg/lib/anyblockjson/verbatimfirst_test.go new file mode 100644 index 0000000000..2e59768d75 --- /dev/null +++ b/pkg/lib/anyblockjson/verbatimfirst_test.go @@ -0,0 +1,499 @@ +package anyblockjson + +// Verbatim-first, made real (§3): an exact stored key is always its own +// address, and the bundled slug table applies only to terms that are NOT +// stored keys. A package-only reader has no stored-key set, so the document +// itself must say which of its terms are stored keys — the legend's identity +// entries. Five reviews traced one family of silent corruptions to the two +// halves disagreeing about this rule; these tests pin the settled behavior, +// one confirmed defect each. + +import ( + "encoding/json" + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +type flatPropsDoc struct { + Properties map[string]any `json:"properties"` + PropertyKeys map[string]string `json:"property_internal_keys"` + Blocks []struct { + Type string `json:"type"` + Key string `json:"property"` + } `json:"blocks"` +} + +func decodeDoc(t *testing.T, data []byte) flatPropsDoc { + t.Helper() + var doc flatPropsDoc + require.NoError(t, json.Unmarshal(data, &doc)) + return doc +} + +// A stored key whose spelling the bundled table binds to a DIFFERENT key +// ("due_date" beside bundled dueDate) is written verbatim — but a package-only +// reader resolves spellings legend → bundled table → verbatim, so without a +// legend entry the value silently moves onto the bundled key. Export owes the +// identity entry: the document's own statement that the spelling is a stored +// key. +func TestExport_ShadowStoredKeyGetsAnIdentityEntry(t *testing.T) { + for _, tc := range []struct{ storedKey, bundledKey string }{ + {"due_date", "dueDate"}, + {"icon_emoji", "iconEmoji"}, + } { + t.Run(tc.storedKey, func(t *testing.T) { + // given + snap := customKeySnapshot(map[string]*types.Value{tc.storedKey: str("custom-value")}) + want := map[string]string{tc.storedKey: tc.storedKey} + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data)) + assert.Equal(t, want, decodeDoc(t, data).PropertyKeys, + "the identity entry is what tells a reader with no store that the spelling is a stored key") + + // and a package-only reader binds the value to the stored key + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "custom-value", back.Details.Fields[tc.storedKey].GetStringValue(), + "the stored key is its own address") + assert.NotContains(t, back.Details.Fields, tc.bundledKey, + "nothing may land on the bundled twin") + }) + } +} + +// A bundled key BESIDE its shadow twin: {"iconEmoji": …, "icon_emoji": …}. +// The bundled key backs off to its stored spelling (the twin owns the slug), +// the twin gets its identity entry, and the whole thing round-trips — Marshal +// used to emit a document its own Unmarshal refused as a duplicate binding. +func TestRoundTrip_BundledKeyBesideItsShadowTwin(t *testing.T) { + for _, tc := range []struct{ bundledKey, shadowKey string }{ + {"iconEmoji", "icon_emoji"}, + {"dueDate", "due_date"}, + } { + t.Run(tc.bundledKey, func(t *testing.T) { + // given + snap := customKeySnapshot(map[string]*types.Value{ + tc.bundledKey: str("bundled"), + tc.shadowKey: str("custom"), + }) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + + // then — I1 and I2 in one breath: what Marshal emits, Validate + // accepts and Unmarshal imports + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "emitted:\n%s", data) + assert.Equal(t, "bundled", back.Details.Fields[tc.bundledKey].GetStringValue()) + assert.Equal(t, "custom", back.Details.Fields[tc.shadowKey].GetStringValue()) + }) + } +} + +// A stored key spelled exactly like the api slug of an INTERNAL bundled key +// ("unique_key", next to stripped "uniqueKey"). Export writes it verbatim; +// without the identity entry, validation resolved the spelling through the +// bundled table, hit the deny rule, and Marshal emitted a document its own +// Validate rejected. +func TestRoundTrip_StoredKeySpelledLikeAnInternalSlug(t *testing.T) { + for _, storedKey := range []string{"unique_key", "space_id", "old_anytype_id", "source_file_path"} { + t.Run(storedKey, func(t *testing.T) { + // given + snap := customKeySnapshot(map[string]*types.Value{storedKey: str("x")}) + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{}) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "emitted:\n%s", data) + assert.Equal(t, "x", back.Details.Fields[storedKey].GetStringValue(), + "a custom key that shadows an internal slug is still a custom key") + }) + } +} + +// blockKeySnapshot is a page whose details and property blocks are the +// caller's to shape — the fixture for the term-ledger collisions. +func blockKeySnapshot(details map[string]*types.Value, blockKeys ...string) *model.SmartBlockSnapshotBase { + details["id"] = str("o1") + root := &model.Block{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + blocks := []*model.Block{root} + for i, key := range blockKeys { + b := &model.Block{Id: "rel" + string(rune('a'+i)), Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{Key: key}}} + root.ChildrenIds = append(root.ChildrenIds, b.Id) + blocks = append(blocks, b) + } + return &model.SmartBlockSnapshotBase{Blocks: blocks, Details: fields(details)} +} + +// The ledger, arm one: a block slot's spelling may not take a term that IS +// a stored key the document names. The vocabulary spells a custom key as +// "dueDate" — the literal stored key of a property the object holds — so +// the claimant degrades through the ladder: its own key is a minted bson +// id, so it takes ` ()`, and the legend inverts the suffix. +func TestExport_BlockSlugMayNotTakeAStoredKeysTerm(t *testing.T) { + // given + vocab := spaceVocabulary{slugOf: map[string]string{"6a32d4856761631534b22f85": "dueDate"}} + snap := blockKeySnapshot(map[string]*types.Value{ + "name": str("x"), + "dueDate": str("2026-07-06T08:44:05Z"), + }, "6a32d4856761631534b22f85") + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + + // then — the stored key keeps its own term; the claimant takes the suffix + require.NoError(t, err) + require.NoError(t, Validate(data)) + doc := decodeDoc(t, data) + require.Len(t, doc.Blocks, 1) + assert.Equal(t, "dueDate (b22f85)", doc.Blocks[0].Key, + "the term `dueDate` is taken — a stored key always keeps its own term (verbatim-first)") + assert.Equal(t, "6a32d4856761631534b22f85", doc.PropertyKeys["dueDate (b22f85)"], + "the suffix owes its inverse: no shipped table has ever heard of it") + + // a package-only reader gets both relations back intact + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.NotNil(t, back.Details.Fields["dueDate"], "dueDate must survive the round trip") + assert.Nil(t, back.Details.Fields["6a32d4856761631534b22f85"], + "the block's key is not a property on this object") + for _, b := range back.Blocks { + if c, ok := b.Content.(*model.BlockContentOfRelation); ok { + assert.Equal(t, "6a32d4856761631534b22f85", c.Relation.Key) + } + } + + // and so does the writer's own reader + _, back, err = Unmarshal(data, Options{GenerateId: seqIds("h"), Keys: vocab}) + require.NoError(t, err) + assert.NotNil(t, back.Details.Fields["dueDate"]) + assert.Nil(t, back.Details.Fields["6a32d4856761631534b22f85"]) +} + +// The ledger, arm two: when the plan degrades a claimant (the term is +// another stored key on the object), EVERY slot naming that key takes the +// same degraded spelling — one key, one spelling, document-wide. The block +// slot used to record the plain slug anyway: one document, two spellings of +// one key, refused by its own importer. +func TestExport_BackedOffSlugBacksOffEverywhere(t *testing.T) { + // given + vocab := spaceVocabulary{slugOf: map[string]string{"6a32d4856761631534b22f85": "due_date"}} + snap := blockKeySnapshot(map[string]*types.Value{ + "due_date": str("shadow"), + "6a32d4856761631534b22f85": str("custom"), + }, "6a32d4856761631534b22f85") + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + doc := decodeDoc(t, data) + require.Len(t, doc.Blocks, 1) + assert.Equal(t, "due_date (b22f85)", doc.Blocks[0].Key, + "one key, one spelling, document-wide — the property slot and the block slot agree") + assert.Equal(t, "custom", doc.Properties["due_date (b22f85)"]) + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err, "emitted:\n%s", data) + assert.Equal(t, "shadow", back.Details.Fields["due_date"].GetStringValue()) + assert.Equal(t, "custom", back.Details.Fields["6a32d4856761631534b22f85"].GetStringValue()) +} + +// The ledger, arm three: two keys, one spelling, in slots outside +// /properties. The last recordPropertyKey used to win the single legend +// entry, so after a round trip BOTH property blocks named the second key — +// two relations collapsed into one. +func TestExport_TwoKeysOneSlugKeepDistinctTerms(t *testing.T) { + // given + vocab := twinSlugVocab{a: "aaa111", b: "bbb222", slug: "priority"} + snap := blockKeySnapshot(map[string]*types.Value{"name": str("x")}, "aaa111", "bbb222") + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + + // then — EVERY claimant of the contested spelling degrades: both keys + // are readable, so both take their stored keys, and neither depends on + // which slot happened to claim first + require.NoError(t, err) + require.NoError(t, Validate(data)) + doc := decodeDoc(t, data) + require.Len(t, doc.Blocks, 2) + assert.Equal(t, "aaa111", doc.Blocks[0].Key) + assert.Equal(t, "bbb222", doc.Blocks[1].Key) + assert.Equal(t, map[string]string{"aaa111": "aaa111", "bbb222": "bbb222"}, + doc.PropertyKeys, + "each names itself — no bundled table binds either term, so nothing else "+ + "in the document says they are stored keys") + + // and the two relations are still two relations after the round trip + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + var keys []string + for _, b := range back.Blocks { + if c, ok := b.Content.(*model.BlockContentOfRelation); ok { + keys = append(keys, c.Relation.Key) + } + } + assert.Equal(t, []string{"aaa111", "bbb222"}, keys) +} + +// twinSlugVocab spells two stored keys as one slug — the shape two properties +// minted from the same name really have. +type twinSlugVocab struct{ a, b, slug string } + +func (v twinSlugVocab) PropertySlug(key string) string { + if key == v.a || key == v.b { + return v.slug + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +func (v twinSlugVocab) PropertyKey(slug string) (string, bool) { + if slug == v.slug { + return v.a, true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +func (v twinSlugVocab) TypeSlug(key string) string { return BundledKeyVocabulary{}.TypeSlug(key) } +func (v twinSlugVocab) TypeKey(slug string) (string, bool) { + return BundledKeyVocabulary{}.TypeKey(slug) +} + +// A legend value is a stored key, and admission judges it like one: the deny +// rule runs on the value itself, not only on whatever /properties member +// happens to spell the entry. Unchecked, {"sneaky": "uniqueKey"} was a +// laundering primitive — admission resolved ONE hop and checked the value +// only for writability. +func TestValidate_LegendValueObeysTheDenyRule(t *testing.T) { + for name, doc := range map[string]string{ + "resolution vector": `{"version": 2, "property_internal_keys": {"sneaky": "uniqueKey"}}`, + "merge selector": `{"version": 2, "property_internal_keys": {"p": "oldAnytypeID"}}`, + "envelope key": `{"version": 2, "property_internal_keys": {"myid": "id"}}`, + "stripped key": `{"version": 2, "property_internal_keys": {"s": "spaceId"}}`, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(doc)) + require.Error(t, err, doc) + assert.Contains(t, err.Error(), "/property_internal_keys/") + _, _, unmErr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr, "I2: Unmarshal refuses what Validate refuses") + }) + } +} + +// The other half of the laundering defect, pinned as settled behavior: under +// verbatim-first a legend value of "unique_key" names the CUSTOM stored key — +// never bundled uniqueKey — and the re-export is a document validation +// accepts (it used to emit /properties/unique_key with no legend entry and +// then reject its own output). +func TestImport_LegendBindingToACustomShadowKeyIsNotLaundering(t *testing.T) { + doc := `{"version": 2, "id": "o1", "property_internal_keys": {"x": "unique_key"}, "properties": {"x": "ot-page"}}` + require.NoError(t, Validate([]byte(doc))) + sbType, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "ot-page", snap.Details.Fields["unique_key"].GetStringValue(), + "the value binds to the custom key the legend names") + assert.NotContains(t, snap.Details.Fields, "uniqueKey", + "nothing lands on the importer's resolution vector") + + out, err := Marshal(sbType, snap, Options{}) + require.NoError(t, err) + require.NoError(t, Validate(out), "re-export:\n%s", out) +} + +// deniedKeyVocab spells an internal key — plausible, and not only for a +// hand-rolled table: a denied key has a display name like any other (the +// bundled table spells `uniqueKey` as "Unique object key"), so the guard has +// to hold at the claim step rather than inside any vocabulary. +type deniedKeyVocab struct{ BundledKeyVocabulary } + +func (deniedKeyVocab) PropertySlug(key string) string { + if key == "uniqueKey" { + return "sneaky" + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +// Export never writes a legend whose value the deny rule refuses: a denied +// stored key never takes a slug, because the slug's legend entry would carry +// that value. The verbatim key is the one honest rendering. +func TestExport_ADeniedKeyNeverTakesASlug(t *testing.T) { + t.Run("property block", func(t *testing.T) { + // given + snap := blockKeySnapshot(map[string]*types.Value{"name": str("x")}, "uniqueKey") + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: deniedKeyVocab{}}) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + doc := decodeDoc(t, data) + require.Len(t, doc.Blocks, 1) + assert.Equal(t, "uniqueKey", doc.Blocks[0].Key) + assert.Empty(t, doc.PropertyKeys, "a denied key cannot be a legend value") + }) + + t.Run("type properties key slot", func(t *testing.T) { + // given — the confirmed input: a PropertyResolver answering an + // internal key for a recommended-list entry + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "t1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("t1"), + "recommendedRelations": {Kind: &types.Value_ListValue{ListValue: &types.ListValue{ + Values: []*types.Value{str("p1")}}}}, + }), + } + resolver := stubPropertyResolver{byId: map[string]PropertyDefinition{ + "p1": {Key: "uniqueKey"}, + }} + + // when + data, err := Marshal(model.SmartBlockType_STType, snap, + Options{Keys: deniedKeyVocab{}, ResolveProperties: resolver}) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + var doc struct { + TypeSettings struct { + PropertyDefinitions []struct { + Key string `json:"property"` + } `json:"property_definitions"` + } `json:"type_settings"` + PropertyKeys map[string]string `json:"property_internal_keys"` + } + require.NoError(t, json.Unmarshal(data, &doc)) + require.Len(t, doc.TypeSettings.PropertyDefinitions, 1) + assert.Equal(t, "uniqueKey", doc.TypeSettings.PropertyDefinitions[0].Key) + assert.Empty(t, doc.PropertyKeys) + }) +} + +type stubPropertyResolver struct{ byId map[string]PropertyDefinition } + +func (r stubPropertyResolver) PropertyById(id string) (PropertyDefinition, bool) { + def, ok := r.byId[id] + return def, ok +} + +func (r stubPropertyResolver) PropertyId(def PropertyDefinition) (string, bool) { return "", false } + +// envelopeSlugVocab spells stored keys as the two spellings validation +// refuses BEFORE any resolution ("id"/"type" belong to the envelope, and the +// legend cannot re-purpose them) — a property named "ID" or "Type" really +// produces this slug. +type envelopeSlugVocab struct{ BundledKeyVocabulary } + +func (envelopeSlugVocab) PropertySlug(key string) string { + switch key { + case "artist": + return "id" + case "myGenre": + return "type" + } + return BundledKeyVocabulary{}.PropertySlug(key) +} + +// writableSlug treats a spelling the deny rule refuses AS A SPELLING as +// unwritable, exactly as it does an over-long slug: the stored key is +// written instead, with a warning. Emitting the slug made Marshal produce +// {"properties": {"id": …}}, which its own Validate refuses and no legend +// can rescue. +func TestExport_ASlugRefusedAsASpellingFallsBackToTheStoredKey(t *testing.T) { + // given + snap := customKeySnapshot(map[string]*types.Value{"artist": str("v"), "myGenre": str("w")}) + var warned []Issue + + // when + data, err := Marshal(model.SmartBlockType_Page, snap, + Options{Keys: envelopeSlugVocab{}, OnWarning: func(i Issue) { warned = append(warned, i) }}) + + // then + require.NoError(t, err) + require.NoError(t, Validate(data), "emitted:\n%s", data) + doc := decodeDoc(t, data) + assert.Contains(t, doc.Properties, "artist") + assert.Contains(t, doc.Properties, "myGenre") + assert.NotContains(t, doc.Properties, "id") + assert.NotContains(t, doc.Properties, "type") + assert.NotEmpty(t, warned, "the fallback is reported, like every vocabulary answer export cannot honor") + + _, back, err := Unmarshal(data, Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, "v", back.Details.Fields["artist"].GetStringValue()) + assert.Equal(t, "w", back.Details.Fields["myGenre"].GetStringValue()) +} + +// blankKeyVocab resolves a spelling to the empty string — a vocabulary bug a +// caller can really ship, and one the import seam used to let through. +type blankKeyVocab struct{ BundledKeyVocabulary } + +func (blankKeyVocab) PropertyKey(slug string) (string, bool) { + if slug == "blank" { + return "", true + } + return BundledKeyVocabulary{}.PropertyKey(slug) +} + +// The seam admits only keys export could write: it ran the deny rule on the +// resolved key but not the writable-key rule, so a vocabulary resolving +// "blank" to "" landed details[""] — Validate clean, Unmarshal clean, and +// re-export then dropped the property with only a warning. A property lost +// in silence. +func TestImport_SeamRefusesAnUnwritableResolvedKey(t *testing.T) { + doc := `{"version": 2, "properties": {"blank": "x"}}` + require.NoError(t, Validate([]byte(doc)), + "the document's own chain resolves blank verbatim — Validate cannot see the vocabulary") + + _, _, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g"), Keys: blankKeyVocab{}}) + + require.Error(t, err, "an unwritable resolved key must be refused at the seam, like a denied one") + var ve *ValidationError + require.ErrorAs(t, err, &ve, "the refusal is path-addressed") + assert.Contains(t, err.Error(), "/properties/blank") +} + +// Validation mirrors ALL of the seam's refusals, not one of three: importer. +// build refuses two spellings binding onto one stored key, semanticIssues did +// not, so with DEFAULT Options a hand-written {"iconEmoji": …, "icon_emoji": +// …} split Validate from Unmarshal — the exact divergence I2 forbids. +func TestValidate_MirrorsTheSeamsDuplicateBindingRefusal(t *testing.T) { + for name, doc := range map[string]string{ + "bundled twin": `{"version": 2, "properties": {"pluralName": "a", "plural_name": "b"}}`, + "date twin": `{"version": 2, "properties": {"dueDate": "x", "due_date": "y"}}`, + "legend-induced": `{"version": 2, "property_internal_keys": {"prio": "customKey"}, "properties": {"prio": 1, "customKey": 2}}`, + } { + t.Run(name, func(t *testing.T) { + err := Validate([]byte(doc)) + require.Error(t, err, "Unmarshal refuses this document, so Validate must too (I2)") + assert.Contains(t, err.Error(), "both address") + + _, _, unmErr := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.Error(t, unmErr) + }) + } +} diff --git a/pkg/lib/anyblockjson/viewvocab.go b/pkg/lib/anyblockjson/viewvocab.go new file mode 100644 index 0000000000..3fb9124613 --- /dev/null +++ b/pkg/lib/anyblockjson/viewvocab.go @@ -0,0 +1,39 @@ +package anyblockjson + +// viewvocab.go exports the §6.2 dataview view vocabulary at fragment +// granularity — the enum name lists a surface editing views needs for +// validation and error text (the API's updateView op). The +// lists are the single source the API layer consumes, so the op's allowed +// values cannot drift from what the codec actually reads and writes; a test +// pins each list against its enum table. + +// ViewTypeNames lists the §6.2 view types in canonical order. `table` is the +// default (export omits it). +func ViewTypeNames() []string { + return []string{"table", "list", "gallery", "kanban", "calendar", "graph"} +} + +// ViewCardSizeNames lists the §6.2 card_size values. `small` is the default. +func ViewCardSizeNames() []string { + return []string{"small", "medium", "large"} +} + +// ViewListSizeNames lists the §6.2 list_size values. `compact` is the default. +func ViewListSizeNames() []string { + return []string{"compact", "regular"} +} + +// ColumnAlignNames lists the §6.2 column align values. +func ColumnAlignNames() []string { + return []string{"left", "center", "right", "justify"} +} + +// ColumnAggregationNames lists the §6.2 column aggregation values. Absent +// means none. +func ColumnAggregationNames() []string { + return []string{ + "count", "count_value", "count_distinct", "count_empty", "count_not_empty", + "percent_empty", "percent_not_empty", "sum", "average", "median", "min", + "max", "range", + } +} diff --git a/pkg/lib/anyblockjson/viewvocab_test.go b/pkg/lib/anyblockjson/viewvocab_test.go new file mode 100644 index 0000000000..55b568111a --- /dev/null +++ b/pkg/lib/anyblockjson/viewvocab_test.go @@ -0,0 +1,39 @@ +package anyblockjson + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestViewVocabularyMatchesEnumTables pins each exported §6.2 vocabulary list +// to the codec's own enum table: every exported name must be readable by the +// importer, and every name the exporter can emit must be exported. A new enum +// value added to a table without updating its list (or vice versa) fails here +// instead of silently splitting the vocabulary between the codec and the API +// surfaces that validate against it. +func TestViewVocabularyMatchesEnumTables(t *testing.T) { + pin := func(t *testing.T, exported []string, has func(string) bool, tableSize int) { + t.Helper() + assert.Len(t, exported, tableSize, "exported list and enum table must be the same size") + for _, name := range exported { + assert.True(t, has(name), "exported name %q must be in the enum table", name) + } + } + + t.Run("view types", func(t *testing.T) { + pin(t, ViewTypeNames(), viewTypeNames.has, len(viewTypeNames.toName)) + }) + t.Run("card sizes", func(t *testing.T) { + pin(t, ViewCardSizeNames(), cardSizeNames.has, len(cardSizeNames.toName)) + }) + t.Run("list sizes", func(t *testing.T) { + pin(t, ViewListSizeNames(), listSizeNames.has, len(listSizeNames.toName)) + }) + t.Run("column align", func(t *testing.T) { + pin(t, ColumnAlignNames(), alignNames.has, len(alignNames.toName)) + }) + t.Run("column aggregation", func(t *testing.T) { + pin(t, ColumnAggregationNames(), aggregationNames.has, len(aggregationNames.toName)) + }) +} diff --git a/pkg/lib/anyblockjson/widgetlimit_test.go b/pkg/lib/anyblockjson/widgetlimit_test.go new file mode 100644 index 0000000000..5294c126b8 --- /dev/null +++ b/pkg/lib/anyblockjson/widgetlimit_test.go @@ -0,0 +1,115 @@ +package anyblockjson + +// widgetlimit_test.go — the widget `limit` bound has ONE home. +// +// The index's flat widget caps `limit` at 100 — a deliberate product bound +// on sidebar listings (the corpus maximum is 50), enforced again by the +// widget-object lift, which keeps the whole document for anything above it. +// The widget BLOCK deliberately takes the whole int32 range instead: the +// block is the fidelity fallback for exactly the widget the index refuses, +// so the cap must not bind there or the fallback document would fail its own +// schema. Every sibling widget field (`layout`, `card_style`, `icon_size`, +// `description`) states its vocabulary once in the object schema and the +// index references it; `limit` inlined its own copy in the index, which is +// the drift channel this file closes: the cap now lives in +// `$defs/widgetListingLimit`, the index references it, the Go lift reads the +// same number through maxIndexWidgetLimit, and this test ties all four +// statements together so no one of them can move alone. + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// decodedSchema parses one embedded schema for structural assertions. +func decodedSchema(t *testing.T, raw []byte) map[string]any { + t.Helper() + var doc map[string]any + require.NoError(t, json.Unmarshal(raw, &doc)) + return doc +} + +// widgetBlockBranch finds the object schema's widget-block conditional and +// returns its `then.properties`, failing loudly when the dispatch reshuffles. +func widgetBlockBranch(t *testing.T, schema map[string]any) map[string]any { + t.Helper() + branches, ok := schemaAt(t, schema, "$defs", "blockCore", "allOf").([]any) + require.True(t, ok, "the block dispatch is an allOf of if/then branches") + for _, b := range branches { + branch, ok := b.(map[string]any) + if !ok { + continue + } + ifClause, ok := branch["if"].(map[string]any) + if !ok { + continue + } + if c, _ := schemaAt(t, ifClause, "properties", "type").(map[string]any); c != nil && + c["const"] == "widget" { + props, ok := schemaAt(t, branch, "then", "properties").(map[string]any) + require.True(t, ok) + return props + } + } + require.Fail(t, "no widget branch in the block dispatch") + return nil +} + +func TestWidgetLimit_OneStatementOfTheCap(t *testing.T) { + object := decodedSchema(t, SchemaJSON()) + index := decodedSchema(t, indexSchemaJSON) + authoringIndex := decodedSchema(t, authoringIndexSchemaJSON) + + t.Run("the shared def carries the cap the Go lift enforces", func(t *testing.T) { + def, ok := schemaAt(t, object, "$defs", "widgetListingLimit").(map[string]any) + require.True(t, ok, "the cap's one home is $defs/widgetListingLimit") + assert.Equal(t, "integer", def["type"]) + assert.Equal(t, float64(0), def["minimum"]) + assert.Equal(t, float64(maxIndexWidgetLimit), def["maximum"], + "the schema's cap and widgetObjectWidgets' bound are one number") + }) + + t.Run("the index references the shared def like its siblings", func(t *testing.T) { + limit, ok := schemaAt(t, index, "$defs", "widget", "properties", "limit").(map[string]any) + require.True(t, ok) + assert.Equal(t, SchemaURL+"#/$defs/widgetListingLimit", limit["$ref"], + "limit states the cap as one $ref into the object schema — a copy is the drift channel") + assert.NotContains(t, limit, "maximum", "no inline copy beside the $ref") + }) + + t.Run("the authoring index inlines the same number, pinned here", func(t *testing.T) { + // the authoring subset is self-contained by design (it inlines the + // layout enum too), so its copy is allowed — and chained to the same + // constant so it cannot drift alone + limit, ok := schemaAt(t, authoringIndex, "$defs", "widget", "properties", "limit").(map[string]any) + require.True(t, ok) + assert.Equal(t, float64(0), limit["minimum"]) + assert.Equal(t, float64(maxIndexWidgetLimit), limit["maximum"]) + }) + + t.Run("the widget BLOCK keeps the whole int32 range — the fidelity fallback", func(t *testing.T) { + limit, ok := widgetBlockBranch(t, object)["limit"].(map[string]any) + require.True(t, ok) + assert.Equal(t, float64(0), limit["minimum"]) + assert.Equal(t, float64(2147483647), limit["maximum"], + "a widget the index refuses (limit above the cap) travels as a full document, "+ + "and that document must validate — capping the block breaks the designed fallback") + }) + + t.Run("the two doors behave as the two statements say", func(t *testing.T) { + // the index: 100 in, 101 out + _, err := UnmarshalIndex([]byte(`{"version":2,"widgets":[{"target":"page-a","limit":100}]}`)) + require.NoError(t, err) + _, err = UnmarshalIndex([]byte(`{"version":2,"widgets":[{"target":"page-a","limit":101}]}`)) + require.Error(t, err) + assert.Contains(t, issuePaths(t, err), "/widgets/0/limit") + + // the block: 101 is a legal document — the fallback the lift's own + // refusal (pinned in widgetobject_test.go) depends on + assert.NoError(t, Validate([]byte( + `{"version":2,"id":"o1","blocks":[{"id":"b1","type":"widget","limit":101}]}`))) + }) +} diff --git a/pkg/lib/anyblockjson/widgetobject.go b/pkg/lib/anyblockjson/widgetobject.go new file mode 100644 index 0000000000..b3f1952d0f --- /dev/null +++ b/pkg/lib/anyblockjson/widgetobject.go @@ -0,0 +1,542 @@ +package anyblockjson + +// widgetobject.go — the sidebar's object, and why a bundle does not carry +// one (§2c). +// +// `kind: "widget"` is a hidden per-space object whose BLOCKS encode the +// sidebar: one widget wrapper block per widget, each with exactly one +// indented link child naming the target. index.json has a first-class +// `widgets` array for exactly this, so the document restates the index the +// way the space document restated it earlier — except the index used to +// say less than the blocks. It says everything now, so the document can go. +// +// That is not an assumption. Measured over a 77-space export: 218 wrapper +// blocks, 218 link children, in 218 perfectly regular pairs — no wrapper +// without its link, no link outside a wrapper, and no other content besides +// the root and, in 11 spaces, the editor's header scaffolding (§7: a Header +// layout holding one EMPTY title block, which export drops from every +// document). The details reduce the same way: +// +// createdDate 77 of 77 → dropped: 0 on every one of them, and a +// restored sidebar is created when it is +// restored (the space-document rule) +// isHidden 77 of 77 → constant true; the rebuild restates it +// layout 77 of 77 → constant `dashboard`; restated +// resolvedLayout 77 of 77 → constant `dashboard`; restated +// lastModifiedDate 49 of 77 → dropped, like createdDate +// autoWidgetTargets 21 of 77 → index.auto_widget_targets +// name 15 of 77 → "" on every one; a non-empty name keeps +// the document +// autoWidgetDisabled 2 of 77 → index.auto_widget_disabled +// +// (everything else the raw snapshots hold — backlinks, links, mentions, +// restrictions, snippet, creator, lastModifiedBy, lastOpenedDate, +// internalFlags, id, spaceId, type — is already stripped or transient §3.) +// +// So export omits it, `IndexFromWidgetObject` is the one place that says +// which block member becomes which index field, and `WidgetsSnapshot` is the +// one function that builds the object back — shared by cmd/anyblockconvert +// (the archive the importer installs) and the round-trip verifier (the +// reconstruction check), so the lift and the rebuild cannot drift apart +// silently. + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// WidgetsObjectId is the rebuilt widget snapshot's own id. Nothing derives +// from it: the importer replaces it with the space's derived Widgets id +// (objectid.widget.GetIDAndPayload returns spc.DerivedIDs().Widgets), and the +// root block is renamed to match before the state is built. It only has to +// be stable and impossible for a bundle object to claim — `widgets` is a +// reserved bundle id already (IsReservedBundleId, via the homepage table). +const WidgetsObjectId = "widgets" + +// widgetWrapperSuffix is the convention core/block/editor/widget uses when a +// wrapper's id has to be derived from its link's rather than random +// (widget.createBlock), so that two devices creating the same widget do not +// end up with two wrappers. +const widgetWrapperSuffix = "-wrapper" + +// the widget object's own preference details. The keys are the clients' +// (anytype-ts writes them); no bundle constant exists for either. +const ( + detailKeyAutoWidgetTargets = "autoWidgetTargets" + detailKeyAutoWidgetDisabled = "autoWidgetDisabled" +) + +// widgetObjectResidualKeys are the two timestamps a widget document carries +// about ITSELF: when the hidden object was minted and last touched. A bundle +// is not that object — a restored sidebar is created when it is restored — +// so they are dropped exactly the way the space document's were (§2c). In +// the corpus every one of the 77 createdDate values is 0 anyway. +var widgetObjectResidualKeys = map[string]bool{ + bundle.RelationKeyCreatedDate.String(): true, + bundle.RelationKeyLastModifiedDate.String(): true, +} + +// WidgetObjectResidualKey reports a stored detail the widget-object omission +// drops without an index field to land in: the two object timestamps, and an +// EMPTY name — the widget object is hidden, nothing renders its name, and +// all 15 corpus documents that carry one carry "". A non-empty name is NOT +// residual; the omission predicate keeps the whole document for it. Exported +// for the round-trip comparator, which must apply the very predicate export +// applies (§11). +func WidgetObjectResidualKey(key string, v *types.Value) bool { + if widgetObjectResidualKeys[key] { + return true + } + return key == bundle.RelationKeyName.String() && v.GetStringValue() == "" +} + +// widgetObjectConstantKeys are the details every widget document carries +// with one distinct value across the 77-document corpus, restated verbatim +// by WidgetsSnapshot — so the document says nothing the rebuild does not. +// Value-checked, unlike the space document's key-only list, because the +// rebuild writes these VALUES back: a widget object storing a layout other +// than `dashboard` would be silently re-laid-out by the omission, so it +// keeps its document instead. +func widgetObjectConstantDetail(key string, v *types.Value) bool { + switch key { + case bundle.RelationKeyIsHidden.String(): + return v.GetBoolValue() + case bundle.RelationKeyLayout.String(), bundle.RelationKeyResolvedLayout.String(): + return v.GetNumberValue() == float64(model.ObjectType_dashboard) + } + return false +} + +// liftableWidgetTarget reports whether a stored target-shaped value — a link +// child's target or an autoWidgetTargets entry — is one the index can spell: +// a wire listing name the importer knows (translated into the `_` namespace +// by FormatWidgetTarget), or a CID-shaped object id, which passes verbatim. +// +// The gate is what keeps the omission honest about the corpus's two strays: +// one space stores a widget targeting `bookmark` and one targeting `lists`, +// words no client constant defines and widget.IsPredefinedWidgetTargetId +// does not know. Written into an index they would read as object ids naming +// nothing, and the widget they mean would be dropped on install without an +// error — so a document holding one KEEPS the document, and the stray +// travels the way it always has. +// maxIndexWidgetLimit is the product cap on how many entries a sidebar +// listing widget shows — the ONE number behind three statements: +// `$defs/widgetListingLimit` in the object schema (which the index schema +// references), the authoring index's self-contained copy, and the lift check +// below. TestWidgetLimit_OneStatementOfTheCap chains all of them to this +// constant. The widget BLOCK's own `limit` deliberately takes the whole +// int32 range instead — the block is the fidelity fallback for exactly the +// widget this cap refuses, so the cap must not bind there. +const maxIndexWidgetLimit = 100 + +func liftableWidgetTarget(stored string) bool { + format := FormatWidgetTarget(stored) + if IsReservedWidgetTarget(format) { + return IsImportableWidgetTarget(format) + } + return isObjectIdShaped(stored) +} + +// widgetObjectWidgets reads the block graph into index widgets, reporting +// whether every block was accounted for. ok is false — and the caller must +// keep the document — for any shape beyond the measured one: an unpaired +// wrapper, a wrapper with extra children, a link with children of its own, a +// block attribute the pair cannot carry, an enum value outside the §5 +// vocabulary, a limit outside the index schema's range, a target the index +// cannot spell, or any block that is not the root, the header scaffolding, +// or half of a pair. +func widgetObjectWidgets(base *model.SmartBlockSnapshotBase) (widgets []Widget, ok bool) { + blocks := base.GetBlocks() + if len(blocks) == 0 { + return nil, true + } + byId := make(map[string]*model.Block, len(blocks)) + isChild := map[string]bool{} + for _, b := range blocks { + if b == nil || b.Id == "" || byId[b.Id] != nil { + return nil, false + } + byId[b.Id] = b + for _, c := range b.ChildrenIds { + isChild[c] = true + } + } + var root *model.Block + for _, b := range blocks { + if isChild[b.Id] { + continue + } + if root != nil { + return nil, false // two roots: not the shape this rule measured + } + root = b + } + if root == nil { + return nil, false // a cycle; nothing to walk + } + if _, isRoot := root.Content.(*model.BlockContentOfSmartblock); !isRoot { + return nil, false + } + if !plainBlock(root) { + return nil, false + } + accounted := map[string]bool{root.Id: true} + for _, id := range root.ChildrenIds { + b := byId[id] + if b == nil { + return nil, false + } + switch c := b.Content.(type) { + case *model.BlockContentOfLayout: + // the editor's header scaffolding (§7), present in 11 of 77 + // corpus documents: a Header layout over one EMPTY title block. + // Export drops it from every document, so the omission loses + // nothing by accepting it — and accepts nothing more. + if c.Layout.GetStyle() != model.BlockContentLayout_Header || !plainBlock(b) { + return nil, false + } + accounted[b.Id] = true + for _, cid := range b.ChildrenIds { + t := byId[cid] + // judged the way pageIsEmpty judges the space document's + // scaffolding: on the text alone. The editor stamps a + // `_detailsKey` binding into every title block's fields — + // 11 of 11 in the corpus — and §7 drops the block, binding + // and all, so the binding is not content to fail closed on. + if t == nil || !emptyStructuralText(t) || len(t.ChildrenIds) > 0 { + return nil, false + } + accounted[t.Id] = true + } + case *model.BlockContentOfWidget: + w, link, admitted := widgetPair(b, byId) + if !admitted { + return nil, false + } + accounted[b.Id] = true + accounted[link] = true + widgets = append(widgets, w) + default: + return nil, false + } + } + for id := range byId { + if !accounted[id] { + return nil, false // unreachable from the root: real content, kept + } + } + return widgets, true +} + +// widgetPair reads one wrapper-and-link pair into the flat index widget, or +// refuses. linkId names the accounted link block on success. +func widgetPair(wrapper *model.Block, byId map[string]*model.Block) (w Widget, linkId string, ok bool) { + wc := wrapper.GetWidget() + if !plainBlock(wrapper) || len(wrapper.ChildrenIds) != 1 { + return w, "", false + } + link := byId[wrapper.ChildrenIds[0]] + if link == nil || !plainBlock(link) || len(link.ChildrenIds) != 0 { + return w, "", false + } + lc := link.GetLink() + if lc == nil { + return w, "", false + } + if !liftableWidgetTarget(lc.TargetBlockId) { + return w, "", false + } + w.Target = FormatWidgetTarget(lc.TargetBlockId) + // the wrapper's §5 members, through the same name tables the block + // export uses; an enum value outside the vocabulary has no spelling + if wc.Layout != model.BlockContentWidget_Link { + if w.Layout = widgetLayoutNames.name(wc.Layout); w.Layout == "" { + return w, "", false + } + } + // the index schema bounds a limit ($defs/widgetListingLimit — the + // deliberate product cap on sidebar listings; the corpus maximum is 50) + // where the block schema takes the whole int32 range on purpose: a + // widget this check refuses travels as a full document, and that + // document's widget block must stay valid + if wc.Limit < 0 || wc.Limit > maxIndexWidgetLimit { + return w, "", false + } + w.Limit = wc.Limit + w.ViewId = wc.ViewId + w.AutoAdded = wc.AutoAdded + // the link's §5 display members. Its deprecated `style` and legacy + // `fields` are not checked: export drops both from every link block by + // design (§5), so the document would not have carried them either. + if lc.CardStyle != model.BlockContentLink_Text { + if w.CardStyle = cardStyleNames.name(lc.CardStyle); w.CardStyle == "" { + return w, "", false + } + } + if lc.IconSize != model.BlockContentLink_SizeNone { + if w.IconSize = iconSizeNames.name(lc.IconSize); w.IconSize == "" { + return w, "", false + } + } + if lc.Description != model.BlockContentLink_None { + if w.Description = linkDescriptionNames.name(lc.Description); w.Description == "" { + return w, "", false + } + } + for _, key := range lc.Relations { + // the same writable-key admission every §5 key slot runs (§3): an + // empty or unwritable key cannot be spelled in the index, so the + // document travels whole — where the link block's own slot rule + // drops the entry with a warning + if !isWritablePropertyKey(key) { + return w, "", false + } + } + w.Properties = lc.Relations + return w, link.Id, true +} + +// plainBlock reports a block carrying none of the generic attributes the +// flat widget cannot express — alignment, background, fields. The corpus +// holds zero widget-object blocks with any of them, and a document that +// grows one keeps travelling as a document. +func plainBlock(b *model.Block) bool { + return b.Align == model.Block_AlignLeft && + b.VerticalAlign == model.Block_VerticalAlignTop && + b.BackgroundColor == "" && + (b.Fields == nil || len(b.Fields.Fields) == 0) +} + +// emptyStructuralText reports the one text block the header scaffolding may +// hold: an EMPTY title or description, the same two styles pageIsEmpty +// admits on the space document. +func emptyStructuralText(b *model.Block) bool { + t := b.GetText() + if t == nil || t.GetText() != "" || len(t.GetMarks().GetMarks()) > 0 { + return false + } + return t.GetStyle() == model.BlockContentText_Title || + t.GetStyle() == model.BlockContentText_Description +} + +// liftedAutoWidgetTargets reads the client's auto-widget ledger into index +// spellings, refusing an entry the index cannot spell — the same gate the +// link targets pass. +func liftedAutoWidgetTargets(det map[string]*types.Value) (targets []string, ok bool) { + v := det[detailKeyAutoWidgetTargets] + if v == nil { + return nil, true + } + for _, entry := range valueStringList(v) { + if !liftableWidgetTarget(entry) { + return nil, false + } + targets = append(targets, FormatWidgetTarget(entry)) + } + return targets, true +} + +// IndexFromWidgetObject reads the widget object into the index fields it is +// the source of (§2c): the widgets, the auto-widget ledger, and the +// auto-widget switch. It is the composer's half of the omission — a bundle +// that drops the document MUST write these, or the space loses its sidebar — +// and it fills only what the object states, like IndexFromSpaceSettings. +func IndexFromWidgetObject(idx *Index, base *model.SmartBlockSnapshotBase) { + if idx == nil || base == nil { + return + } + if widgets, ok := widgetObjectWidgets(base); ok && len(widgets) > 0 { + idx.Widgets = widgets + } + det := base.GetDetails().GetFields() + if targets, ok := liftedAutoWidgetTargets(det); ok && len(targets) > 0 { + idx.AutoWidgetTargets = targets + } + if det[detailKeyAutoWidgetDisabled].GetBoolValue() { + idx.AutoWidgetDisabled = true + } +} + +// OmittedWidgetObject reports a widget document a bundle does not write, +// because `index.json` states everything it holds (§2c). +// +// Fail-closed, like the space-document omission beside it: a member this +// package cannot account for — a block shape beyond the wrapper-and-link +// pair, a target the index cannot spell, a non-empty name, an unforeseen +// detail — keeps the document, so a widget object carrying something +// unforeseen travels rather than vanishing. Measured, that keeps 2 of 77 +// corpus documents: the two whose link targets are the stray words +// `bookmark` and `lists` (see liftableWidgetTarget). +func OmittedWidgetObject(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase) bool { + if sbType != model.SmartBlockType_Widget || base == nil { + return false + } + if _, ok := widgetObjectWidgets(base); !ok { + return false + } + det := base.GetDetails().GetFields() + if _, ok := liftedAutoWidgetTargets(det); !ok { + return false + } + stripped := strippedDetailKeys() + for k, v := range det { + switch { + case k == detailKeyAutoWidgetTargets, k == detailKeyAutoWidgetDisabled: + // index.json carries them — IndexFromWidgetObject is the proof + case isTransientProperty(k), stripped[k]: + // already refused or dropped by a rule of its own + case WidgetObjectResidualKey(k, v): + // the object's own timestamps, and an empty name (a NON-empty + // name falls through to the default and keeps the document) + case widgetObjectConstantDetail(k, v): + // one distinct value across all 77 corpus documents, restated + // verbatim by WidgetsSnapshot + default: + return false // unaccounted: keep the document + } + } + return true +} + +// WidgetsSnapshot builds the widget object back from the index — the exact +// snapshot cmd/anyblockconvert puts in an archive, and the reconstruction +// the round-trip verifier holds against the original, one function so the +// two cannot drift. It returns nil when the index carries no sidebar state +// at all: a bundle declaring nothing gets no snapshot rather than an empty +// one. +// +// Four things about the block graph are load-bearing, none of them obvious: +// +// - The root block must carry smartblock content. objectcreator.setRootBlock +// hands the blocks to anymark.AddRootBlock, which renames the first block +// with that content to the derived widgets id. Without one, AddRootBlock +// *appends* a second root instead, the state's root becomes that new block, +// and every wrapper is orphaned. +// - Every wrapper must be reachable from the root. updateWidgetObject walks +// state.Blocks(), which is a breadth-first traversal from the root — a block +// the root does not reach is simply not there. +// - That traversal is also why root.ChildrenIds order is sidebar order: +// addWidgetBlock appends each widget to the existing widget object in +// traversal order (InsertTo with Block_Inner appends, despite the +// prependChildrenIds it calls). So the order here is index.json's order. +// - A wrapper gets exactly one child. addWidgetBlock reads ChildrenIds[0] and +// ignores the rest. +// +// The auto-widget ledger and switch ride along as details. On today's +// experience path they are inert — objectcreator.updateWidgetObject merges +// only the BLOCKS into the space's own widget object — but they are the +// snapshot's truthful state, written so the path that starts reading details +// does not have to change this function. The §2c table records the inertness. +func WidgetsSnapshot(idx *Index) (*model.SmartBlockSnapshotBase, error) { + if idx == nil { + return nil, nil + } + if len(idx.Widgets) == 0 && len(idx.AutoWidgetTargets) == 0 && !idx.AutoWidgetDisabled { + return nil, nil + } + + root := &model.Block{ + Id: WidgetsObjectId, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + } + blocks := make([]*model.Block, 0, 1+2*len(idx.Widgets)) + blocks = append(blocks, root) + + for i, w := range idx.Widgets { + if w.Layout != "" && !widgetLayoutNames.has(w.Layout) { + return nil, fmt.Errorf("widgets[%d]: unknown layout %q", i, w.Layout) + } + if w.CardStyle != "" && !cardStyleNames.has(w.CardStyle) { + return nil, fmt.Errorf("widgets[%d]: unknown card_style %q", i, w.CardStyle) + } + if w.IconSize != "" && !iconSizeNames.has(w.IconSize) { + return nil, fmt.Errorf("widgets[%d]: unknown icon_size %q", i, w.IconSize) + } + if w.Description != "" && !linkDescriptionNames.has(w.Description) { + return nil, fmt.Errorf("widgets[%d]: unknown description %q", i, w.Description) + } + linkId := widgetBlockId(i, w.Target) + wrapperId := linkId + widgetWrapperSuffix + + root.ChildrenIds = append(root.ChildrenIds, wrapperId) + blocks = append(blocks, &model.Block{ + Id: wrapperId, + ChildrenIds: []string{linkId}, + Content: &model.BlockContentOfWidget{Widget: &model.BlockContentWidget{ + Layout: widgetLayoutNames.value(w.Layout), + Limit: w.Limit, + ViewId: w.ViewId, + AutoAdded: w.AutoAdded, + }}, + }, &model.Block{ + Id: linkId, + // the target is the bundle's own object id, relinked on import like + // every other reference (common.UpdateLinksToObjects); a reserved + // listing is translated out of the format's `_` namespace into the + // bare word the importer knows (WireWidgetTarget) and then passes + // through untouched, because handleLinkBlock returns early for + // widget.IsPredefinedWidgetTargetId. Anything else that does not + // resolve is rewritten to _missing_object and then stripped — link + // and wrapper both — by WidgetObject.Init, which is why + // anyblockbatch.CheckIndexTargets rejects such a bundle up front. + Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: WireWidgetTarget(w.Target), + // Style's zero value, spelled out because this is the shape an + // app export writes and the shape a reader will compare against + Style: model.BlockContentLink_Page, + CardStyle: cardStyleNames.value(w.CardStyle), + IconSize: iconSizeNames.value(w.IconSize), + Description: linkDescriptionNames.value(w.Description), + Relations: w.Properties, + }}, + }) + } + + details := map[string]*types.Value{ + detailKeyId: {Kind: &types.Value_StringValue{StringValue: WidgetsObjectId}}, + bundle.RelationKeyLayout.String(): {Kind: &types.Value_NumberValue{ + NumberValue: float64(model.ObjectType_dashboard)}}, + bundle.RelationKeyResolvedLayout.String(): {Kind: &types.Value_NumberValue{ + NumberValue: float64(model.ObjectType_dashboard)}}, + // the widget object is never listed anywhere as an object + bundle.RelationKeyIsHidden.String(): {Kind: &types.Value_BoolValue{BoolValue: true}}, + } + if len(idx.AutoWidgetTargets) > 0 { + entries := make([]*types.Value, 0, len(idx.AutoWidgetTargets)) + for _, t := range idx.AutoWidgetTargets { + entries = append(entries, &types.Value{Kind: &types.Value_StringValue{ + StringValue: WireWidgetTarget(t)}}) + } + details[detailKeyAutoWidgetTargets] = &types.Value{Kind: &types.Value_ListValue{ + ListValue: &types.ListValue{Values: entries}}} + } + if idx.AutoWidgetDisabled { + details[detailKeyAutoWidgetDisabled] = &types.Value{Kind: &types.Value_BoolValue{BoolValue: true}} + } + + return &model.SmartBlockSnapshotBase{ + Blocks: blocks, + Details: &types.Struct{Fields: details}, + ObjectTypes: []string{bundle.TypeKeyDashboard.URL()}, + }, nil +} + +// widgetBlockId mints a link block's id: 24 hex characters, the shape +// bson.NewObjectId().Hex() gives every block id in an app export. +// +// Derived from the widget's position rather than drawn at random, because +// the rebuild is deterministic by design — re-converting an unchanged bundle +// produces identical bytes (see anyblockconvert's batch.optionLocalKey, +// which does the same for options). Seeding on the position rather than the +// target alone is what makes the ids unique even when a bundle lists the +// same target twice. +func widgetBlockId(i int, target string) string { + sum := sha1.Sum([]byte(fmt.Sprintf("widget\x00%d\x00%s", i, target))) + return hex.EncodeToString(sum[:])[:24] +} diff --git a/pkg/lib/anyblockjson/widgetobject_test.go b/pkg/lib/anyblockjson/widgetobject_test.go new file mode 100644 index 0000000000..d6bbd10b16 --- /dev/null +++ b/pkg/lib/anyblockjson/widgetobject_test.go @@ -0,0 +1,292 @@ +package anyblockjson + +// widgetobject_test.go — the sidebar's object, and why a bundle carries +// index.widgets instead of one (§2c). + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// two ids the shape every object id in a real export has (isObjectIdShaped) +const ( + widgetTargetA = "bafyreidft7aqr2fgy6g57hme4rmcdkynf24cd2jfhlyr3duxjevj6vewsu" + widgetTargetB = "bafyreicy4lwi5kigqgfclic3qtqvxeuu5rgtb4edhwqwcswmpdkghutxk4" +) + +func widgetWrapper(id, target string, wc *model.BlockContentWidget, lc *model.BlockContentLink) []*model.Block { + if wc == nil { + wc = &model.BlockContentWidget{} + } + if lc == nil { + lc = &model.BlockContentLink{} + } + lc.TargetBlockId = target + return []*model.Block{ + {Id: id, ChildrenIds: []string{id + "-link"}, + Content: &model.BlockContentOfWidget{Widget: wc}}, + {Id: id + "-link", Content: &model.BlockContentOfLink{Link: lc}}, + } +} + +// widgetSnapshot assembles a widget object the way a real export holds one: +// a smartblock root over wrapper-and-link pairs, with the constant details +// every corpus document carries. +func widgetSnapshot(extraDetails map[string]*types.Value, pairs ...[]*model.Block) *model.SmartBlockSnapshotBase { + root := &model.Block{Id: "widget-root", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + blocks := []*model.Block{root} + for _, pair := range pairs { + root.ChildrenIds = append(root.ChildrenIds, pair[0].Id) + blocks = append(blocks, pair...) + } + det := map[string]*types.Value{ + "id": str("widget-root"), "isHidden": boolean(true), + "layout": num(float64(model.ObjectType_dashboard)), + "resolvedLayout": num(float64(model.ObjectType_dashboard)), + "createdDate": num(0), + } + for k, v := range extraDetails { + det[k] = v + } + return &model.SmartBlockSnapshotBase{Blocks: blocks, Details: fields(det), + ObjectTypes: []string{"ot-dashboard"}} +} + +// Measured over a 77-space export, the widget object reduces to exactly what +// index.json now states: 218 wrapper-and-link pairs in perfect regularity, +// the auto-widget ledger and switch, the constant hidden-dashboard details, +// and the object's own timestamps. The predicate is FAIL-CLOSED: a member +// this package cannot account for keeps the document. +// +// How this can fail: make the default arm return true and an unaccounted +// detail disappears with the document; drop a block-shape check and a +// sidebar richer than the pair is silently flattened into the index. +func TestWidgetObject_OmittedOnlyWhenTheIndexSaysItAll(t *testing.T) { + t.Run("a plain widget object is omitted", func(t *testing.T) { + assert.True(t, OmittedWidgetObject(model.SmartBlockType_Widget, widgetSnapshot(nil, + widgetWrapper("w1", widgetTargetA, + &model.BlockContentWidget{Layout: model.BlockContentWidget_Tree, Limit: 6}, nil), + widgetWrapper("w2", "chat", nil, nil)))) + }) + + t.Run("only the widget kind is eligible", func(t *testing.T) { + assert.False(t, OmittedWidgetObject(model.SmartBlockType_Page, widgetSnapshot(nil))) + }) + + t.Run("the full member set is accounted for", func(t *testing.T) { + assert.True(t, OmittedWidgetObject(model.SmartBlockType_Widget, widgetSnapshot( + map[string]*types.Value{ + "autoWidgetTargets": strList("bin", widgetTargetA), + "autoWidgetDisabled": boolean(true), + "lastModifiedDate": num(1.7e9), + "name": str(""), + }, + widgetWrapper("w1", widgetTargetA, + &model.BlockContentWidget{Layout: model.BlockContentWidget_View, Limit: 6, + ViewId: "view-1", AutoAdded: true}, + &model.BlockContentLink{CardStyle: model.BlockContentLink_Card, + IconSize: model.BlockContentLink_SizeMedium, + Description: model.BlockContentLink_Content, + Relations: []string{"name"}})))) + }) + + t.Run("the editor's header scaffolding is accepted, empty title and binding included", func(t *testing.T) { + // 11 of 77 corpus documents carry it: a Header layout over one EMPTY + // title block whose fields hold the editor's `_detailsKey` binding. + // §7 drops all of it from every document, so nothing is lost. + snap := widgetSnapshot(nil, widgetWrapper("w1", widgetTargetA, nil, nil)) + title := &model.Block{Id: "title", Fields: fields(map[string]*types.Value{ + "_detailsKey": strList("name", "done")}), + Content: &model.BlockContentOfText{Text: &model.BlockContentText{ + Style: model.BlockContentText_Title}}} + header := &model.Block{Id: "header", ChildrenIds: []string{"title"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_Header}}} + snap.Blocks = append(snap.Blocks, header, title) + snap.Blocks[0].ChildrenIds = append([]string{"header"}, snap.Blocks[0].ChildrenIds...) + assert.True(t, OmittedWidgetObject(model.SmartBlockType_Widget, snap)) + + t.Run("but a title with text is content", func(t *testing.T) { + title.GetText().Text = "My sidebar" + assert.False(t, OmittedWidgetObject(model.SmartBlockType_Widget, snap), + "fail closed: a widget object with real text must travel") + }) + }) + + refusals := map[string]*model.SmartBlockSnapshotBase{ + "an unforeseen detail": widgetSnapshot(map[string]*types.Value{ + "somethingNobodyPlannedFor": str("x")}), + "a non-empty name": widgetSnapshot(map[string]*types.Value{ + "name": str("My sidebar")}), + "a layout other than dashboard": widgetSnapshot(map[string]*types.Value{ + "layout": num(float64(model.ObjectType_basic))}), + // the corpus's two strays: bare words no client constant defines. + // Written into an index they would read as object ids naming + // nothing, and the widget would be dropped on install with no error + "a target the index cannot spell": widgetSnapshot(nil, + widgetWrapper("w1", "lists", nil, nil)), + "a dangling target": widgetSnapshot(nil, + widgetWrapper("w1", "_missing_object", nil, nil)), + "a ledger entry the index cannot spell": widgetSnapshot(map[string]*types.Value{ + "autoWidgetTargets": strList("bookmark")}), + "a limit outside the index schema's range": widgetSnapshot(nil, + widgetWrapper("w1", widgetTargetA, &model.BlockContentWidget{Limit: 101}, nil)), + "a wrapper attribute the pair cannot carry": widgetSnapshot(nil, + func() []*model.Block { + pair := widgetWrapper("w1", widgetTargetA, nil, nil) + pair[0].BackgroundColor = "red" + return pair + }()), + } + for name, snap := range refusals { + t.Run(name+" keeps the document", func(t *testing.T) { + assert.False(t, OmittedWidgetObject(model.SmartBlockType_Widget, snap)) + }) + } + + t.Run("a wrapper with two children keeps the document", func(t *testing.T) { + snap := widgetSnapshot(nil, widgetWrapper("w1", widgetTargetA, nil, nil)) + extra := &model.Block{Id: "second-link", + Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{TargetBlockId: widgetTargetB}}} + snap.Blocks = append(snap.Blocks, extra) + snap.Blocks[1].ChildrenIds = append(snap.Blocks[1].ChildrenIds, "second-link") + assert.False(t, OmittedWidgetObject(model.SmartBlockType_Widget, snap), + "addWidgetBlock reads ChildrenIds[0] and ignores the rest, so the rest is content the lift would lose") + }) + + t.Run("an unreachable block keeps the document", func(t *testing.T) { + snap := widgetSnapshot(nil, widgetWrapper("w1", widgetTargetA, nil, nil)) + snap.Blocks = append(snap.Blocks, textBlock("stray", model.BlockContentText_Paragraph, "note")) + assert.False(t, OmittedWidgetObject(model.SmartBlockType_Widget, snap)) + }) + + t.Run("a widget block that is not a root child keeps the document", func(t *testing.T) { + snap := widgetSnapshot(nil) + snap.Blocks[0].ChildrenIds = []string{"p"} + snap.Blocks = append(snap.Blocks, + &model.Block{Id: "p", ChildrenIds: []string{"w1"}, + Content: &model.BlockContentOfText{Text: &model.BlockContentText{}}}) + snap.Blocks = append(snap.Blocks, widgetWrapper("w1", widgetTargetA, nil, nil)...) + assert.False(t, OmittedWidgetObject(model.SmartBlockType_Widget, snap)) + }) +} + +// The lift is one place saying which block member becomes which index field, +// so a composer cannot drop the document while carrying fewer of them than +// the omission assumed were carried. +func TestIndexFromWidgetObject(t *testing.T) { + snap := widgetSnapshot( + map[string]*types.Value{ + "autoWidgetTargets": strList("bin", "favorite", widgetTargetB), + "autoWidgetDisabled": boolean(true), + }, + widgetWrapper("w1", widgetTargetA, + &model.BlockContentWidget{Layout: model.BlockContentWidget_View, Limit: 6, + ViewId: "view-1", AutoAdded: true}, + &model.BlockContentLink{CardStyle: model.BlockContentLink_Card, + IconSize: model.BlockContentLink_SizeMedium, + Description: model.BlockContentLink_Content, + Relations: []string{"name"}}), + widgetWrapper("w2", "chat", &model.BlockContentWidget{Limit: 6}, nil)) + + var idx Index + IndexFromWidgetObject(&idx, snap) + + want := []Widget{ + {Target: widgetTargetA, Layout: "view", Limit: 6, ViewId: "view-1", AutoAdded: true, + CardStyle: "card", IconSize: "medium", Description: "content", Properties: []string{"name"}}, + // the wire word is translated into the `_` namespace; the defaults + // (link layout, text card style, …) stay empty, the §4 omit canon + {Target: "_chat", Limit: 6}, + } + assert.Equal(t, want, idx.Widgets) + assert.Equal(t, []string{"_bin", "_favorite", widgetTargetB}, idx.AutoWidgetTargets, + "ledger entries are targets and get the same translation") + assert.True(t, idx.AutoWidgetDisabled) +} + +// WidgetsSnapshot is the builder both cmd/anyblockconvert and the round-trip +// verifier use; the lift must read its output back into the very index it +// was built from, or the two sides have drifted. +func TestWidgetsSnapshot_TheLiftReadsItBack(t *testing.T) { + idx := &Index{ + Widgets: []Widget{ + {Target: widgetTargetA, Layout: "tree", Limit: 6}, + {Target: "_chat", Limit: 6, AutoAdded: true}, + {Target: "_all_objects", CardStyle: "card", IconSize: "medium", + Description: "content", Properties: []string{"name"}}, + {Target: widgetTargetB, Layout: "view", ViewId: "view-9"}, + }, + AutoWidgetTargets: []string{"_bin", widgetTargetA}, + AutoWidgetDisabled: true, + } + snap, err := WidgetsSnapshot(idx) + require.NoError(t, err) + require.NotNil(t, snap) + + assert.True(t, OmittedWidgetObject(model.SmartBlockType_Widget, snap), + "the omission predicate must admit the very snapshot the rebuild writes") + + var back Index + IndexFromWidgetObject(&back, snap) + assert.Equal(t, idx.Widgets, back.Widgets) + assert.Equal(t, idx.AutoWidgetTargets, back.AutoWidgetTargets) + assert.Equal(t, idx.AutoWidgetDisabled, back.AutoWidgetDisabled) + + t.Run("deterministic across runs", func(t *testing.T) { + again, err := WidgetsSnapshot(idx) + require.NoError(t, err) + assert.Equal(t, snap.Blocks, again.Blocks, + "re-converting an unchanged bundle must produce identical bytes") + }) +} + +// Nothing to show means no snapshot — but the ledger alone is something to +// show: it is the state that stops a restored client re-adding widgets the +// user deleted. +func TestWidgetsSnapshot_Empty(t *testing.T) { + snap, err := WidgetsSnapshot(&Index{Name: "X", Entrypoint: "page-home"}) + require.NoError(t, err) + assert.Nil(t, snap) + + snap, err = WidgetsSnapshot(&Index{AutoWidgetDisabled: true}) + require.NoError(t, err) + require.NotNil(t, snap) + assert.True(t, snap.Details.GetFields()["autoWidgetDisabled"].GetBoolValue()) + require.Len(t, snap.Blocks, 1, "just the root; there are no widgets to hang under it") +} + +func TestWidgetsSnapshot_UnknownVocabulary(t *testing.T) { + for name, w := range map[string]Widget{ + "layout": {Target: "a", Layout: "grid"}, + "card_style": {Target: "a", CardStyle: "poster"}, + "icon_size": {Target: "a", IconSize: "huge"}, + "description": {Target: "a", Description: "excerpt"}, + } { + t.Run(name, func(t *testing.T) { + _, err := WidgetsSnapshot(&Index{Widgets: []Widget{w}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown") + }) + } +} + +// The residual predicate is what the round-trip comparator consults, so its +// scope is pinned: the two object timestamps whatever their value, a name +// only when EMPTY — a non-empty name keeps the whole document instead, and +// the comparator must report it if it ever goes missing anyway. +func TestWidgetObjectResidualKey(t *testing.T) { + assert.True(t, WidgetObjectResidualKey("createdDate", num(0))) + assert.True(t, WidgetObjectResidualKey("lastModifiedDate", num(1.7e9))) + assert.True(t, WidgetObjectResidualKey("name", str(""))) + assert.False(t, WidgetObjectResidualKey("name", str("My sidebar"))) + assert.False(t, WidgetObjectResidualKey("autoWidgetTargets", strList("bin")), + "the ledger is lifted state, not residue: it travels in the index and comes back") + assert.False(t, WidgetObjectResidualKey("isHidden", boolean(true))) +} diff --git a/pkg/lib/anyblockjson/wirenames.go b/pkg/lib/anyblockjson/wirenames.go new file mode 100644 index 0000000000..1a7b3ebeb1 --- /dev/null +++ b/pkg/lib/anyblockjson/wirenames.go @@ -0,0 +1,53 @@ +package anyblockjson + +// wirenames.go — the wire spellings of the members the pre-freeze key/spelling +// split renamed (§2, §2e, §3, §5, §6.2), each defined exactly once. +// +// The word `key` used to mean two different things in one format: the envelope +// member held a STORED internal key (a bson id the app mints, or a bundled +// camelCase key), while a property definition's member held a document-facing +// SPELLING (`due_date`) — one word, two concepts, which is the §15 #14 +// disease. The split gives each concept its own name: +// +// - `internal_key` is the ONLY thing called a key that is a stored id — the +// envelope identity of a definition document, and the optional stored-id +// member of a property definition (export fidelity; an author never needs +// to write one, because the app mints internal keys). +// - `property` is the spelling a property definition states — the same +// document-facing label every other key slot writes. +// - the two legends say what their VALUES are: `property_internal_keys` and +// `type_internal_keys` map a document's spellings to stored internal keys. +// +// Struct tags cannot reference constants, so the decoder tags in import.go, +// typeproperties.go and cmd/internal/anyblockbatch state the same strings; +// the schema files are the third statement. Tests pin all three against each +// other. +const ( + // memberInternalKey is the envelope's stored identity key on definition + // documents (§2) and a property definition's optional stored-id member + // (§2e). Minted by the app, written by export, never required from an + // author. + memberInternalKey = "internal_key" + // memberProperty is THE property-naming slot: the member that names one + // property by its document-facing spelling, wherever a structure names + // exactly one — a property definition (§2e), a dataview's `properties[]` + // entry and the `property` block (both spelled `key` in an earlier revision), and a + // view's column/sort/filter, which spelled `property` from birth. One + // concept, one spelling (§15 #14): measured over 28,599 real exports the + // two spellings sat twelve lines apart inside single dataview blocks, + // each a hard schema error in the other's position, and 2,504 blocks + // wrote the same spelling under both names. + memberProperty = "property" + // memberPropertyInternalKeys is the property legend: document spelling → + // stored internal key (§3). + memberPropertyInternalKeys = "property_internal_keys" + // memberTypeInternalKeys is the same legend on the type namespace (§3). + memberTypeInternalKeys = "type_internal_keys" + // memberPropertySettings is a property document's definition group (§2d) + // — the group that was born `relation_settings`. The `relation`→`property` rename is the + // same disease cured one word later: the product calls these things + // properties, the format called the definition kind `relation`, and one + // document said both (`featured_properties` the block type beside + // `featured_relations` the key). One concept, one spelling (§15 #14). + memberPropertySettings = "property_settings" +) diff --git a/pkg/lib/bundle/apislug.go b/pkg/lib/bundle/apislug.go new file mode 100644 index 0000000000..82b896c28b --- /dev/null +++ b/pkg/lib/bundle/apislug.go @@ -0,0 +1,250 @@ +package bundle + +import ( + "fmt" + "sort" + "strings" + + "github.com/gosimple/unidecode" + "github.com/iancoleman/strcase" + "golang.org/x/text/unicode/norm" + + "github.com/anyproto/anytype-heart/core/domain" +) + +// apislug.go is the derived api-slug table for bundled keys — the authority +// the identifier layer names: bundled +// relations and types are addressed on the API surface by a snake_case slug +// derived from the internal key IN CODE, both directions. The stored +// apiObjectKey detail cannot be the authority for bundled keys (old spaces +// predate it and no reviser backfills it), and a case transform cannot be +// the reverse mechanism (mediaArtistURL → media_artist_url → ToLowerCamel +// yields mediaArtistUrl; _score does not round-trip) — so the reverse is a +// table lookup, never a string transform. +// +// ApiSlug is deliberately the derive half of what objectcreator's +// injectApiObjectKey applies at mint — MintApiSlug is that half plus the +// sanitize one — so a derived slug and a stored apiObjectKey for the same key +// can never disagree: every bundled key is already inside the key grammar, +// where sanitizing changes nothing. + +// ApiSlug derives the snake_case api key ("slug") from an internal key or a +// caller-supplied key. It is the DERIVE half of the mint — v2 creates store +// MintApiSlug's result as apiObjectKey — and the bundled tables below are +// built with it alone, because a bundled key is already inside the grammar. +func ApiSlug(key string) string { + return strcase.ToSnake(key) +} + +// ApiSlugFromName derives a slug from a display name (transliterate, then +// snake) — the transform objectcreator applies when no key is supplied. +func ApiSlugFromName(name string) string { + return strcase.ToSnake(unidecode.Unidecode(strings.TrimSpace(name))) +} + +// SanitizeApiSlug constrains a DERIVED slug (from a display name or a +// document key — inputs no pattern ever checked) to the advertised key +// grammar `^[a-zA-Z0-9_]+$` and a maximum length: every disallowed rune +// becomes `_`, runs collapse, edges trim. Without it, "50% done", "C++" or +// "☕" (unidecode: "?") become identity-bearing apiObjectKey values that no +// key route can accept. An empty result means "no derivable slug" — the +// caller falls back to the minted internal key as the only address. +func SanitizeApiSlug(raw string, maxLen int) string { + var b strings.Builder + lastUnderscore := false + for _, r := range raw { + valid := r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') + if !valid { + r = '_' + } + if r == '_' { + if lastUnderscore { + continue + } + lastUnderscore = true + } else { + lastUnderscore = false + } + b.WriteRune(r) + } + out := strings.Trim(b.String(), "_") + if len(out) > maxLen { + out = strings.Trim(out[:maxLen], "_") + } + return out +} + +// MaxApiSlugLen bounds a minted slug. The api surface declares no length of +// its own, so the bound is the format's: 255 is what an object reference in +// an exported document may hold, and a key longer than that could not be +// written as a reference there at all. +const MaxApiSlugLen = 255 + +// MintApiSlug is the whole mint for a SUPPLIED key — a caller's `key` field, +// or the internal key an object was created with. Derive, then constrain. +// +// The derive half alone leaves in place every character strcase does not +// understand, which is how apiObjectKey values outside the advertised key +// grammar came to be minted: measured over a 38,123-object account, 27 of +// 1,530 stored api keys fall outside `^[a-zA-Z0-9_]+$` — `Lists [in work]` +// stored as `lists_[in_work]`, `Manual export & import` as +// `manual_export_&_import`. A stored key is the spelling callers address the +// object by, and the api promises that spelling is snake_case; minting one +// the grammar does not admit breaks the promise at the only moment it could +// have been kept. +// +// An empty result means the supplied key holds nothing the grammar admits. +// What that means is the caller's to decide: refuse it, or store no slug and +// leave the object addressed by its internal key. +func MintApiSlug(key string) string { + return SanitizeApiSlug(ApiSlug(strings.TrimSpace(key)), MaxApiSlugLen) +} + +// MintApiSlugFromName is MintApiSlug for a display NAME, which is +// transliterated first. Unidecode renders what it cannot romanize as a +// literal `[?]`, so the sanitize half is what stops the name `➡️ Medium` from +// minting the key `[?]_medium`. All but four of the off-grammar keys measured +// above are on options, whose key is derived from the name and nothing else, +// which is why this arm carries most of the damage. +func MintApiSlugFromName(name string) string { + return SanitizeApiSlug(ApiSlugFromName(name), MaxApiSlugLen) +} + +// FoldApiKey is the forgiving-layer fold: lowercase +// with `_` and `-` stripped, so `dueDate`, `due_date` and `due-date` fold +// together. Exact match always wins before folding is consulted; two keys +// folding together is an ambiguity the caller must surface loudly, never +// resolve by guess — which is why the fold lookups below return every match. +func FoldApiKey(s string) string { + // NFC first. A label is MINTED in NFC, but a hand-edited document or an + // editor that decomposes on write can spell the same word in NFD — and a + // decomposed `tiếng_việt` is a different byte sequence from the composed + // one, so every exact-match step of the chain misses it and the value + // lands under a key no relation owns. Folding is the layer whose whole job + // is forgiving a near-miss a reader cannot see; a normalization difference + // is the least visible near-miss there is, so it belongs here rather than + // nowhere. Exact matching upstream is untouched. + return strings.Map(func(r rune) rune { + switch r { + case '_', '-': + return -1 + } + return r + }, strings.ToLower(norm.NFC.String(s))) +} + +var ( + relationKeyByApiSlug map[string]domain.RelationKey + typeKeyByApiSlug map[string]domain.TypeKey + relationKeysByFold map[string][]domain.RelationKey + typeKeysByFold map[string][]domain.TypeKey +) + +// init builds the two reverse tables, SORTED and with an injectivity guard. +// +// The guard is the point. `key -> slug` is a lossy transform, so two bundled +// keys can in principle land on one slug (or one fold), and the reverse table +// is a plain map: the winner would be whichever key Go's map iteration +// reached last — a different address per process, with no signal anywhere. +// The bundled table is the ONE authority for bundled api keys in every space +// and offline — it ships in code, with no store behind it; an authority that +// disagrees with itself between restarts is worse than no authority. Today the table is injective on both counts +// (194 relations → 194 slugs → 194 folds; 29 types → 29 → 29), so this can +// only fire on a bundled key ADDED later, at the moment it is added, in every +// test binary — which is exactly when it is cheap to rename. +// +// The fold arm panics too, even though relationKeysByFold is a slice and +// could hold both: two bundled keys sharing a fold would make that whole fold +// class permanently ambiguous for every caller of the forgiving layer, which +// is a defect to fix in the table, not to serve. +func init() { + relationKeys := sortedApiSlugKeys(len(relations), func(yield func(string)) { + for key := range relations { + yield(key.String()) + } + }) + if err := checkApiSlugInjectivity("relation", relationKeys); err != nil { + panic(err) + } + relationKeyByApiSlug = make(map[string]domain.RelationKey, len(relationKeys)) + relationKeysByFold = make(map[string][]domain.RelationKey, len(relationKeys)) + for _, raw := range relationKeys { + slug := ApiSlug(raw) + relationKeyByApiSlug[slug] = domain.RelationKey(raw) + fold := FoldApiKey(slug) + relationKeysByFold[fold] = append(relationKeysByFold[fold], domain.RelationKey(raw)) + } + + typeKeys := sortedApiSlugKeys(len(types), func(yield func(string)) { + for key := range types { + yield(key.String()) + } + }) + if err := checkApiSlugInjectivity("type", typeKeys); err != nil { + panic(err) + } + typeKeyByApiSlug = make(map[string]domain.TypeKey, len(typeKeys)) + typeKeysByFold = make(map[string][]domain.TypeKey, len(typeKeys)) + for _, raw := range typeKeys { + slug := ApiSlug(raw) + typeKeyByApiSlug[slug] = domain.TypeKey(raw) + fold := FoldApiKey(slug) + typeKeysByFold[fold] = append(typeKeysByFold[fold], domain.TypeKey(raw)) + } +} + +func sortedApiSlugKeys(size int, each func(yield func(string))) []string { + out := make([]string, 0, size) + each(func(key string) { out = append(out, key) }) + sort.Strings(out) + return out +} + +// checkApiSlugInjectivity is the guard init panics on. Two keys sharing a +// slug make the reverse table a coin flip; two keys sharing a fold make that +// whole fold class permanently ambiguous for the forgiving layer. Both are +// defects in the TABLE, to be fixed by renaming a key, never served. +func checkApiSlugInjectivity(kind string, keys []string) error { + bySlug := make(map[string]string, len(keys)) + byFold := make(map[string]string, len(keys)) + for _, key := range keys { + slug := ApiSlug(key) + if first, taken := bySlug[slug]; taken { + return fmt.Errorf("bundled %s keys %q and %q both derive the api slug %q — the reverse table would resolve it to whichever key the map reached last; rename one", kind, first, key, slug) + } + bySlug[slug] = key + fold := FoldApiKey(slug) + if first, taken := byFold[fold]; taken { + return fmt.Errorf("bundled %s keys %q and %q fold together (%q) — the forgiving layer would be permanently ambiguous for that spelling; rename one", kind, first, key, fold) + } + byFold[fold] = key + } + return nil +} + +// RelationKeyByApiSlug resolves a bundled relation's derived slug back to its +// internal key (`due_date` → `dueDate`). Bundled keys only; stored keys and +// stored slugs are the space's to resolve. +func RelationKeyByApiSlug(slug string) (domain.RelationKey, bool) { + key, ok := relationKeyByApiSlug[slug] + return key, ok +} + +// TypeKeyByApiSlug resolves a bundled type's derived slug back to its +// internal key (`object_type` → `objectType`). +func TypeKeyByApiSlug(slug string) (domain.TypeKey, bool) { + key, ok := typeKeyByApiSlug[slug] + return key, ok +} + +// RelationKeysByApiFold returns every bundled relation key whose derived slug +// folds to the input's fold — the forgiving layer's candidate set. Zero +// matches: not bundled; one: the match; two or more: ambiguous, fail loud. +func RelationKeysByApiFold(input string) []domain.RelationKey { + return relationKeysByFold[FoldApiKey(input)] +} + +// TypeKeysByApiFold is RelationKeysByApiFold for bundled type keys. +func TypeKeysByApiFold(input string) []domain.TypeKey { + return typeKeysByFold[FoldApiKey(input)] +} diff --git a/pkg/lib/bundle/apislug_test.go b/pkg/lib/bundle/apislug_test.go new file mode 100644 index 0000000000..f4e60a14f9 --- /dev/null +++ b/pkg/lib/bundle/apislug_test.go @@ -0,0 +1,195 @@ +package bundle + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" +) + +// The derived table is the API surface's one authority for bundled keys; its +// whole claim to authority is collision-freedom. These tests are the loud +// failure a future bundle addition hits if it ever mints a colliding slug. + +func TestApiSlugTableIsCollisionFree(t *testing.T) { + t.Run("every bundled relation key has a distinct slug", func(t *testing.T) { + require.NotEmpty(t, relations) + assert.Len(t, relationKeyByApiSlug, len(relations), + "two bundled relation keys derive the same slug — the table lost an entry") + }) + + t.Run("every bundled type key has a distinct slug", func(t *testing.T) { + require.NotEmpty(t, types) + assert.Len(t, typeKeyByApiSlug, len(types), + "two bundled type keys derive the same slug — the table lost an entry") + }) + + t.Run("the fold layer is clean over the bundle", func(t *testing.T) { + // two bundled keys folding together would make the forgiving layer + // permanently ambiguous for both — fail here, at the bundle change + for fold, keys := range relationKeysByFold { + assert.Len(t, keys, 1, "relation fold %q is ambiguous: %v", fold, keys) + } + for fold, keys := range typeKeysByFold { + assert.Len(t, keys, 1, "type fold %q is ambiguous: %v", fold, keys) + } + }) +} + +func TestApiSlugRoundTrip(t *testing.T) { + t.Run("every relation slug resolves back to its key", func(t *testing.T) { + for key := range relations { + got, ok := RelationKeyByApiSlug(ApiSlug(key.String())) + require.True(t, ok, "slug of %q does not resolve", key) + assert.Equal(t, key, got) + } + }) + + t.Run("every type slug resolves back to its key", func(t *testing.T) { + for key := range types { + got, ok := TypeKeyByApiSlug(ApiSlug(key.String())) + require.True(t, ok, "slug of %q does not resolve", key) + assert.Equal(t, key, got) + } + }) + + t.Run("string inversion is not the reverse mechanism", func(t *testing.T) { + // the documented non-invertible cases: the + // table must carry them because no case transform can + key, ok := RelationKeyByApiSlug("media_artist_url") + require.True(t, ok) + assert.Equal(t, domain.RelationKey("mediaArtistURL"), key) + }) +} + +func TestApiSlugSpellings(t *testing.T) { + // the spellings the API surface promises: a drift in the + // snake transform respells the API — pin the load-bearing examples + assert.Equal(t, "due_date", ApiSlug("dueDate")) + assert.Equal(t, "icon_emoji", ApiSlug("iconEmoji")) + assert.Equal(t, "object_type", ApiSlug("objectType")) + assert.Equal(t, "name", ApiSlug("name")) +} + +func TestFoldApiKey(t *testing.T) { + assert.Equal(t, "duedate", FoldApiKey("due_date")) + assert.Equal(t, "duedate", FoldApiKey("dueDate")) + assert.Equal(t, "duedate", FoldApiKey("due-date")) + assert.NotEqual(t, FoldApiKey("dueDate"), FoldApiKey("dueDates")) +} + +func TestApiSlugFromName(t *testing.T) { + assert.Equal(t, "manual_property", ApiSlugFromName("Manual property")) + assert.Equal(t, "uber", ApiSlugFromName(" Über ")) +} + +// The mint pair is what the app STORES as apiObjectKey. Measured over a +// 38,123-object account, 27 of 1,530 stored api keys sat outside the key +// grammar the api advertises; every fixture below is one of those shapes, +// taken from that account. +func TestMintApiSlugFromName(t *testing.T) { + t.Run("punctuation a display name carries never reaches the key", func(t *testing.T) { + assert.Equal(t, "lists_[in_work]", ApiSlugFromName("Lists [in work]"), + "the derive half leaves brackets in place — this is the stored key today") + assert.Equal(t, "lists_in_work", MintApiSlugFromName("Lists [in work]")) + assert.Equal(t, "manual_export_import", MintApiSlugFromName("Manual export & import")) + assert.Equal(t, "50_done", MintApiSlugFromName("50% done")) + }) + + t.Run("an emoji arrives as unidecode's literal [?] and is dropped", func(t *testing.T) { + assert.Equal(t, "[?]_medium", ApiSlugFromName("➡️ Medium"), + "unidecode romanizes an unmappable rune to the three bytes `[?]`") + assert.Equal(t, "medium", MintApiSlugFromName("➡️ Medium")) + }) + + t.Run("a name that romanizes keeps its word", func(t *testing.T) { + assert.Equal(t, "zadacha", MintApiSlugFromName("Задача")) + }) + + t.Run("a name with nothing to romanize mints no slug at all", func(t *testing.T) { + // the caller's cue to leave the object addressed by its internal key + assert.Equal(t, "", MintApiSlugFromName("➡️")) + assert.Equal(t, "", MintApiSlugFromName("☕")) + assert.Equal(t, "", MintApiSlugFromName(" ")) + }) + + t.Run("length is bounded", func(t *testing.T) { + got := MintApiSlugFromName(strings.Repeat("a", 300)) + assert.Len(t, got, MaxApiSlugLen) + assert.Equal(t, strings.Repeat("a", MaxApiSlugLen), got) + }) +} + +func TestMintApiSlug(t *testing.T) { + t.Run("a supplied key keeps its spelling where the grammar allows", func(t *testing.T) { + assert.Equal(t, "due_date", MintApiSlug("dueDate")) + assert.Equal(t, "due_date", MintApiSlug("due date")) + assert.Equal(t, "already_snake", MintApiSlug("already_snake")) + assert.Equal(t, "web_3", MintApiSlug("Web3"), + "the snake transform splits a digit run; that is the rule both surfaces already use") + }) + + t.Run("and loses what the grammar does not admit", func(t *testing.T) { + assert.Equal(t, "my_key", MintApiSlug("my key!")) + assert.Equal(t, "lists_in_work", MintApiSlug("Lists [in work]")) + }) + + t.Run("no transliteration on a supplied key", func(t *testing.T) { + // the name arm romanizes because nobody is there to ask; a key was + // CHOSEN, and answering `Задача` with `zadacha` would name the object + // something its author never wrote. Empty says so instead. + assert.Equal(t, "", MintApiSlug("Задача")) + assert.Equal(t, "", MintApiSlug("!!!")) + assert.Equal(t, "", MintApiSlug("___")) + }) + + t.Run("length is bounded", func(t *testing.T) { + assert.Len(t, MintApiSlug(strings.Repeat("k", 300)), MaxApiSlugLen) + }) +} + +// TestApiSlugTablesAreInjective is the guard init panics on, run over the +// REAL tables. `key -> slug` is lossy, the reverse tables are plain maps, and +// nothing anywhere checked: a bundled key added tomorrow that snakes onto an +// existing slug would make `RelationKeyByApiSlug` a per-process coin flip — +// a different address on a different restart, with no signal at all. The +// tables are clean today, so this can only ever fail on the commit that +// breaks them. +func TestApiSlugTablesAreInjective(t *testing.T) { + relationKeys := sortedApiSlugKeys(len(relations), func(yield func(string)) { + for key := range relations { + yield(key.String()) + } + }) + typeKeys := sortedApiSlugKeys(len(types), func(yield func(string)) { + for key := range types { + yield(key.String()) + } + }) + + assert.NoError(t, checkApiSlugInjectivity("relation", relationKeys)) + assert.NoError(t, checkApiSlugInjectivity("type", typeKeys)) + assert.Equal(t, len(relationKeys), len(relationKeyByApiSlug), "one slug per bundled relation") + assert.Equal(t, len(typeKeys), len(typeKeyByApiSlug), "one slug per bundled type") +} + +func TestApiSlugInjectivityGuardFires(t *testing.T) { + t.Run("two keys deriving one slug", func(t *testing.T) { + err := checkApiSlugInjectivity("relation", []string{"dueDate", "due_date"}) + require.Error(t, err) + assert.Contains(t, err.Error(), `both derive the api slug "due_date"`) + }) + + t.Run("two slugs folding together", func(t *testing.T) { + err := checkApiSlugInjectivity("type", []string{"moodlevel", "moodLevel"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "fold together") + }) + + t.Run("a clean table passes", func(t *testing.T) { + assert.NoError(t, checkApiSlugInjectivity("relation", []string{"dueDate", "name", "_score"})) + }) +} diff --git a/pkg/lib/bundle/init.go b/pkg/lib/bundle/init.go index ab6750a75e..8e8f47a94f 100644 --- a/pkg/lib/bundle/init.go +++ b/pkg/lib/bundle/init.go @@ -250,3 +250,15 @@ func RelationKeyFromID(id string) (domain.RelationKey, error) { return "", fmt.Errorf("invalid type url: no prefix found") } + +// ListRelationsKeys returns every bundled relation key, in map order — the +// relation half of ListTypesKeys, for callers that build their own tables +// over the whole bundled population (sort before deriving anything +// order-sensitive). +func ListRelationsKeys() []domain.RelationKey { + keys := make([]domain.RelationKey, 0, len(relations)) + for k := range relations { + keys = append(keys, k) + } + return keys +} diff --git a/pkg/lib/bundle/relation.gen.go b/pkg/lib/bundle/relation.gen.go index 149b90278f..1337daf037 100644 --- a/pkg/lib/bundle/relation.gen.go +++ b/pkg/lib/bundle/relation.gen.go @@ -9,7 +9,7 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) -const RelationChecksum = "cb8c8504438f2846044f86bbc2274d236d682c9f088ac3514d5268dbad219f3b" +const RelationChecksum = "2ab1c36becd82f326e627ef3616b7eddab7008e1fd1e3afa95e11429f440d3a7" const ( RelationKeyTag domain.RelationKey = "tag" RelationKeyCamera domain.RelationKey = "camera" @@ -340,9 +340,10 @@ var ( Id: "_braudioGenre", Key: "audioGenre", MaxCount: 1, - Name: "Genre", + Name: "Audio genre", ReadOnly: false, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyAudioLyrics: { @@ -723,10 +724,11 @@ var ( Hidden: true, Id: "_brfeaturedRelations", Key: "featuredRelations", - Name: "Featured Relations", + Name: "Featured properties", ObjectTypes: []string{TypePrefix + "relation"}, ReadOnly: false, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyFileAvailableOffline: { @@ -1007,9 +1009,10 @@ var ( Id: "_brheaderRelationsLayout", Key: "headerRelationsLayout", MaxCount: 1, - Name: "Header relations layout", + Name: "Header properties layout", ReadOnly: false, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyHeightInPixels: { @@ -1826,10 +1829,11 @@ var ( Hidden: true, Id: "_brrecommendedFeaturedRelations", Key: "recommendedFeaturedRelations", - Name: "Recommended featured relations", + Name: "Recommended featured properties", ObjectTypes: []string{TypePrefix + "relation"}, ReadOnly: false, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyRecommendedFileRelations: { @@ -1840,10 +1844,11 @@ var ( Hidden: true, Id: "_brrecommendedFileRelations", Key: "recommendedFileRelations", - Name: "Recommended file relations", + Name: "Recommended file properties", ObjectTypes: []string{TypePrefix + "relation"}, ReadOnly: false, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyRecommendedHiddenRelations: { @@ -1854,10 +1859,11 @@ var ( Hidden: true, Id: "_brrecommendedHiddenRelations", Key: "recommendedHiddenRelations", - Name: "Recommended hidden relations", + Name: "Recommended hidden properties", ObjectTypes: []string{TypePrefix + "relation"}, ReadOnly: false, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyRecommendedLayout: { @@ -1882,10 +1888,11 @@ var ( Hidden: true, Id: "_brrecommendedRelations", Key: "recommendedRelations", - Name: "Recommended relations", + Name: "Recommended properties", ObjectTypes: []string{TypePrefix + "relation"}, ReadOnly: false, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyRelationDefaultValue: { @@ -1938,9 +1945,10 @@ var ( Hidden: true, Id: "_brrelationFormatObjectTypes", Key: "relationFormatObjectTypes", - Name: "Relation's target object types", + Name: "Property's target object types", ReadOnly: true, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyRelationKey: { @@ -1952,9 +1960,10 @@ var ( Id: "_brrelationKey", Key: "relationKey", MaxCount: 1, - Name: "Relation key", + Name: "Property key", ReadOnly: true, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyRelationMaxCount: { @@ -1981,9 +1990,10 @@ var ( Id: "_brrelationOptionColor", Key: "relationOptionColor", MaxCount: 1, - Name: "Relation option color", + Name: "Property option color", ReadOnly: true, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyRelationReadonlyValue: { @@ -1995,9 +2005,10 @@ var ( Id: "_brrelationReadonlyValue", Key: "relationReadonlyValue", MaxCount: 1, - Name: "Relation value is readonly", + Name: "Property value is readonly", ReadOnly: true, ReadOnlyRelation: true, + Revision: 1, Scope: model.Relation_type, }, RelationKeyReleasedYear: { diff --git a/pkg/lib/bundle/relations.json b/pkg/lib/bundle/relations.json index 2eea9e5632..89911607f0 100644 --- a/pkg/lib/bundle/relations.json +++ b/pkg/lib/bundle/relations.json @@ -56,9 +56,10 @@ "hidden": true, "key": "relationFormatObjectTypes", "maxCount": 0, - "name": "Relation's target object types", + "name": "Property's target object types", "readonly": true, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "Relation key", @@ -66,9 +67,10 @@ "hidden": true, "key": "relationKey", "maxCount": 1, - "name": "Relation key", + "name": "Property key", "readonly": true, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "Relation option color", @@ -76,9 +78,10 @@ "hidden": true, "key": "relationOptionColor", "maxCount": 1, - "name": "Relation option color", + "name": "Property option color", "readonly": true, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "Latest Acl head id", @@ -210,9 +213,10 @@ "hidden": true, "key": "relationReadonlyValue", "maxCount": 1, - "name": "Relation value is readonly", + "name": "Property value is readonly", "readonly": true, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "Image icon", @@ -392,12 +396,13 @@ "hidden": true, "key": "recommendedRelations", "maxCount": 0, - "name": "Recommended relations", + "name": "Recommended properties", "objectTypes": [ "relation" ], "readonly": false, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "Human which created this object", @@ -618,9 +623,10 @@ "hidden": false, "key": "audioGenre", "maxCount": 1, - "name": "Genre", + "name": "Audio genre", "readonly": false, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "Name of the object", @@ -898,12 +904,13 @@ "hidden": true, "key": "featuredRelations", "maxCount": 0, - "name": "Featured Relations", + "name": "Featured properties", "objectTypes": [ "relation" ], "readonly": false, - "source": "details" + "source": "details", + "revision": 1 }, { "format": "phone", @@ -1645,12 +1652,13 @@ "hidden": true, "key": "recommendedFeaturedRelations", "maxCount": 0, - "name": "Recommended featured relations", + "name": "Recommended featured properties", "objectTypes": [ "relation" ], "readonly": false, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "List of recommended relations that are hidden in layout", @@ -1658,12 +1666,13 @@ "hidden": true, "key": "recommendedHiddenRelations", "maxCount": 0, - "name": "Recommended hidden relations", + "name": "Recommended hidden properties", "objectTypes": [ "relation" ], "readonly": false, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "List of recommended file-specific relations", @@ -1671,12 +1680,13 @@ "hidden": true, "key": "recommendedFileRelations", "maxCount": 0, - "name": "Recommended file relations", + "name": "Recommended file properties", "objectTypes": [ "relation" ], "readonly": false, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "Default view type that will be used for new sets/collections", @@ -1714,9 +1724,10 @@ "hidden": true, "key": "headerRelationsLayout", "maxCount": 1, - "name": "Header relations layout", + "name": "Header properties layout", "readonly": false, - "source": "details" + "source": "details", + "revision": 1 }, { "description": "Identifier to use in intergrations with Anytype API", diff --git a/pkg/lib/bundle/types.gen.go b/pkg/lib/bundle/types.gen.go index 8df2ac5660..2a476b839b 100644 --- a/pkg/lib/bundle/types.gen.go +++ b/pkg/lib/bundle/types.gen.go @@ -9,7 +9,7 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) -const TypeChecksum = "e8960524577bd1c93cc4c622b7fd2287eef37db55501400e0c3e00cae7041cf0" +const TypeChecksum = "71f6229d94c4bdf847dcab52efd4f85affaff2648d2a9b671cab46141b5c2902" const ( TypePrefix = "_ot" ) @@ -368,8 +368,9 @@ var ( Hidden: true, IconColor: 0, Layout: model.ObjectType_relationOption, - Name: "Relation option", + Name: "Property option", Readonly: true, + Revision: 1, Types: []model.SmartBlockType{model.SmartBlockType_SubObject}, Url: TypePrefix + "relationOption", }, @@ -394,12 +395,12 @@ var ( IconColor: 10, IconName: "folder", Layout: model.ObjectType_space, - Name: "Space", + Name: "Space settings", PluralName: "Spaces", Readonly: true, RelationLinks: []*model.RelationLink{MustGetRelationLink(RelationKeyTag)}, RestrictObjectCreation: true, - Revision: 3, + Revision: 4, Types: []model.SmartBlockType{model.SmartBlockType_Workspace}, Url: TypePrefix + "space", }, diff --git a/pkg/lib/bundle/types.json b/pkg/lib/bundle/types.json index 3490d95d9a..f9c7e56df5 100644 --- a/pkg/lib/bundle/types.json +++ b/pkg/lib/bundle/types.json @@ -224,17 +224,18 @@ }, { "id": "relationOption", - "name": "Relation option", + "name": "Property option", "types": [ "SubObject" ], "hidden": true, "layout": "relationOption", - "relations": [] + "relations": [], + "revision": 1 }, { "id": "space", - "name": "Space", + "name": "Space settings", "pluralName": "Spaces", "types": [ "Workspace" @@ -247,7 +248,7 @@ "tag" ], "restrictObjectCreation": true, - "revision": 3 + "revision": 4 }, { "id": "spaceView", diff --git a/pkg/lib/pb/model/models.pb.go b/pkg/lib/pb/model/models.pb.go index 2093606a56..95adf537f3 100644 --- a/pkg/lib/pb/model/models.pb.go +++ b/pkg/lib/pb/model/models.pb.go @@ -2356,6 +2356,11 @@ const ( Export_DOT ExportFormat = 3 Export_SVG ExportFormat = 4 Export_GRAPH_JSON ExportFormat = 5 + // AnyBlockJSON is the native AnyBlock JSON bundle (pkg/lib/anyblockjson + // SPEC.md): a directory of `.anyblock.json` documents beside an + // index.json and properties.json. Additive — existing values keep + // their numbers, so a client that does not know it is unaffected. + Export_AnyBlockJSON ExportFormat = 6 ) var ExportFormat_name = map[int32]string{ @@ -2365,15 +2370,17 @@ var ExportFormat_name = map[int32]string{ 3: "DOT", 4: "SVG", 5: "GRAPH_JSON", + 6: "AnyBlockJSON", } var ExportFormat_value = map[string]int32{ - "Markdown": 0, - "Protobuf": 1, - "JSON": 2, - "DOT": 3, - "SVG": 4, - "GRAPH_JSON": 5, + "Markdown": 0, + "Protobuf": 1, + "JSON": 2, + "DOT": 3, + "SVG": 4, + "GRAPH_JSON": 5, + "AnyBlockJSON": 6, } func (x ExportFormat) String() string { @@ -12204,7 +12211,7 @@ func init() { } var fileDescriptor_98a910b73321e591 = []byte{ - // 11242 bytes of a gzipped FileDescriptorProto + // 11250 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x7d, 0x6d, 0x6c, 0x23, 0x59, 0x72, 0x98, 0xf8, 0x4d, 0x16, 0x45, 0xe9, 0xe9, 0xcd, 0x17, 0x97, 0x3b, 0x9e, 0x8c, 0x79, 0x7b, 0xbb, 0x73, 0xba, 0x3d, 0xcd, 0xee, 0xcc, 0xee, 0xed, 0xdc, 0xfa, 0xf6, 0x83, 0x92, 0xa8, 0x11, @@ -12648,266 +12655,267 @@ var fileDescriptor_98a910b73321e591 = []byte{ 0xc9, 0xf4, 0x17, 0x53, 0x70, 0xfb, 0x32, 0x7d, 0xff, 0xff, 0x5d, 0xae, 0x26, 0x6b, 0x58, 0x7f, 0x27, 0x4c, 0x02, 0x29, 0x43, 0x41, 0x5d, 0xd5, 0xa7, 0x52, 0xf9, 0x07, 0xce, 0x73, 0x5b, 0x46, 0xc4, 0x35, 0xa1, 0xab, 0x9b, 0x2f, 0x34, 0x31, 0xb2, 0x4c, 0xda, 0xab, 0xbd, 0x05, 0xd0, 0x20, - 0xbf, 0x2e, 0x38, 0x8e, 0xb5, 0xb1, 0xd3, 0xee, 0x34, 0xd9, 0x42, 0xdc, 0x88, 0xfd, 0x34, 0x50, - 0xc4, 0xf5, 0x7d, 0xc8, 0x47, 0x47, 0x5f, 0x76, 0x75, 0xf7, 0xc4, 0x90, 0x3b, 0xa2, 0x8b, 0x50, - 0xdc, 0x57, 0x2e, 0x94, 0x7c, 0xd5, 0x47, 0x9d, 0xf6, 0x9e, 0x0c, 0xbe, 0x6f, 0xb6, 0xbb, 0xf2, - 0x00, 0x4d, 0xe7, 0xe0, 0xb1, 0xdc, 0x9a, 0x7b, 0xac, 0x35, 0xf6, 0xb7, 0x0f, 0x09, 0x23, 0x57, - 0xff, 0x5b, 0xd9, 0x60, 0x55, 0xab, 0xff, 0x88, 0xda, 0x6b, 0x05, 0xc8, 0xa3, 0x36, 0x77, 0x14, - 0xe3, 0xf0, 0x35, 0x94, 0xf4, 0xdd, 0x3c, 0x93, 0x71, 0x08, 0x96, 0xe6, 0x79, 0x48, 0xef, 0x1f, - 0xc9, 0x2c, 0xb2, 0x6d, 0x7f, 0x68, 0xc9, 0x03, 0xc6, 0xdd, 0x33, 0x9f, 0xe5, 0xf0, 0xcf, 0x86, - 0x77, 0x2a, 0xf7, 0xf9, 0xda, 0x47, 0x9e, 0x49, 0x67, 0x5a, 0x0a, 0xf5, 0x7f, 0x94, 0x81, 0x52, - 0xa8, 0x38, 0xaf, 0xa2, 0xc8, 0x39, 0x87, 0xa5, 0xd6, 0x5e, 0xb7, 0xa9, 0xed, 0x35, 0x76, 0x14, - 0x4a, 0x86, 0x5f, 0x83, 0xe5, 0xad, 0xd6, 0x4e, 0xf3, 0x70, 0xa7, 0xdd, 0xd8, 0x54, 0xc0, 0x22, - 0xbf, 0x09, 0xbc, 0xb5, 0xbb, 0xdf, 0xd6, 0xba, 0x87, 0xad, 0xce, 0xe1, 0x46, 0x63, 0x6f, 0xa3, - 0xb9, 0xd3, 0xdc, 0x64, 0x79, 0xfe, 0x0a, 0xdc, 0xdd, 0x6b, 0x77, 0x5b, 0xed, 0xbd, 0xc3, 0xbd, - 0xf6, 0x61, 0x7b, 0xfd, 0xa3, 0xe6, 0x46, 0xb7, 0x73, 0xd8, 0xda, 0x3b, 0x44, 0xae, 0x8f, 0xb5, - 0x06, 0x3e, 0x61, 0x39, 0x7e, 0x17, 0x6e, 0x2b, 0xac, 0x4e, 0x53, 0x3b, 0x68, 0x6a, 0xc8, 0xe4, - 0xe9, 0x5e, 0xe3, 0xa0, 0xd1, 0xda, 0x69, 0xac, 0xef, 0x34, 0xd9, 0x22, 0xbf, 0x03, 0x35, 0x85, - 0xa1, 0x35, 0xba, 0xcd, 0xc3, 0x9d, 0xd6, 0x6e, 0xab, 0x7b, 0xd8, 0xfc, 0xc6, 0x46, 0xb3, 0xb9, - 0xd9, 0xdc, 0x64, 0x15, 0xfe, 0x25, 0xf8, 0x22, 0x55, 0x4a, 0x55, 0x22, 0xf9, 0xb2, 0x4f, 0x5b, - 0xfb, 0x87, 0x0d, 0x6d, 0x63, 0xbb, 0x75, 0xd0, 0x64, 0x4b, 0xfc, 0x35, 0xf8, 0xc2, 0xc5, 0xa8, - 0x9b, 0x2d, 0xad, 0xb9, 0xd1, 0x6d, 0x6b, 0x9f, 0xb0, 0x15, 0xfe, 0x7d, 0xf0, 0xd2, 0x76, 0x77, - 0x77, 0xe7, 0xf0, 0x99, 0xd6, 0xde, 0x7b, 0x7c, 0x48, 0x7f, 0x3b, 0x5d, 0xed, 0xe9, 0x46, 0xf7, - 0xa9, 0xd6, 0x64, 0xc0, 0x6b, 0x70, 0x73, 0x7f, 0xfd, 0x70, 0xaf, 0xdd, 0x3d, 0x6c, 0xec, 0x7d, - 0xb2, 0xbe, 0xd3, 0xde, 0x78, 0x72, 0xb8, 0xd5, 0xd6, 0x76, 0x1b, 0x5d, 0x56, 0xe6, 0x5f, 0x86, - 0xd7, 0x36, 0x3a, 0x07, 0xaa, 0x9a, 0xed, 0xad, 0x43, 0xad, 0xfd, 0xac, 0x73, 0xd8, 0xd6, 0x0e, - 0xb5, 0xe6, 0x0e, 0xb5, 0xb9, 0x13, 0xd5, 0xbd, 0xc0, 0x6f, 0x43, 0xb5, 0xb5, 0xd7, 0x79, 0xba, - 0xb5, 0xd5, 0xda, 0x68, 0x35, 0xf7, 0xba, 0x87, 0xfb, 0x4d, 0x6d, 0xb7, 0xd5, 0xe9, 0x20, 0x1a, - 0x2b, 0xd5, 0x3f, 0x84, 0x7c, 0xcb, 0x3e, 0x35, 0x7d, 0x9a, 0x6d, 0x4a, 0x34, 0x95, 0xff, 0x15, - 0x14, 0x69, 0x92, 0x98, 0x7d, 0x9b, 0xee, 0xdf, 0xa0, 0xb9, 0xb6, 0xa8, 0x45, 0x80, 0xfa, 0xff, - 0xca, 0x42, 0x45, 0xb2, 0x08, 0xfc, 0xb9, 0x7b, 0xb0, 0xac, 0x82, 0xaa, 0xad, 0xa4, 0x42, 0x9b, - 0x04, 0xd3, 0xc5, 0x76, 0x12, 0x14, 0x53, 0x6b, 0x71, 0x10, 0x7f, 0x15, 0x96, 0x02, 0xa2, 0x9e, - 0x63, 0x6f, 0x98, 0x86, 0xda, 0x60, 0x9b, 0x80, 0xf2, 0x1f, 0x86, 0x97, 0x62, 0x90, 0xa6, 0xdd, - 0x73, 0xcf, 0x47, 0xe1, 0x15, 0xa0, 0x95, 0x99, 0x01, 0x81, 0x2d, 0xd3, 0x12, 0x09, 0x44, 0xed, - 0x62, 0x16, 0x94, 0xb2, 0xd2, 0xb3, 0xd0, 0x39, 0x95, 0xbb, 0xb9, 0xaa, 0xf4, 0x79, 0x35, 0x2a, - 0x6a, 0x6b, 0x89, 0xa8, 0x5a, 0x25, 0x0d, 0xb5, 0x04, 0x0c, 0xfb, 0x31, 0x2c, 0xab, 0x23, 0x2e, - 0xe8, 0x82, 0x55, 0xb4, 0x49, 0x70, 0x98, 0xd1, 0xf4, 0xf4, 0x8c, 0x8c, 0xb0, 0x32, 0x61, 0xc5, - 0x41, 0x61, 0x6d, 0xe8, 0xf9, 0x12, 0x3d, 0x8f, 0x00, 0xfc, 0x53, 0xb8, 0x15, 0xb2, 0x9c, 0xe8, - 0xbb, 0xc2, 0x9c, 0x7d, 0x77, 0x11, 0x03, 0xfe, 0x35, 0x00, 0x93, 0xc4, 0x83, 0x5e, 0x2d, 0x4f, - 0x31, 0xbe, 0x34, 0x15, 0x07, 0x0d, 0x10, 0xb4, 0x18, 0x32, 0x2e, 0x89, 0x7d, 0x5c, 0x69, 0x9e, - 0xa8, 0xfb, 0x55, 0x17, 0xb5, 0xb0, 0x5c, 0xff, 0xdd, 0x54, 0x2c, 0x90, 0x20, 0x03, 0x05, 0x97, - 0x2e, 0xa1, 0xb3, 0x36, 0xc4, 0xd0, 0x95, 0x57, 0xfd, 0xaf, 0x2c, 0x3b, 0x55, 0xe4, 0xfb, 0xc0, - 0xcd, 0xe9, 0xbe, 0xc8, 0xce, 0xd9, 0x17, 0x33, 0x68, 0x27, 0xf7, 0x33, 0x72, 0xd3, 0xfb, 0x19, - 0x77, 0x00, 0xfa, 0x96, 0x73, 0xa4, 0x36, 0x68, 0xf3, 0x2a, 0xa5, 0x2c, 0x84, 0xd4, 0x7f, 0x3a, - 0x05, 0x37, 0x27, 0x5a, 0xfc, 0xcc, 0xf4, 0x07, 0x28, 0x85, 0xdb, 0xb0, 0x6c, 0x26, 0x9f, 0xa8, - 0xa8, 0xcd, 0x64, 0xe4, 0x7a, 0x82, 0x5e, 0x9b, 0x24, 0x43, 0x99, 0x53, 0xbe, 0x6a, 0x10, 0xe0, - 0x51, 0x33, 0x7e, 0x12, 0x5c, 0xb7, 0xa0, 0x18, 0x5c, 0x45, 0x8b, 0xb3, 0x83, 0x2e, 0xa3, 0x0d, - 0x03, 0xc6, 0xb2, 0xc4, 0xb7, 0x61, 0x49, 0x24, 0xbb, 0x30, 0x3d, 0x67, 0x17, 0x4e, 0xd0, 0xd5, - 0xbf, 0x06, 0x2b, 0x53, 0x48, 0x38, 0xa6, 0x23, 0xdd, 0x0f, 0xef, 0x20, 0xc1, 0xff, 0xd3, 0x69, - 0x18, 0xf5, 0x7f, 0x9f, 0x86, 0xc5, 0x5d, 0xdd, 0x36, 0x8f, 0x85, 0xe7, 0x53, 0x6d, 0x6f, 0x41, - 0xde, 0xeb, 0x0d, 0xc4, 0x50, 0x0f, 0xcc, 0x8a, 0x57, 0x64, 0x51, 0x85, 0x91, 0xd2, 0xf1, 0xad, - 0x9b, 0xa9, 0xbd, 0x40, 0x54, 0x04, 0x63, 0x7f, 0x10, 0x9e, 0x41, 0x51, 0x25, 0x94, 0x25, 0xcb, - 0xec, 0x09, 0xdb, 0x0b, 0x26, 0x7b, 0x50, 0x8c, 0xd2, 0xb2, 0xf2, 0x97, 0xa4, 0x65, 0x15, 0xa6, - 0xe5, 0x01, 0x27, 0x75, 0xcf, 0x15, 0xc2, 0xf6, 0x06, 0x8e, 0x1f, 0xdc, 0x63, 0x1c, 0x07, 0x51, - 0x5e, 0xa9, 0xf3, 0xdc, 0x46, 0xa5, 0xbb, 0x63, 0xda, 0x27, 0x2a, 0x19, 0x32, 0x01, 0xc3, 0x39, - 0x41, 0x41, 0x34, 0xf3, 0x33, 0x41, 0xda, 0x23, 0xa7, 0x85, 0x65, 0x0a, 0x93, 0xe9, 0xbe, 0xe8, - 0x3b, 0xae, 0x29, 0x64, 0xac, 0xb8, 0xa4, 0xc5, 0x20, 0x48, 0x6b, 0xe9, 0x76, 0x7f, 0xac, 0xf7, - 0x85, 0x52, 0xbb, 0x61, 0xb9, 0xfe, 0xdf, 0x73, 0x00, 0xbb, 0x62, 0x78, 0x24, 0x5c, 0x6f, 0x60, - 0x8e, 0x68, 0x97, 0xcb, 0x54, 0x09, 0xf6, 0x15, 0x8d, 0xfe, 0xf3, 0x47, 0x89, 0x43, 0x31, 0xd3, - 0x3b, 0xd7, 0x11, 0xf9, 0x64, 0x8c, 0x0d, 0x3b, 0x47, 0xf7, 0x85, 0xca, 0x88, 0xa3, 0xfe, 0xcf, - 0x6a, 0x71, 0x10, 0x65, 0x89, 0xea, 0xbe, 0x68, 0xda, 0x86, 0x8c, 0xe1, 0x65, 0xb5, 0xb0, 0x4c, - 0xbb, 0x75, 0x5e, 0x63, 0xec, 0x3b, 0x9a, 0xb0, 0xc5, 0xf3, 0xf0, 0x78, 0x69, 0x04, 0xe2, 0xbb, - 0x50, 0x19, 0xe9, 0xe7, 0x43, 0x61, 0xa3, 0x38, 0x0f, 0x1c, 0x43, 0xa5, 0xaf, 0xbd, 0x76, 0x71, - 0x05, 0xf7, 0xe3, 0xe8, 0x5a, 0x92, 0x1a, 0x65, 0xc2, 0xf6, 0x68, 0xd6, 0xca, 0x61, 0x54, 0x25, - 0xbe, 0x0e, 0x20, 0xff, 0xc5, 0x54, 0xdf, 0x54, 0x58, 0x4f, 0x1f, 0x0a, 0x4f, 0xb8, 0xa7, 0xa6, - 0x5c, 0x18, 0xa4, 0x0e, 0x8c, 0xa8, 0x50, 0x71, 0x8f, 0x3d, 0xe1, 0x36, 0x87, 0xba, 0x69, 0xa9, - 0x01, 0x8e, 0x00, 0xfc, 0x2d, 0xb8, 0xe1, 0x8d, 0x8f, 0x50, 0x66, 0x8e, 0x44, 0xd7, 0xd9, 0x13, - 0xcf, 0x3d, 0x4b, 0xf8, 0xbe, 0x70, 0x55, 0x8a, 0xcc, 0xec, 0x87, 0xf5, 0x7e, 0x68, 0xd7, 0xd2, - 0x05, 0x4b, 0xf8, 0x2f, 0xca, 0xc3, 0x0b, 0x41, 0x2a, 0x49, 0x91, 0xa5, 0x38, 0x83, 0x45, 0x09, - 0x52, 0x39, 0x8c, 0x69, 0xfe, 0x45, 0xf8, 0xfe, 0x04, 0x92, 0x26, 0xb3, 0x04, 0xbc, 0x2d, 0xd3, - 0xd6, 0x2d, 0xf3, 0x33, 0x99, 0xf2, 0x90, 0xa9, 0x8f, 0xa0, 0x92, 0xe8, 0x38, 0x3a, 0x0f, 0x4d, - 0xff, 0x54, 0x22, 0x17, 0x83, 0x45, 0x59, 0xee, 0xf8, 0xae, 0x49, 0x5b, 0x58, 0x21, 0x64, 0x03, - 0x27, 0xba, 0xc3, 0xd2, 0xfc, 0x3a, 0x30, 0x09, 0x69, 0xd9, 0xfa, 0x68, 0xd4, 0x18, 0x8d, 0x2c, - 0xc1, 0x32, 0x74, 0xd6, 0x3c, 0x82, 0xca, 0xa3, 0x31, 0x2c, 0x5b, 0xff, 0x06, 0xdc, 0xa2, 0x9e, - 0x39, 0x10, 0x6e, 0x18, 0xb9, 0x50, 0x6d, 0xbd, 0x01, 0x2b, 0xf2, 0xdf, 0x9e, 0xe3, 0xcb, 0xc7, - 0x64, 0xcd, 0x73, 0x58, 0x92, 0x60, 0x34, 0x5f, 0x3b, 0x82, 0x4e, 0x90, 0x87, 0xb0, 0x10, 0x2f, - 0x5d, 0xff, 0xcd, 0x3c, 0xf0, 0x48, 0x20, 0xba, 0xa6, 0x70, 0x37, 0x75, 0x5f, 0x8f, 0x85, 0x9e, - 0x2b, 0x17, 0x26, 0x5e, 0xbc, 0x38, 0x05, 0xf3, 0x26, 0xe4, 0x4d, 0x0f, 0x7d, 0x6d, 0x95, 0xbb, - 0xae, 0x4a, 0x7c, 0x07, 0x60, 0x24, 0x5c, 0xd3, 0x31, 0x48, 0x82, 0x72, 0x33, 0xcf, 0x26, 0x4d, - 0x57, 0x6a, 0x6d, 0x3f, 0xa4, 0xd1, 0x62, 0xf4, 0x58, 0x0f, 0x59, 0x92, 0x69, 0x0c, 0x79, 0x69, - 0x26, 0xc4, 0x40, 0xfc, 0x0d, 0xb8, 0x36, 0x72, 0xcd, 0x9e, 0x90, 0xc3, 0xf1, 0xd4, 0x33, 0x36, - 0xe8, 0x5a, 0xd2, 0x02, 0x61, 0xce, 0x7a, 0x84, 0x12, 0xa8, 0xdb, 0xe4, 0x81, 0x7a, 0xb4, 0x71, - 0xaf, 0xee, 0x5c, 0x90, 0xb9, 0xdb, 0x15, 0x6d, 0xf6, 0x43, 0xbe, 0x0a, 0x4c, 0x3d, 0xd8, 0x35, - 0xed, 0x1d, 0x61, 0xf7, 0xfd, 0x01, 0x09, 0x77, 0x45, 0x9b, 0x82, 0x93, 0x06, 0x93, 0x97, 0xbf, - 0xc9, 0x8d, 0xb9, 0x92, 0x16, 0x96, 0xe5, 0x75, 0x25, 0x96, 0xe3, 0x76, 0x7c, 0x57, 0xa5, 0xa9, - 0x87, 0x65, 0x32, 0x9f, 0xa8, 0xae, 0xfb, 0xae, 0x63, 0x8c, 0x69, 0xdb, 0x48, 0x2a, 0xb1, 0x49, - 0x70, 0x84, 0xb9, 0xab, 0xdb, 0x2a, 0x0f, 0xb6, 0x12, 0xc7, 0x0c, 0xc1, 0xe4, 0x64, 0x3b, 0x5e, - 0xc4, 0x70, 0x59, 0x39, 0xd9, 0x31, 0x98, 0xc2, 0x89, 0x58, 0xb1, 0x10, 0x27, 0xe2, 0x43, 0xed, - 0x37, 0x5c, 0xc7, 0x34, 0x22, 0x5e, 0x32, 0x25, 0x6b, 0x0a, 0x1e, 0xc3, 0x8d, 0x78, 0xf2, 0x04, - 0x6e, 0xc4, 0xf7, 0x3a, 0xe4, 0x9c, 0xe3, 0x63, 0xe1, 0xd2, 0xfd, 0x20, 0x25, 0x4d, 0x16, 0xea, - 0x3f, 0x93, 0x02, 0x88, 0x44, 0x02, 0x27, 0x42, 0x54, 0x8a, 0x26, 0xfe, 0x2d, 0xb8, 0x16, 0x07, - 0x5b, 0x2a, 0xc3, 0x99, 0x66, 0x43, 0xf4, 0x60, 0x53, 0x3f, 0xf7, 0x58, 0x5a, 0xdd, 0x85, 0xa0, - 0x60, 0xcf, 0x84, 0xa0, 0x74, 0xd1, 0xeb, 0xc0, 0x22, 0x20, 0x1d, 0x70, 0xf5, 0x58, 0x36, 0x89, - 0xfa, 0x89, 0xd0, 0x5d, 0x8f, 0xe5, 0xea, 0x3f, 0x7b, 0x13, 0xe7, 0x79, 0x20, 0xb8, 0x07, 0x0f, - 0x6a, 0x5b, 0x90, 0x6f, 0x0c, 0x29, 0xf3, 0x03, 0xc7, 0x94, 0x4e, 0xb6, 0xf6, 0x42, 0x2b, 0x2e, - 0x28, 0x53, 0x3a, 0x3c, 0x61, 0x49, 0xb9, 0x94, 0x3b, 0x3b, 0x71, 0x50, 0xed, 0x19, 0x14, 0x5a, - 0xf6, 0xa9, 0x63, 0xf6, 0x84, 0xca, 0xbf, 0x94, 0xa6, 0x50, 0x56, 0x9e, 0x48, 0xe4, 0x8f, 0x20, - 0xe7, 0x3b, 0xbe, 0x6e, 0xa9, 0xbd, 0xca, 0xfa, 0x85, 0x73, 0xe9, 0xe0, 0xc1, 0x9a, 0xac, 0x8f, - 0x26, 0x09, 0x6a, 0xbf, 0x92, 0x86, 0xe2, 0x56, 0x20, 0x77, 0x68, 0xbe, 0xcb, 0x0c, 0xfe, 0xf5, - 0x73, 0x5f, 0x78, 0xea, 0x15, 0x09, 0x58, 0x68, 0xe2, 0x6b, 0x94, 0xe4, 0x29, 0x2b, 0x5b, 0xd1, - 0x12, 0xb0, 0x10, 0xe7, 0x99, 0x6b, 0xd2, 0x59, 0xf0, 0x4c, 0x0c, 0x47, 0xc1, 0x08, 0x67, 0xa0, - 0xbb, 0xc2, 0x88, 0x9d, 0x6e, 0x41, 0x9c, 0x18, 0x0c, 0x57, 0x09, 0x5f, 0xe8, 0xc3, 0x8e, 0xd0, - 0x7d, 0x79, 0xf6, 0xa7, 0xa2, 0x45, 0x00, 0xe4, 0xa0, 0x66, 0x95, 0xcc, 0xb5, 0x91, 0x13, 0x3f, - 0x01, 0xa3, 0x1b, 0x66, 0xe2, 0x33, 0x4f, 0xcd, 0xf9, 0x24, 0x50, 0x6e, 0xd5, 0x99, 0xa7, 0xb8, - 0x10, 0xcb, 0xca, 0xc8, 0x59, 0x9e, 0x04, 0xd6, 0x7e, 0x21, 0x0b, 0x05, 0x25, 0xbf, 0xb3, 0xd2, - 0x61, 0x3e, 0x87, 0x7e, 0xa4, 0x64, 0xcd, 0xae, 0x33, 0xda, 0x11, 0xa7, 0xc2, 0x52, 0x3a, 0x32, - 0x06, 0x51, 0x67, 0xfc, 0x64, 0x8e, 0x54, 0x2e, 0x3c, 0xe3, 0x27, 0xb3, 0xa4, 0x68, 0xc3, 0xae, - 0x65, 0xfb, 0xae, 0xa3, 0xd2, 0xa7, 0x82, 0x22, 0xb6, 0xc6, 0xf4, 0x9e, 0x8e, 0xfa, 0xae, 0x6e, - 0x08, 0xba, 0x2e, 0x59, 0x26, 0x51, 0x25, 0x81, 0x7c, 0x0b, 0x16, 0x49, 0xf1, 0x79, 0x28, 0xbb, - 0xd6, 0x39, 0x19, 0x62, 0xf3, 0x49, 0x4e, 0x82, 0x8e, 0x6f, 0x53, 0xdf, 0xf5, 0x84, 0x47, 0x33, - 0xc3, 0x3a, 0xa7, 0x8c, 0xaa, 0xf9, 0x18, 0x25, 0x09, 0x13, 0x5a, 0x0f, 0x26, 0xb4, 0x5e, 0xa8, - 0x01, 0xca, 0x31, 0x0d, 0xc0, 0x3f, 0x8c, 0xe9, 0xd0, 0x45, 0x92, 0xfc, 0x57, 0x2e, 0x7b, 0x6d, - 0x20, 0xe7, 0x31, 0x4d, 0xfb, 0x11, 0x2c, 0xc9, 0x4a, 0xec, 0x98, 0xc7, 0xc2, 0x37, 0x87, 0x42, - 0x79, 0xd5, 0xf3, 0x54, 0x7f, 0x82, 0xb2, 0xf6, 0xab, 0x29, 0x58, 0xdc, 0x1f, 0xbb, 0xbd, 0x81, - 0xee, 0x49, 0xff, 0x61, 0xc2, 0xde, 0x4b, 0x5d, 0x6e, 0xef, 0xa5, 0x2f, 0xb7, 0xf7, 0x32, 0xd3, - 0xf6, 0xde, 0xbb, 0x90, 0x97, 0xab, 0xdc, 0x05, 0xfb, 0xbd, 0x89, 0x4a, 0x4b, 0x65, 0xa5, 0x29, - 0x8a, 0xda, 0xbf, 0x4a, 0x41, 0x45, 0x09, 0xb3, 0x32, 0x24, 0xb6, 0x43, 0xbb, 0x56, 0x66, 0xf5, - 0xbc, 0x71, 0x29, 0xb7, 0x38, 0xe9, 0x84, 0x9d, 0x5b, 0xb7, 0xbe, 0x67, 0x43, 0x6c, 0x15, 0x5e, - 0x9d, 0x69, 0x88, 0x35, 0xe4, 0xb4, 0x6d, 0x58, 0x96, 0xa3, 0x12, 0x50, 0x33, 0xb5, 0xff, 0x91, - 0x02, 0x16, 0x74, 0x7b, 0xb0, 0xbe, 0xf0, 0xf7, 0x28, 0xc1, 0x17, 0xff, 0x2a, 0x97, 0xf1, 0x0b, - 0x73, 0xb4, 0x46, 0x0b, 0x68, 0xf8, 0x0e, 0x2c, 0x8e, 0x62, 0x23, 0xa9, 0xd4, 0xea, 0xbd, 0x4b, - 0x79, 0xc4, 0xf0, 0xb5, 0x04, 0x35, 0x6f, 0x53, 0x26, 0x40, 0xd4, 0x5f, 0x17, 0x1c, 0x84, 0xbf, - 0xb8, 0x83, 0xb5, 0x24, 0x7d, 0xed, 0x5b, 0x29, 0x28, 0x6f, 0xe8, 0xae, 0xff, 0x07, 0xd4, 0x5a, - 0x52, 0x33, 0x4a, 0x0d, 0xa4, 0x03, 0x35, 0xa3, 0xa6, 0xf7, 0x45, 0x97, 0x7e, 0x93, 0xea, 0x0a, - 0x27, 0x4d, 0xa8, 0xba, 0xc2, 0xc9, 0xf0, 0x77, 0xd2, 0x74, 0xbd, 0xb1, 0xcf, 0x37, 0xa0, 0xa8, - 0xde, 0x13, 0xe4, 0x9d, 0xbd, 0x76, 0x59, 0xe5, 0x62, 0xcd, 0xd2, 0x42, 0xc2, 0xcf, 0xbf, 0xbe, - 0xf1, 0x3d, 0x60, 0xf4, 0x67, 0x4f, 0x9c, 0xf9, 0x6a, 0x05, 0x55, 0xdd, 0x3f, 0x0f, 0x93, 0x29, - 0x5a, 0x34, 0xaa, 0xec, 0xa8, 0x48, 0xe7, 0x1e, 0xa5, 0xa3, 0x36, 0x09, 0xe6, 0xaf, 0xc3, 0x8a, - 0x3e, 0xa2, 0xd0, 0xf9, 0xbe, 0xeb, 0x0c, 0x9d, 0x9e, 0x63, 0x84, 0x5f, 0x07, 0x9a, 0x7e, 0x50, - 0xfb, 0xe9, 0x14, 0x2c, 0x49, 0xdf, 0x80, 0xae, 0x07, 0x70, 0xc6, 0x3e, 0x75, 0xb1, 0xe4, 0xf7, - 0x54, 0xdb, 0x51, 0x6b, 0x4d, 0x0c, 0xf2, 0x3d, 0x74, 0x0a, 0xa9, 0x96, 0x0d, 0xdd, 0xde, 0xd0, - 0xed, 0x5e, 0x78, 0x0e, 0x35, 0x0e, 0xaa, 0xfd, 0x46, 0x9a, 0x6e, 0x20, 0xd0, 0xf9, 0xf6, 0xd4, - 0xf0, 0xbd, 0x3e, 0xcf, 0x2c, 0x30, 0xa6, 0xc7, 0xb0, 0x09, 0xe5, 0x58, 0x17, 0xa9, 0x4a, 0x5f, - 0x2a, 0xa8, 0x0a, 0x55, 0x8b, 0xd3, 0xc9, 0x43, 0x8c, 0xfa, 0xb0, 0xfd, 0xdc, 0x16, 0x6e, 0x6b, - 0x33, 0x58, 0x55, 0x63, 0x20, 0xfe, 0x14, 0x96, 0x95, 0x23, 0xbb, 0xef, 0x3a, 0xa7, 0xa6, 0x21, - 0x5c, 0xa5, 0x1f, 0xbf, 0x7c, 0x69, 0xcd, 0x93, 0x24, 0xda, 0x24, 0x8f, 0xab, 0x8d, 0x67, 0xbd, - 0x0d, 0xc5, 0x7d, 0x4b, 0xf7, 0x8f, 0x1d, 0x77, 0x98, 0x3c, 0x10, 0x46, 0x87, 0x8f, 0xbc, 0x13, - 0x9f, 0xce, 0xa8, 0x57, 0xa0, 0xb4, 0xeb, 0x1c, 0x99, 0x96, 0x68, 0xb5, 0x3b, 0xf2, 0x4a, 0x2f, - 0x59, 0x6c, 0x48, 0x0b, 0x58, 0x6e, 0x86, 0x3c, 0x13, 0x47, 0x2c, 0x5b, 0x37, 0x60, 0x79, 0xa2, - 0x8a, 0xc9, 0x83, 0x4f, 0xa1, 0xab, 0x09, 0x90, 0x0f, 0x9d, 0xcc, 0x15, 0xa8, 0xac, 0x9b, 0x96, - 0x65, 0xda, 0xfd, 0x7d, 0xc7, 0xf5, 0x75, 0x4b, 0x5e, 0xfb, 0xd2, 0x18, 0x8d, 0x3a, 0xbe, 0xe3, - 0x0a, 0xb5, 0xc5, 0x42, 0x4e, 0xe6, 0xbe, 0xa5, 0x9f, 0xb3, 0x5c, 0x5d, 0x83, 0xbc, 0x5c, 0x28, - 0xf8, 0x0a, 0x94, 0x22, 0x6b, 0x79, 0xa1, 0x96, 0x2e, 0x52, 0x26, 0xac, 0x5a, 0xab, 0xe5, 0x6b, - 0xa4, 0x8a, 0x60, 0x69, 0x3a, 0x43, 0x35, 0x70, 0x85, 0x32, 0x83, 0xe5, 0xd5, 0x32, 0x4a, 0x11, - 0xb0, 0x6c, 0x7d, 0x1b, 0xf2, 0x32, 0xd1, 0x6e, 0x46, 0x8a, 0xec, 0xd5, 0x72, 0xe5, 0xff, 0x5c, - 0x0a, 0x60, 0x53, 0x9e, 0x9a, 0x45, 0xbd, 0x3a, 0x8f, 0x11, 0x26, 0x2f, 0xd1, 0xa5, 0x79, 0x9a, - 0x09, 0x2f, 0xd1, 0xa5, 0xf9, 0x59, 0x83, 0xa2, 0x1e, 0x9c, 0x93, 0x91, 0xfa, 0x2b, 0x2c, 0xab, - 0x09, 0xe2, 0xd8, 0xb6, 0xe8, 0xe1, 0xca, 0x1d, 0xc6, 0x5a, 0x42, 0x50, 0xfd, 0xdf, 0xa6, 0xa1, - 0xb4, 0x31, 0xd0, 0x7d, 0x79, 0xe7, 0xec, 0x87, 0x50, 0x54, 0xdf, 0xd9, 0xf1, 0x94, 0x06, 0x7e, - 0x65, 0xc6, 0x37, 0x79, 0x08, 0x77, 0xed, 0xa9, 0xed, 0x0a, 0xdd, 0x90, 0x17, 0xed, 0x86, 0x54, - 0x92, 0x83, 0xed, 0x87, 0x1b, 0x7f, 0x57, 0xe0, 0x60, 0x87, 0x9f, 0xd3, 0xb1, 0x74, 0x4f, 0xa2, - 0x84, 0x9b, 0xfa, 0x71, 0x10, 0x19, 0x51, 0x6e, 0x30, 0x1d, 0x32, 0x9a, 0x2c, 0xa0, 0xab, 0x3b, - 0x26, 0x86, 0x9a, 0xd0, 0x69, 0x77, 0xaf, 0xad, 0x72, 0xb1, 0x65, 0xc0, 0x6f, 0xf6, 0xc3, 0xda, - 0x2e, 0x94, 0x63, 0xd5, 0x40, 0x9b, 0xd3, 0xb1, 0x0c, 0xe1, 0xf9, 0x01, 0xb1, 0xfa, 0x44, 0x42, - 0x02, 0x48, 0x69, 0xea, 0xa8, 0x87, 0x84, 0xab, 0xf2, 0x0d, 0x83, 0x62, 0xfd, 0xaf, 0x5e, 0x83, - 0x72, 0xec, 0xb3, 0x45, 0x53, 0x43, 0x1b, 0x4b, 0x11, 0x4f, 0x27, 0x52, 0xc4, 0xe3, 0xa9, 0xef, - 0x99, 0x64, 0xea, 0x7b, 0xe2, 0xdc, 0x6d, 0x76, 0xf2, 0xdc, 0xed, 0x1d, 0x80, 0xa1, 0x63, 0x50, - 0x18, 0xa4, 0x21, 0xf3, 0xca, 0x32, 0x5a, 0x0c, 0x42, 0x5b, 0x20, 0xaa, 0x2b, 0xcb, 0x6a, 0x0b, - 0x44, 0x75, 0x23, 0x9d, 0x41, 0x18, 0x59, 0xe7, 0x5d, 0x67, 0x37, 0xfc, 0x30, 0x54, 0x78, 0x9f, - 0x58, 0x12, 0xce, 0x37, 0x26, 0xbf, 0xd5, 0xf4, 0xa5, 0x8b, 0xbf, 0xd5, 0x14, 0x7c, 0x01, 0x4b, - 0x5d, 0xc6, 0x11, 0x7e, 0xba, 0x89, 0x3f, 0x86, 0xb2, 0xee, 0xfb, 0x7a, 0x6f, 0x30, 0x54, 0x61, - 0x8b, 0xcc, 0x8c, 0x44, 0xda, 0x38, 0xa3, 0x46, 0x88, 0xad, 0xc5, 0x29, 0xf9, 0x3a, 0x94, 0x5c, - 0x35, 0x8e, 0x41, 0x2e, 0xef, 0x2b, 0x97, 0xb0, 0x09, 0xc6, 0xdc, 0xd3, 0x22, 0xb2, 0xf0, 0x6b, - 0x21, 0x10, 0xfb, 0x5a, 0x08, 0x1d, 0xb2, 0x26, 0x31, 0x44, 0x0f, 0x50, 0x5d, 0x86, 0x1a, 0x07, - 0x61, 0x6f, 0x0f, 0x74, 0x4f, 0xdd, 0x3f, 0xae, 0x0e, 0x83, 0xc5, 0x20, 0x94, 0x3b, 0x7b, 0x6e, - 0xf7, 0x54, 0xda, 0x5b, 0x51, 0x53, 0x25, 0x84, 0x8f, 0x4c, 0xdb, 0x16, 0x86, 0x3a, 0xe5, 0xa1, - 0x4a, 0xfc, 0x55, 0x58, 0x4a, 0xca, 0x25, 0xc5, 0x23, 0x8a, 0xda, 0x04, 0x94, 0x7f, 0x10, 0xe6, - 0x76, 0xaf, 0xcc, 0xb4, 0x3d, 0x66, 0x74, 0x7f, 0x22, 0xdd, 0xbb, 0xf6, 0x4b, 0x29, 0x58, 0x4a, - 0x8e, 0xcb, 0x1f, 0xc6, 0xc5, 0xfc, 0x5f, 0x8f, 0x2e, 0xe6, 0xff, 0x1c, 0x97, 0xdc, 0xff, 0x62, - 0x0a, 0x20, 0x1a, 0x72, 0xec, 0x33, 0x79, 0x0f, 0x78, 0xb0, 0x6d, 0x21, 0x4b, 0x7c, 0x3b, 0x71, - 0xab, 0xe2, 0x5b, 0x73, 0xc9, 0x4f, 0xec, 0x6f, 0xec, 0xa4, 0xf3, 0x7d, 0x58, 0x4a, 0xc2, 0xe9, - 0x84, 0x78, 0x6b, 0xa7, 0x29, 0xd3, 0x08, 0x5a, 0xbb, 0x8d, 0xc7, 0x4d, 0x75, 0x9b, 0x4b, 0x6b, - 0xef, 0x09, 0x4b, 0xd7, 0xfe, 0x67, 0x0a, 0x4a, 0xa1, 0x34, 0xf1, 0x8f, 0xe3, 0x62, 0x28, 0x8d, - 0x8a, 0x87, 0xf3, 0x88, 0x61, 0xf4, 0xaf, 0x69, 0xfb, 0xee, 0x79, 0x4c, 0x2a, 0x6b, 0x0e, 0x2c, - 0x25, 0x1f, 0xce, 0x58, 0x6b, 0x1e, 0x27, 0xd7, 0x9a, 0x37, 0xe7, 0x7a, 0x65, 0xb0, 0xb5, 0xb4, - 0x63, 0x7a, 0xbe, 0x5a, 0x86, 0xde, 0x4d, 0x3f, 0x4a, 0xd5, 0xee, 0xc2, 0x62, 0xfc, 0xd1, 0x8c, - 0x9b, 0x9e, 0x7e, 0x3e, 0x03, 0x8b, 0x71, 0x91, 0xe2, 0x8d, 0x98, 0xdc, 0x94, 0xa7, 0x8c, 0x91, - 0x8b, 0x24, 0x31, 0xf1, 0x99, 0xaf, 0x86, 0xfa, 0x74, 0x55, 0xfa, 0x4a, 0x2c, 0x12, 0x1f, 0xb1, - 0xda, 0x84, 0x1c, 0x9a, 0x3d, 0x86, 0xb2, 0x82, 0x5f, 0x9f, 0x93, 0x07, 0x5d, 0xb3, 0xbe, 0xbd, - 0xa0, 0x49, 0x62, 0x7e, 0x00, 0x65, 0x79, 0x24, 0x83, 0x6e, 0xf8, 0x57, 0x47, 0xc7, 0x1e, 0xcc, - 0xcb, 0x2b, 0xa2, 0xdc, 0x5e, 0xd0, 0xe2, 0x8c, 0xf8, 0x27, 0xb0, 0xa8, 0xb4, 0x9e, 0x64, 0x2c, - 0x2f, 0x7c, 0x79, 0x38, 0x27, 0xe3, 0xdd, 0x18, 0xe9, 0xf6, 0x82, 0x96, 0x60, 0x15, 0xbb, 0x22, - 0xa7, 0xf6, 0xef, 0x52, 0xc0, 0x26, 0xfb, 0xf8, 0x8f, 0xdc, 0xb4, 0xbe, 0xe4, 0x73, 0x1b, 0x1c, - 0xb2, 0x96, 0x6e, 0xf7, 0x83, 0x04, 0x49, 0xfc, 0x5f, 0xfb, 0xa7, 0x13, 0x6d, 0xa2, 0x6d, 0xb2, - 0xe9, 0x0f, 0x05, 0xa4, 0x66, 0x7e, 0x28, 0x60, 0x7e, 0xd5, 0x30, 0xf9, 0x0a, 0xba, 0x0c, 0x21, - 0xa6, 0x1a, 0xbe, 0x86, 0xa6, 0x9f, 0x84, 0xc4, 0xbe, 0x2c, 0x71, 0xc1, 0xd5, 0xff, 0x8b, 0xd1, - 0x97, 0x68, 0x58, 0xa6, 0xe6, 0xc1, 0xca, 0x94, 0xc4, 0xfd, 0x61, 0xdf, 0x6c, 0x56, 0xfb, 0x0c, - 0x6e, 0x5d, 0x20, 0x9a, 0x97, 0x1c, 0x79, 0x68, 0x86, 0xa2, 0x74, 0xc5, 0x99, 0x88, 0x63, 0xac, - 0x85, 0x62, 0xf8, 0xcb, 0x29, 0xa8, 0x5e, 0x24, 0xbe, 0xc9, 0xef, 0x4e, 0xa6, 0x26, 0xbf, 0x3b, - 0x39, 0xd7, 0xa1, 0x8b, 0x78, 0x3d, 0x33, 0x9f, 0xbf, 0x9e, 0xf5, 0x5f, 0x4e, 0x43, 0x99, 0xae, - 0x95, 0x1d, 0xa0, 0x39, 0x37, 0x75, 0xa2, 0x2f, 0x35, 0x7d, 0xa2, 0xef, 0x11, 0xe4, 0x49, 0x55, - 0x5e, 0xb4, 0x33, 0x1e, 0xe3, 0xa6, 0x2c, 0x7c, 0x85, 0x5f, 0xfb, 0x66, 0x70, 0x42, 0x2c, 0xb8, - 0x82, 0x23, 0x35, 0x33, 0xb3, 0x35, 0xce, 0x20, 0x76, 0x52, 0xea, 0x6a, 0xee, 0xc4, 0x9e, 0x4a, - 0x22, 0xbb, 0x0e, 0x2c, 0xc6, 0x4f, 0xde, 0xaf, 0xbc, 0x30, 0x01, 0x95, 0x77, 0x68, 0xa6, 0x28, - 0x81, 0x29, 0x82, 0xaa, 0xbb, 0x2a, 0x9f, 0x7a, 0xc2, 0x65, 0xe9, 0xd5, 0x7f, 0x99, 0x85, 0xa5, - 0xe4, 0x01, 0x2e, 0xba, 0x12, 0x4c, 0x1e, 0x1e, 0x6c, 0x5b, 0x46, 0xec, 0x3a, 0x10, 0xc6, 0x97, - 0xa1, 0xac, 0xb2, 0x13, 0x08, 0xb0, 0x42, 0xa9, 0x69, 0xce, 0x50, 0xb0, 0xbb, 0xf1, 0xcf, 0x4d, - 0xbd, 0x81, 0x93, 0x48, 0xde, 0xca, 0xc6, 0x46, 0xbc, 0xa4, 0x26, 0xd1, 0x8f, 0xa7, 0x79, 0x25, - 0x76, 0x29, 0xc5, 0xb7, 0xd2, 0xfc, 0x3a, 0x2c, 0xaf, 0x8f, 0x6d, 0xc3, 0x12, 0x46, 0x08, 0xfd, - 0xa5, 0x38, 0x34, 0xbc, 0x55, 0xe2, 0xc7, 0xd1, 0xe7, 0x2b, 0x75, 0xc6, 0x47, 0x6a, 0x66, 0xfe, - 0xc9, 0x2c, 0xbf, 0x09, 0x2b, 0x0a, 0x2b, 0x3a, 0x74, 0xcd, 0xfe, 0x54, 0x96, 0x5f, 0x83, 0xa5, - 0x86, 0xec, 0x74, 0x55, 0x51, 0xf6, 0xa7, 0xb3, 0x58, 0x05, 0xba, 0x67, 0xf4, 0xcf, 0x10, 0x9f, - 0xf0, 0xc6, 0x24, 0xf6, 0x13, 0x59, 0xbe, 0x0c, 0xd0, 0xe9, 0x86, 0x2f, 0xfa, 0xa9, 0x2c, 0x2f, - 0x43, 0xbe, 0xd3, 0x25, 0x6e, 0x3f, 0x93, 0xe5, 0x37, 0x80, 0x45, 0x4f, 0xd5, 0xb1, 0xf6, 0xbf, - 0x20, 0x2b, 0x13, 0x9e, 0x53, 0xff, 0xd9, 0x2c, 0xb6, 0x2b, 0x58, 0x57, 0xd9, 0x5f, 0xcc, 0x72, - 0x06, 0xe5, 0x58, 0xc2, 0x23, 0xfb, 0x4b, 0x59, 0xce, 0xa1, 0xb2, 0x9b, 0x38, 0x6f, 0xfe, 0x93, - 0xf4, 0xe6, 0xad, 0xf0, 0xd2, 0x27, 0xf6, 0x73, 0x59, 0x7e, 0x0b, 0x78, 0x3c, 0xc9, 0x5b, 0x3d, - 0xf8, 0xcb, 0x44, 0x2d, 0x1d, 0x48, 0x4f, 0xc1, 0x7e, 0x3e, 0xcb, 0x5f, 0x82, 0xeb, 0x38, 0x0f, - 0x24, 0x20, 0x76, 0x0e, 0xfe, 0xaf, 0x50, 0xd7, 0x6c, 0x44, 0x07, 0xe1, 0x15, 0xc9, 0xb7, 0x88, - 0x4d, 0x30, 0xac, 0x12, 0xf6, 0x4b, 0xd4, 0xc0, 0xcd, 0xf0, 0xa0, 0xbb, 0x02, 0xff, 0xf5, 0x2c, - 0x8e, 0x41, 0x37, 0xb8, 0x61, 0x49, 0x41, 0xff, 0x46, 0x96, 0xbf, 0x0c, 0x37, 0x43, 0xe8, 0x81, - 0xe9, 0xfa, 0x63, 0xdd, 0x52, 0x0f, 0xff, 0x66, 0x76, 0xf5, 0x37, 0x28, 0xf1, 0x37, 0x7e, 0x36, - 0x14, 0x35, 0xa5, 0xe5, 0xd8, 0x7d, 0x5f, 0x7e, 0x3a, 0xac, 0x02, 0x25, 0x6f, 0xe0, 0xb8, 0x3e, - 0x15, 0xc9, 0x15, 0xb7, 0xe9, 0x5e, 0x55, 0x79, 0xcf, 0x89, 0x8c, 0xc2, 0xca, 0xd8, 0x81, 0xaf, - 0xf7, 0x59, 0x39, 0xbc, 0x16, 0x20, 0x1b, 0x5e, 0x5d, 0x40, 0xf7, 0xbb, 0x06, 0x57, 0x62, 0xb2, - 0x3c, 0xa2, 0x8e, 0x5d, 0x4b, 0x5e, 0x61, 0x20, 0x86, 0xba, 0x69, 0xc9, 0x6f, 0x04, 0x8d, 0x06, - 0x8e, 0xad, 0xee, 0x30, 0x10, 0xf4, 0xb9, 0x20, 0x88, 0x9d, 0xc4, 0x35, 0xb0, 0x1e, 0xe1, 0x61, - 0x33, 0x46, 0x57, 0x49, 0x0f, 0xf5, 0x11, 0x3b, 0x5e, 0xfd, 0x6b, 0x29, 0x58, 0x0c, 0x6e, 0x2c, - 0x35, 0xfb, 0xa6, 0x2d, 0x6f, 0x43, 0x08, 0xbe, 0xcc, 0xd6, 0xb3, 0xcc, 0x51, 0xf0, 0xa5, 0xa3, - 0x65, 0x28, 0x1b, 0xae, 0xde, 0x6f, 0xd8, 0xc6, 0xa6, 0xeb, 0x8c, 0x64, 0xfd, 0xe5, 0x19, 0x01, - 0x79, 0x0b, 0xc3, 0x73, 0x71, 0x84, 0xe8, 0x23, 0xe1, 0xb2, 0x2c, 0x1d, 0xf7, 0x1d, 0xe8, 0xae, - 0x69, 0xf7, 0x9b, 0x67, 0xbe, 0xb0, 0x3d, 0x79, 0x1b, 0x43, 0x19, 0x0a, 0x63, 0x4f, 0xf4, 0x74, - 0x4f, 0xb0, 0x3c, 0x16, 0x8e, 0xc6, 0xa6, 0xe5, 0x9b, 0xb6, 0xfc, 0xc0, 0x50, 0x78, 0xdd, 0x42, - 0x11, 0x6b, 0xa7, 0x8f, 0x4c, 0x56, 0x5a, 0xfd, 0x27, 0x29, 0x28, 0x53, 0xc7, 0x47, 0x59, 0xb0, - 0x89, 0xf0, 0xcc, 0x4e, 0xf8, 0xc1, 0x98, 0x3c, 0xa4, 0xdb, 0x27, 0x32, 0x0b, 0x56, 0xc9, 0x9c, - 0xbc, 0x41, 0x50, 0x7e, 0x3b, 0x06, 0x25, 0xe7, 0x86, 0x26, 0x86, 0x8e, 0x2f, 0x9e, 0xe9, 0xa6, - 0x1f, 0xbf, 0xf9, 0x28, 0xc7, 0x57, 0xa0, 0x22, 0x1f, 0x05, 0x57, 0x1d, 0xe5, 0x29, 0xcc, 0x8d, - 0xaf, 0x0d, 0x20, 0x05, 0x6c, 0x3d, 0x41, 0x54, 0xdc, 0xbb, 0x18, 0xa2, 0x7c, 0xe4, 0x98, 0x36, - 0xbe, 0x8d, 0xae, 0xbb, 0xec, 0xc8, 0xad, 0xb5, 0xa1, 0x73, 0x8a, 0x20, 0x58, 0xfd, 0x04, 0x6e, - 0xce, 0x4e, 0x02, 0x96, 0x17, 0x61, 0xd2, 0xe7, 0x0d, 0x29, 0x24, 0x24, 0xb7, 0xda, 0xe4, 0x92, - 0x4b, 0xc1, 0x31, 0x19, 0x11, 0xda, 0x73, 0x62, 0x34, 0xea, 0xab, 0x3b, 0xc6, 0xd0, 0xb4, 0x59, - 0x76, 0xf5, 0x1d, 0x80, 0x28, 0x69, 0x4d, 0x7e, 0x16, 0x81, 0xe4, 0x8a, 0xcc, 0xfa, 0xc7, 0x63, - 0xe1, 0xa9, 0x34, 0x82, 0x67, 0xa6, 0x3f, 0x70, 0xc6, 0x41, 0xae, 0x3c, 0x4b, 0xaf, 0xf6, 0x12, - 0xb9, 0xdf, 0x51, 0xc7, 0x06, 0x0d, 0x59, 0x88, 0xdd, 0x15, 0x95, 0x92, 0x59, 0xc5, 0xf4, 0xe1, - 0x72, 0x69, 0x04, 0xa8, 0x9c, 0x6b, 0x43, 0x86, 0x8e, 0xc2, 0xa6, 0x66, 0xe5, 0x47, 0x1c, 0xec, - 0x9e, 0xb0, 0x84, 0xc1, 0x72, 0xab, 0x8f, 0x60, 0x59, 0x75, 0x17, 0x2e, 0xde, 0xc1, 0x5d, 0x4b, - 0xfb, 0x72, 0x93, 0x4e, 0x65, 0x16, 0x0b, 0xd7, 0x73, 0x6c, 0xba, 0xf5, 0x19, 0x20, 0xdf, 0xa1, - 0xcd, 0x44, 0x96, 0x5e, 0x6d, 0xaa, 0x8e, 0x56, 0x69, 0x83, 0x89, 0x2f, 0x12, 0x6d, 0xea, 0xbe, - 0xae, 0xd0, 0x7d, 0x57, 0xe8, 0xea, 0xba, 0x48, 0x9c, 0xf6, 0xb2, 0x3a, 0x6d, 0x5b, 0x74, 0x9d, - 0xb6, 0x2d, 0x58, 0x76, 0xd5, 0x57, 0xca, 0x2a, 0x58, 0x35, 0xc2, 0x42, 0x24, 0x3f, 0x71, 0x68, - 0xf8, 0x8d, 0x88, 0x70, 0x14, 0x11, 0x8a, 0xf3, 0x5e, 0x0e, 0x44, 0x08, 0x52, 0x2f, 0xbb, 0xa1, - 0x6e, 0xd1, 0x41, 0x50, 0xec, 0xad, 0x1b, 0x50, 0x22, 0x83, 0xe9, 0x89, 0x69, 0x1b, 0x38, 0x0e, - 0xeb, 0xea, 0x12, 0x13, 0xfa, 0x36, 0xc0, 0x29, 0x0d, 0x70, 0x51, 0x7e, 0x65, 0x8e, 0xa5, 0xf9, - 0x4d, 0xe0, 0x8d, 0xb1, 0xef, 0x0c, 0x75, 0xba, 0x24, 0xd2, 0x3a, 0x97, 0x5f, 0x24, 0xcc, 0xac, - 0x7e, 0x00, 0x5c, 0xe6, 0xb5, 0x19, 0xe2, 0xcc, 0xb4, 0xfb, 0xe1, 0xb5, 0xb5, 0x40, 0x17, 0x56, - 0x1b, 0xe2, 0x2c, 0xb8, 0xf5, 0x2b, 0x28, 0x04, 0xd7, 0x66, 0x6f, 0x39, 0x63, 0x1b, 0xbb, 0xf0, - 0x00, 0xae, 0xcb, 0x49, 0x83, 0x7d, 0x4a, 0x37, 0x10, 0x5e, 0xb8, 0xc5, 0x23, 0x6f, 0xfd, 0xf2, - 0xc7, 0x5e, 0x88, 0xcb, 0x52, 0x58, 0xb1, 0x30, 0x4f, 0x25, 0x82, 0xa7, 0x57, 0xeb, 0x70, 0x6d, - 0x46, 0xb2, 0x10, 0xad, 0x81, 0x72, 0xb3, 0x87, 0x2d, 0xac, 0xbe, 0x0f, 0x2b, 0x52, 0x6b, 0xef, - 0xc9, 0x1b, 0xe0, 0x82, 0x41, 0x7c, 0xd6, 0xda, 0x6a, 0xc9, 0x71, 0xdf, 0x68, 0xee, 0xec, 0x3c, - 0xdd, 0x69, 0x68, 0xb2, 0xaf, 0xf7, 0xda, 0xdd, 0xc3, 0x8d, 0xf6, 0xde, 0x5e, 0x73, 0xa3, 0xdb, - 0xdc, 0x64, 0xe9, 0x55, 0x03, 0xa0, 0x73, 0x6e, 0xf7, 0x54, 0x8d, 0x71, 0x88, 0xc2, 0x52, 0x87, - 0xe2, 0x0c, 0xf2, 0x2b, 0x0f, 0x49, 0xa8, 0x54, 0x01, 0xd8, 0x96, 0x10, 0x2c, 0xe7, 0x7d, 0x3a, - 0xc9, 0xe1, 0xe3, 0xb1, 0x18, 0x53, 0x17, 0x7b, 0x50, 0x42, 0x28, 0x21, 0x51, 0xb7, 0x04, 0x85, - 0xbd, 0x31, 0x7d, 0x65, 0xe4, 0x2e, 0xdc, 0x0e, 0x41, 0x2d, 0xbb, 0xe7, 0x0c, 0x47, 0xba, 0x6f, - 0x1e, 0x59, 0xe2, 0x40, 0xb8, 0x9e, 0xbc, 0x19, 0xed, 0x25, 0xb8, 0x11, 0x11, 0xc9, 0xa6, 0xaa, - 0xcf, 0x54, 0x51, 0xf7, 0x05, 0x8f, 0xda, 0xa7, 0x48, 0xf1, 0x99, 0x30, 0x58, 0x76, 0xf5, 0x5d, - 0xb8, 0x15, 0xac, 0xf8, 0xd8, 0x59, 0xfb, 0xae, 0x38, 0x36, 0x2d, 0x2b, 0x48, 0xd0, 0x8f, 0x9d, - 0xab, 0xdf, 0x72, 0x9d, 0x61, 0x1c, 0x93, 0xa5, 0xd6, 0x57, 0xff, 0xf5, 0x77, 0xee, 0xa4, 0xbe, - 0xfd, 0x9d, 0x3b, 0xa9, 0xff, 0xf4, 0x9d, 0x3b, 0xa9, 0x9f, 0xf9, 0xee, 0x9d, 0x85, 0x6f, 0x7f, - 0xf7, 0xce, 0xc2, 0x6f, 0x7d, 0xf7, 0xce, 0xc2, 0xa7, 0x6c, 0x74, 0xd2, 0xbf, 0x6f, 0x99, 0x47, - 0xf7, 0x47, 0x47, 0xf7, 0xc9, 0xe0, 0x3a, 0xca, 0x93, 0x09, 0xf5, 0xf0, 0xff, 0x05, 0x00, 0x00, - 0xff, 0xff, 0xf9, 0x36, 0x5a, 0x10, 0x3b, 0x82, 0x00, 0x00, + 0xbf, 0x2e, 0x38, 0x8e, 0xb5, 0xb1, 0xd3, 0xee, 0x34, 0xd9, 0x42, 0xdc, 0x88, 0xb5, 0x02, 0x45, + 0x5c, 0x3f, 0x82, 0x7c, 0x74, 0xf4, 0x65, 0x57, 0x77, 0x4f, 0x0c, 0xb9, 0x23, 0xba, 0x08, 0xc5, + 0x7d, 0xe5, 0x42, 0xc9, 0x57, 0x7d, 0xd4, 0x69, 0xef, 0xc9, 0xe0, 0xfb, 0x66, 0xbb, 0x2b, 0x0f, + 0xd0, 0x74, 0x0e, 0x1e, 0xcb, 0xad, 0xb9, 0xc7, 0x5a, 0x63, 0x7f, 0xfb, 0x90, 0x30, 0x68, 0x6b, + 0xae, 0x61, 0x9f, 0x93, 0x6b, 0x42, 0x90, 0x7c, 0xfd, 0x6f, 0x65, 0x83, 0x75, 0xae, 0xfe, 0x23, + 0x6a, 0xf7, 0x15, 0x20, 0x8f, 0xfa, 0xdd, 0x51, 0xaf, 0x0a, 0x5f, 0x4c, 0x69, 0xe0, 0xcd, 0x33, + 0x19, 0x99, 0x60, 0x69, 0x9e, 0x87, 0xf4, 0xfe, 0x91, 0xcc, 0x2b, 0xdb, 0xf6, 0x87, 0x96, 0x3c, + 0x72, 0xdc, 0x3d, 0xf3, 0x59, 0x0e, 0xff, 0x6c, 0x78, 0xa7, 0x72, 0xe7, 0xaf, 0x7d, 0xe4, 0x99, + 0x74, 0xca, 0xa5, 0x50, 0xff, 0x47, 0x19, 0x28, 0x85, 0xaa, 0xf4, 0x2a, 0xaa, 0x9d, 0x73, 0x58, + 0x6a, 0xed, 0x75, 0x9b, 0xda, 0x5e, 0x63, 0x47, 0xa1, 0x64, 0xf8, 0x35, 0x58, 0xde, 0x6a, 0xed, + 0x34, 0x0f, 0x77, 0xda, 0x8d, 0x4d, 0x05, 0x2c, 0xf2, 0x9b, 0xc0, 0x5b, 0xbb, 0xfb, 0x6d, 0xad, + 0x7b, 0xd8, 0xea, 0x1c, 0x6e, 0x34, 0xf6, 0x36, 0x9a, 0x3b, 0xcd, 0x4d, 0x96, 0xe7, 0xaf, 0xc0, + 0xdd, 0xbd, 0x76, 0xb7, 0xd5, 0xde, 0x3b, 0xdc, 0x6b, 0x1f, 0xb6, 0xd7, 0x3f, 0x6a, 0x6e, 0x74, + 0x3b, 0x87, 0xad, 0xbd, 0x43, 0xe4, 0xfa, 0x58, 0x6b, 0xe0, 0x13, 0x96, 0xe3, 0x77, 0xe1, 0xb6, + 0xc2, 0xea, 0x34, 0xb5, 0x83, 0xa6, 0x86, 0x4c, 0x9e, 0xee, 0x35, 0x0e, 0x1a, 0xad, 0x9d, 0xc6, + 0xfa, 0x4e, 0x93, 0x2d, 0xf2, 0x3b, 0x50, 0x53, 0x18, 0x5a, 0xa3, 0xdb, 0x3c, 0xdc, 0x69, 0xed, + 0xb6, 0xba, 0x87, 0xcd, 0x6f, 0x6c, 0x34, 0x9b, 0x9b, 0xcd, 0x4d, 0x56, 0xe1, 0x5f, 0x82, 0x2f, + 0x52, 0xa5, 0x54, 0x25, 0x92, 0x2f, 0xfb, 0xb4, 0xb5, 0x7f, 0xd8, 0xd0, 0x36, 0xb6, 0x5b, 0x07, + 0x4d, 0xb6, 0xc4, 0x5f, 0x83, 0x2f, 0x5c, 0x8c, 0xba, 0xd9, 0xd2, 0x9a, 0x1b, 0xdd, 0xb6, 0xf6, + 0x09, 0x5b, 0xe1, 0xdf, 0x07, 0x2f, 0x6d, 0x77, 0x77, 0x77, 0x0e, 0x9f, 0x69, 0xed, 0xbd, 0xc7, + 0x87, 0xf4, 0xb7, 0xd3, 0xd5, 0x9e, 0x6e, 0x74, 0x9f, 0x6a, 0x4d, 0x06, 0xbc, 0x06, 0x37, 0xf7, + 0xd7, 0x0f, 0xf7, 0xda, 0xdd, 0xc3, 0xc6, 0xde, 0x27, 0xeb, 0x3b, 0xed, 0x8d, 0x27, 0x87, 0x5b, + 0x6d, 0x6d, 0xb7, 0xd1, 0x65, 0x65, 0xfe, 0x65, 0x78, 0x6d, 0xa3, 0x73, 0xa0, 0xaa, 0xd9, 0xde, + 0x3a, 0xd4, 0xda, 0xcf, 0x3a, 0x87, 0x6d, 0xed, 0x50, 0x6b, 0xee, 0x50, 0x9b, 0x3b, 0x51, 0xdd, + 0x0b, 0xfc, 0x36, 0x54, 0x5b, 0x7b, 0x9d, 0xa7, 0x5b, 0x5b, 0xad, 0x8d, 0x56, 0x73, 0xaf, 0x7b, + 0xb8, 0xdf, 0xd4, 0x76, 0x5b, 0x9d, 0x0e, 0xa2, 0xb1, 0x52, 0xfd, 0x43, 0xc8, 0xb7, 0xec, 0x53, + 0xd3, 0xa7, 0xf9, 0xa7, 0x84, 0x55, 0x79, 0x64, 0x41, 0x91, 0xa6, 0x8d, 0xd9, 0xb7, 0xe9, 0x46, + 0x0e, 0x9a, 0x7d, 0x8b, 0x5a, 0x04, 0xa8, 0xff, 0xaf, 0x2c, 0x54, 0x24, 0x8b, 0xc0, 0xc3, 0xbb, + 0x07, 0xcb, 0x2a, 0xcc, 0xda, 0x4a, 0xaa, 0xb8, 0x49, 0x30, 0x5d, 0x75, 0x27, 0x41, 0x31, 0x45, + 0x17, 0x07, 0xf1, 0x57, 0x61, 0x29, 0x20, 0xea, 0x39, 0xf6, 0x86, 0x69, 0xa8, 0x2d, 0xb7, 0x09, + 0x28, 0xff, 0x61, 0x78, 0x29, 0x06, 0x69, 0xda, 0x3d, 0xf7, 0x7c, 0x14, 0x5e, 0x0a, 0x5a, 0x99, + 0x19, 0x22, 0xd8, 0x32, 0x2d, 0x91, 0x40, 0xd4, 0x2e, 0x66, 0x41, 0x49, 0x2c, 0x3d, 0x0b, 0xdd, + 0x55, 0xb9, 0xbf, 0xab, 0x4a, 0x9f, 0x57, 0xc7, 0xa2, 0xfe, 0x96, 0x88, 0xaa, 0x55, 0xd2, 0x74, + 0x4b, 0xc0, 0xb0, 0x1f, 0xc3, 0xb2, 0x3a, 0xf4, 0x82, 0x4e, 0x59, 0x45, 0x9b, 0x04, 0x87, 0x39, + 0x4e, 0x4f, 0xcf, 0xc8, 0x2c, 0x2b, 0x13, 0x56, 0x1c, 0x14, 0xd6, 0x86, 0x9e, 0x2f, 0xd1, 0xf3, + 0x08, 0xc0, 0x3f, 0x85, 0x5b, 0x21, 0xcb, 0x89, 0xbe, 0x2b, 0xcc, 0xd9, 0x77, 0x17, 0x31, 0xe0, + 0x5f, 0x03, 0x30, 0x49, 0x3c, 0xe8, 0xd5, 0xf2, 0x5c, 0xe3, 0x4b, 0x53, 0x91, 0xd1, 0x00, 0x41, + 0x8b, 0x21, 0xe3, 0x22, 0xd9, 0xc7, 0xb5, 0xe7, 0x89, 0xba, 0x71, 0x75, 0x51, 0x0b, 0xcb, 0xf5, + 0xdf, 0x4d, 0xc5, 0x42, 0x0b, 0x32, 0x74, 0x70, 0xe9, 0xa2, 0x3a, 0x6b, 0x8b, 0x0c, 0x9d, 0x7b, + 0xd5, 0xff, 0xca, 0xd6, 0x53, 0x45, 0xbe, 0x0f, 0xdc, 0x9c, 0xee, 0x8b, 0xec, 0x9c, 0x7d, 0x31, + 0x83, 0x76, 0x72, 0x87, 0x23, 0x37, 0xbd, 0xc3, 0x71, 0x07, 0xa0, 0x6f, 0x39, 0x47, 0x6a, 0xcb, + 0x36, 0xaf, 0x92, 0xcc, 0x42, 0x48, 0xfd, 0xa7, 0x53, 0x70, 0x73, 0xa2, 0xc5, 0xcf, 0x4c, 0x7f, + 0x80, 0x52, 0xb8, 0x0d, 0xcb, 0x66, 0xf2, 0x89, 0x8a, 0xe3, 0x4c, 0xc6, 0xb2, 0x27, 0xe8, 0xb5, + 0x49, 0x32, 0x94, 0x39, 0xe5, 0xbd, 0x06, 0x21, 0x1f, 0x35, 0xe3, 0x27, 0xc1, 0x75, 0x0b, 0x8a, + 0xc1, 0xe5, 0xb4, 0x38, 0x3b, 0xe8, 0x7a, 0xda, 0x30, 0x84, 0x2c, 0x4b, 0x7c, 0x1b, 0x96, 0x44, + 0xb2, 0x0b, 0xd3, 0x73, 0x76, 0xe1, 0x04, 0x5d, 0xfd, 0x6b, 0xb0, 0x32, 0x85, 0x84, 0x63, 0x3a, + 0xd2, 0xfd, 0xf0, 0x56, 0x12, 0xfc, 0x3f, 0x9d, 0x98, 0x51, 0xff, 0xf7, 0x69, 0x58, 0xdc, 0xd5, + 0x6d, 0xf3, 0x58, 0x78, 0x3e, 0xd5, 0xf6, 0x16, 0xe4, 0xbd, 0xde, 0x40, 0x0c, 0xf5, 0xc0, 0xd0, + 0x78, 0x45, 0x16, 0x55, 0x60, 0x29, 0x1d, 0xdf, 0xcc, 0x99, 0xda, 0x1d, 0x44, 0x45, 0x30, 0xf6, + 0x07, 0xe1, 0xa9, 0x14, 0x55, 0x42, 0x59, 0xb2, 0xcc, 0x9e, 0xb0, 0xbd, 0x60, 0xb2, 0x07, 0xc5, + 0x28, 0x51, 0x2b, 0x7f, 0x49, 0xa2, 0x56, 0x61, 0x5a, 0x1e, 0x70, 0x52, 0xf7, 0x5c, 0x21, 0x6c, + 0x6f, 0xe0, 0xf8, 0xc1, 0xcd, 0xc6, 0x71, 0x10, 0x65, 0x9a, 0x3a, 0xcf, 0x6d, 0x54, 0xba, 0x3b, + 0xa6, 0x7d, 0xa2, 0xd2, 0x23, 0x13, 0x30, 0x9c, 0x13, 0x14, 0x56, 0x33, 0x3f, 0x13, 0xa4, 0x3d, + 0x72, 0x5a, 0x58, 0xa6, 0xc0, 0x99, 0xee, 0x8b, 0xbe, 0xe3, 0x9a, 0x42, 0x46, 0x8f, 0x4b, 0x5a, + 0x0c, 0x82, 0xb4, 0x96, 0x6e, 0xf7, 0xc7, 0x7a, 0x5f, 0x28, 0xb5, 0x1b, 0x96, 0xeb, 0xff, 0x3d, + 0x07, 0xb0, 0x2b, 0x86, 0x47, 0xc2, 0xf5, 0x06, 0xe6, 0x88, 0xf6, 0xbd, 0x4c, 0x95, 0x72, 0x5f, + 0xd1, 0xe8, 0x3f, 0x7f, 0x94, 0x38, 0x26, 0x33, 0xbd, 0x97, 0x1d, 0x91, 0x4f, 0x46, 0xdd, 0xb0, + 0x73, 0x74, 0x5f, 0xa8, 0x1c, 0x39, 0xea, 0xff, 0xac, 0x16, 0x07, 0x51, 0xde, 0xa8, 0xee, 0x8b, + 0xa6, 0x6d, 0xc8, 0xa8, 0x5e, 0x56, 0x0b, 0xcb, 0xb4, 0x7f, 0xe7, 0x35, 0xc6, 0xbe, 0xa3, 0x09, + 0x5b, 0x3c, 0x0f, 0x0f, 0x9c, 0x46, 0x20, 0xbe, 0x0b, 0x95, 0x91, 0x7e, 0x3e, 0x14, 0x36, 0x8a, + 0xf3, 0xc0, 0x31, 0x54, 0x42, 0xdb, 0x6b, 0x17, 0x57, 0x70, 0x3f, 0x8e, 0xae, 0x25, 0xa9, 0x51, + 0x26, 0x6c, 0x8f, 0x66, 0xad, 0x1c, 0x46, 0x55, 0xe2, 0xeb, 0x00, 0xf2, 0x5f, 0x4c, 0xf5, 0x4d, + 0x05, 0xfa, 0xf4, 0xa1, 0xf0, 0x84, 0x7b, 0x6a, 0xca, 0x85, 0x41, 0xea, 0xc0, 0x88, 0x0a, 0x15, + 0xf7, 0xd8, 0x13, 0x6e, 0x73, 0xa8, 0x9b, 0x96, 0x1a, 0xe0, 0x08, 0xc0, 0xdf, 0x82, 0x1b, 0xde, + 0xf8, 0x08, 0x65, 0xe6, 0x48, 0x74, 0x9d, 0x3d, 0xf1, 0xdc, 0xb3, 0x84, 0xef, 0x0b, 0x57, 0x25, + 0xcd, 0xcc, 0x7e, 0x58, 0xef, 0x87, 0x96, 0x2e, 0x5d, 0xb9, 0x84, 0xff, 0xa2, 0xcc, 0xbc, 0x10, + 0xa4, 0xd2, 0x16, 0x59, 0x0a, 0x0d, 0x4c, 0x09, 0x52, 0x59, 0x8d, 0x69, 0xfe, 0x45, 0xf8, 0xfe, + 0x04, 0x92, 0x26, 0xf3, 0x06, 0xbc, 0x2d, 0xd3, 0xd6, 0x2d, 0xf3, 0x33, 0x99, 0x04, 0x91, 0xa9, + 0x8f, 0xa0, 0x92, 0xe8, 0x38, 0x3a, 0x21, 0x4d, 0xff, 0x54, 0x6a, 0x17, 0x83, 0x45, 0x59, 0xee, + 0xf8, 0xae, 0x49, 0x9b, 0x5a, 0x21, 0x64, 0x03, 0x27, 0xba, 0xc3, 0xd2, 0xfc, 0x3a, 0x30, 0x09, + 0x69, 0xd9, 0xfa, 0x68, 0xd4, 0x18, 0x8d, 0x2c, 0xc1, 0x32, 0x74, 0xfa, 0x3c, 0x82, 0xca, 0xc3, + 0x32, 0x2c, 0x5b, 0xff, 0x06, 0xdc, 0xa2, 0x9e, 0x39, 0x10, 0x6e, 0x18, 0xcb, 0x50, 0x6d, 0xbd, + 0x01, 0x2b, 0xf2, 0xdf, 0x9e, 0xe3, 0xcb, 0xc7, 0x64, 0xdf, 0x73, 0x58, 0x92, 0x60, 0x34, 0x5f, + 0x3b, 0x82, 0xce, 0x94, 0x87, 0xb0, 0x10, 0x2f, 0x5d, 0xff, 0xcd, 0x3c, 0xf0, 0x48, 0x20, 0xba, + 0xa6, 0x70, 0x37, 0x75, 0x5f, 0x8f, 0x05, 0xa3, 0x2b, 0x17, 0xa6, 0x62, 0xbc, 0x38, 0x29, 0xf3, + 0x26, 0xe4, 0x4d, 0x0f, 0xbd, 0x6f, 0x95, 0xcd, 0xae, 0x4a, 0x7c, 0x07, 0x60, 0x24, 0x5c, 0xd3, + 0x31, 0x48, 0x82, 0x72, 0x33, 0x4f, 0x2b, 0x4d, 0x57, 0x6a, 0x6d, 0x3f, 0xa4, 0xd1, 0x62, 0xf4, + 0x58, 0x0f, 0x59, 0x92, 0x89, 0x0d, 0x79, 0x69, 0x26, 0xc4, 0x40, 0xfc, 0x0d, 0xb8, 0x36, 0x72, + 0xcd, 0x9e, 0x90, 0xc3, 0xf1, 0xd4, 0x33, 0x36, 0xe8, 0xa2, 0xd2, 0x02, 0x61, 0xce, 0x7a, 0x84, + 0x12, 0xa8, 0xdb, 0xe4, 0x93, 0x7a, 0xb4, 0x95, 0xaf, 0x6e, 0x61, 0x90, 0xd9, 0xdc, 0x15, 0x6d, + 0xf6, 0x43, 0xbe, 0x0a, 0x4c, 0x3d, 0xd8, 0x35, 0xed, 0x1d, 0x61, 0xf7, 0xfd, 0x01, 0x09, 0x77, + 0x45, 0x9b, 0x82, 0x93, 0x06, 0x93, 0xd7, 0xc1, 0xc9, 0xad, 0xba, 0x92, 0x16, 0x96, 0xe5, 0x05, + 0x26, 0x96, 0xe3, 0x76, 0x7c, 0x57, 0x25, 0xae, 0x87, 0x65, 0x32, 0x9f, 0xa8, 0xae, 0xfb, 0xae, + 0x63, 0x8c, 0x69, 0x23, 0x49, 0x2a, 0xb1, 0x49, 0x70, 0x84, 0xb9, 0xab, 0xdb, 0x2a, 0x33, 0xb6, + 0x12, 0xc7, 0x0c, 0xc1, 0xe4, 0x76, 0x3b, 0x5e, 0xc4, 0x70, 0x59, 0xb9, 0xdd, 0x31, 0x98, 0xc2, + 0x89, 0x58, 0xb1, 0x10, 0x27, 0xe2, 0x43, 0xed, 0x37, 0x5c, 0xc7, 0x34, 0x22, 0x5e, 0x32, 0x49, + 0x6b, 0x0a, 0x1e, 0xc3, 0x8d, 0x78, 0xf2, 0x04, 0x6e, 0xc4, 0xf7, 0x3a, 0xe4, 0x9c, 0xe3, 0x63, + 0xe1, 0xd2, 0x8d, 0x21, 0x25, 0x4d, 0x16, 0xea, 0x3f, 0x93, 0x02, 0x88, 0x44, 0x02, 0x27, 0x42, + 0x54, 0x8a, 0x26, 0xfe, 0x2d, 0xb8, 0x16, 0x07, 0x5b, 0x2a, 0xe7, 0x99, 0x66, 0x43, 0xf4, 0x60, + 0x53, 0x3f, 0xf7, 0x58, 0x5a, 0xdd, 0x8e, 0xa0, 0x60, 0xcf, 0x84, 0xa0, 0x04, 0xd2, 0xeb, 0xc0, + 0x22, 0x20, 0x1d, 0x79, 0xf5, 0x58, 0x36, 0x89, 0xfa, 0x89, 0xd0, 0x5d, 0x8f, 0xe5, 0xea, 0x3f, + 0x7b, 0x13, 0xe7, 0x79, 0x20, 0xb8, 0x07, 0x0f, 0x6a, 0x5b, 0x90, 0x6f, 0x0c, 0x29, 0x17, 0x04, + 0xc7, 0x94, 0xce, 0xba, 0xf6, 0x42, 0x2b, 0x2e, 0x28, 0x53, 0x82, 0x3c, 0x61, 0x49, 0xb9, 0x94, + 0x7b, 0x3d, 0x71, 0x50, 0xed, 0x19, 0x14, 0x5a, 0xf6, 0xa9, 0x63, 0xf6, 0x84, 0xca, 0xc8, 0x94, + 0xa6, 0x50, 0x56, 0x9e, 0x51, 0xe4, 0x8f, 0x20, 0xe7, 0x3b, 0xbe, 0x6e, 0xa9, 0xdd, 0xcb, 0xfa, + 0x85, 0x73, 0xe9, 0xe0, 0xc1, 0x9a, 0xac, 0x8f, 0x26, 0x09, 0x6a, 0xbf, 0x92, 0x86, 0xe2, 0x56, + 0x20, 0x77, 0x68, 0xbe, 0xcb, 0x9c, 0xfe, 0xf5, 0x73, 0x5f, 0x78, 0xea, 0x15, 0x09, 0x58, 0x68, + 0xe2, 0x6b, 0x94, 0xf6, 0x29, 0x2b, 0x5b, 0xd1, 0x12, 0xb0, 0x10, 0xe7, 0x99, 0x6b, 0xd2, 0xe9, + 0xf0, 0x4c, 0x0c, 0x47, 0xc1, 0x08, 0x67, 0xa0, 0xbb, 0xc2, 0x88, 0x9d, 0x77, 0x41, 0x9c, 0x18, + 0x0c, 0x57, 0x09, 0x5f, 0xe8, 0xc3, 0x8e, 0xd0, 0x7d, 0x79, 0x1a, 0xa8, 0xa2, 0x45, 0x00, 0xe4, + 0xa0, 0x66, 0x95, 0xcc, 0xbe, 0x91, 0x13, 0x3f, 0x01, 0xa3, 0x3b, 0x67, 0xe2, 0x33, 0x4f, 0xcd, + 0xf9, 0x24, 0x50, 0x6e, 0xde, 0x99, 0xa7, 0xb8, 0x10, 0xcb, 0xca, 0xc8, 0x59, 0x9e, 0x04, 0xd6, + 0x7e, 0x21, 0x0b, 0x05, 0x25, 0xbf, 0xb3, 0x12, 0x64, 0x3e, 0x87, 0x7e, 0xa4, 0xf4, 0xcd, 0xae, + 0x33, 0xda, 0x11, 0xa7, 0xc2, 0x52, 0x3a, 0x32, 0x06, 0x51, 0xa7, 0xfe, 0x64, 0xd6, 0x54, 0x2e, + 0x3c, 0xf5, 0x27, 0xf3, 0xa6, 0x68, 0x0b, 0xaf, 0x65, 0xfb, 0xae, 0xa3, 0x12, 0xaa, 0x82, 0x22, + 0xb6, 0xc6, 0xf4, 0x9e, 0x8e, 0xfa, 0xae, 0x6e, 0x08, 0xba, 0x40, 0x59, 0xa6, 0x55, 0x25, 0x81, + 0x7c, 0x0b, 0x16, 0x49, 0xf1, 0x79, 0x28, 0xbb, 0xd6, 0x39, 0x19, 0x62, 0xf3, 0x49, 0x4e, 0x82, + 0x8e, 0x6f, 0x53, 0xdf, 0xf5, 0x84, 0x47, 0x33, 0xc3, 0x3a, 0xa7, 0x1c, 0xab, 0xf9, 0x18, 0x25, + 0x09, 0x13, 0x5a, 0x0f, 0x26, 0xb4, 0x5e, 0xa8, 0x01, 0xca, 0x31, 0x0d, 0xc0, 0x3f, 0x8c, 0xe9, + 0xd0, 0x45, 0x92, 0xfc, 0x57, 0x2e, 0x7b, 0x6d, 0x20, 0xe7, 0x31, 0x4d, 0xfb, 0x11, 0x2c, 0xc9, + 0x4a, 0xec, 0x98, 0xc7, 0xc2, 0x37, 0x87, 0x42, 0x79, 0xd5, 0xf3, 0x54, 0x7f, 0x82, 0xb2, 0xf6, + 0xab, 0x29, 0x58, 0xdc, 0x1f, 0xbb, 0xbd, 0x81, 0xee, 0x49, 0xff, 0x61, 0xc2, 0xde, 0x4b, 0x5d, + 0x6e, 0xef, 0xa5, 0x2f, 0xb7, 0xf7, 0x32, 0xd3, 0xf6, 0xde, 0xbb, 0x90, 0x97, 0xab, 0xdc, 0x05, + 0x3b, 0xc0, 0x89, 0x4a, 0x4b, 0x65, 0xa5, 0x29, 0x8a, 0xda, 0xbf, 0x4a, 0x41, 0x45, 0x09, 0xb3, + 0x32, 0x24, 0xb6, 0x43, 0xbb, 0x56, 0xe6, 0xf9, 0xbc, 0x71, 0x29, 0xb7, 0x38, 0xe9, 0x84, 0x9d, + 0x5b, 0xb7, 0xbe, 0x67, 0x43, 0x6c, 0x15, 0x5e, 0x9d, 0x69, 0x88, 0x35, 0xe4, 0xb4, 0x6d, 0x58, + 0x96, 0xa3, 0x52, 0x52, 0x33, 0xb5, 0xff, 0x91, 0x02, 0x16, 0x74, 0x7b, 0xb0, 0xbe, 0xf0, 0xf7, + 0x28, 0xe5, 0x17, 0xff, 0x2a, 0x97, 0xf1, 0x0b, 0x73, 0xb4, 0x46, 0x0b, 0x68, 0xf8, 0x0e, 0x2c, + 0x8e, 0x62, 0x23, 0xa9, 0xd4, 0xea, 0xbd, 0x4b, 0x79, 0xc4, 0xf0, 0xb5, 0x04, 0x35, 0x6f, 0x53, + 0x6e, 0x40, 0xd4, 0x5f, 0x17, 0x1c, 0x8d, 0xbf, 0xb8, 0x83, 0xb5, 0x24, 0x7d, 0xed, 0x5b, 0x29, + 0x28, 0x6f, 0xe8, 0xae, 0xff, 0x07, 0xd4, 0x5a, 0x52, 0x33, 0x4a, 0x0d, 0xa4, 0x03, 0x35, 0xa3, + 0xa6, 0xf7, 0x45, 0xd7, 0x80, 0x93, 0xea, 0x0a, 0x27, 0x4d, 0xa8, 0xba, 0xc2, 0xc9, 0xf0, 0x77, + 0xd2, 0x74, 0xe1, 0xb1, 0xcf, 0x37, 0xa0, 0xa8, 0xde, 0x13, 0x64, 0xa2, 0xbd, 0x76, 0x59, 0xe5, + 0x62, 0xcd, 0xd2, 0x42, 0xc2, 0xcf, 0xbf, 0xbe, 0xf1, 0x3d, 0x60, 0xf4, 0x67, 0x4f, 0x9c, 0xf9, + 0x6a, 0x05, 0x55, 0xdd, 0x3f, 0x0f, 0x93, 0x29, 0x5a, 0x34, 0xaa, 0xec, 0xa8, 0x48, 0x27, 0x21, + 0xa5, 0xa3, 0x36, 0x09, 0xe6, 0xaf, 0xc3, 0x8a, 0x3e, 0xa2, 0x60, 0xfa, 0xbe, 0xeb, 0x0c, 0x9d, + 0x9e, 0x63, 0x84, 0xdf, 0x0b, 0x9a, 0x7e, 0x50, 0xfb, 0xe9, 0x14, 0x2c, 0x49, 0xdf, 0x80, 0x2e, + 0x0c, 0x70, 0xc6, 0x3e, 0x75, 0xb1, 0xe4, 0xf7, 0x54, 0xdb, 0x51, 0x6b, 0x4d, 0x0c, 0xf2, 0x3d, + 0x74, 0x0a, 0xa9, 0x96, 0x0d, 0xdd, 0xde, 0xd0, 0xed, 0x5e, 0x78, 0x32, 0x35, 0x0e, 0xaa, 0xfd, + 0x46, 0x9a, 0xee, 0x24, 0xd0, 0xf9, 0xf6, 0xd4, 0xf0, 0xbd, 0x3e, 0xcf, 0x2c, 0x30, 0xa6, 0xc7, + 0xb0, 0x09, 0xe5, 0x58, 0x17, 0xa9, 0x4a, 0x5f, 0x2a, 0xa8, 0x0a, 0x55, 0x8b, 0xd3, 0xc9, 0x63, + 0x8d, 0xfa, 0xb0, 0xfd, 0xdc, 0x16, 0x6e, 0x6b, 0x33, 0x58, 0x55, 0x63, 0x20, 0xfe, 0x14, 0x96, + 0x95, 0x23, 0xbb, 0xef, 0x3a, 0xa7, 0xa6, 0x21, 0x5c, 0xa5, 0x1f, 0xbf, 0x7c, 0x69, 0xcd, 0x93, + 0x24, 0xda, 0x24, 0x8f, 0xab, 0x8d, 0x67, 0xbd, 0x0d, 0xc5, 0x7d, 0x4b, 0xf7, 0x8f, 0x1d, 0x77, + 0x98, 0x3c, 0x22, 0x46, 0xc7, 0x91, 0xbc, 0x13, 0x9f, 0x4e, 0xad, 0x57, 0xa0, 0xb4, 0xeb, 0x1c, + 0x99, 0x96, 0x68, 0xb5, 0x3b, 0xf2, 0x92, 0x2f, 0x59, 0x6c, 0x48, 0x0b, 0x58, 0x6e, 0x8f, 0x3c, + 0x13, 0x47, 0x2c, 0x5b, 0x37, 0x60, 0x79, 0xa2, 0x8a, 0xc9, 0xa3, 0x50, 0xa1, 0xab, 0x09, 0x90, + 0x0f, 0x9d, 0xcc, 0x15, 0xa8, 0xac, 0x9b, 0x96, 0x65, 0xda, 0xfd, 0x7d, 0xc7, 0xf5, 0x75, 0x4b, + 0x5e, 0x04, 0xd3, 0x18, 0x8d, 0x3a, 0xbe, 0xe3, 0x0a, 0xb5, 0xe9, 0x42, 0x4e, 0xe6, 0xbe, 0xa5, + 0x9f, 0xb3, 0x5c, 0x5d, 0x83, 0xbc, 0x5c, 0x28, 0xf8, 0x0a, 0x94, 0x22, 0x6b, 0x79, 0xa1, 0x96, + 0x2e, 0x52, 0x6e, 0xac, 0x5a, 0xab, 0xe5, 0x6b, 0xa4, 0x8a, 0x60, 0x69, 0x3a, 0x55, 0x35, 0x70, + 0x85, 0x32, 0x83, 0xe5, 0x65, 0x33, 0x4a, 0x11, 0xb0, 0x6c, 0x7d, 0x1b, 0xf2, 0x32, 0xf5, 0x6e, + 0x46, 0xd2, 0xec, 0xd5, 0xb2, 0xe7, 0xff, 0x5c, 0x0a, 0x60, 0x53, 0x9e, 0xa3, 0x45, 0xbd, 0x3a, + 0x8f, 0x11, 0x26, 0xaf, 0xd5, 0xa5, 0x79, 0x9a, 0x09, 0xaf, 0xd5, 0xa5, 0xf9, 0x59, 0x83, 0xa2, + 0x1e, 0x9c, 0x9c, 0x91, 0xfa, 0x2b, 0x2c, 0xab, 0x09, 0xe2, 0xd8, 0xb6, 0xe8, 0xe1, 0xca, 0x1d, + 0xc6, 0x5a, 0x42, 0x50, 0xfd, 0xdf, 0xa6, 0xa1, 0xb4, 0x31, 0xd0, 0x7d, 0x79, 0x0b, 0xed, 0x87, + 0x50, 0x54, 0x5f, 0xde, 0xf1, 0x94, 0x06, 0x7e, 0x65, 0xc6, 0x57, 0x7a, 0x08, 0x77, 0xed, 0xa9, + 0xed, 0x0a, 0xdd, 0x90, 0x57, 0xef, 0x86, 0x54, 0x92, 0x83, 0xed, 0x87, 0x5b, 0x81, 0x57, 0xe0, + 0x60, 0x87, 0x1f, 0xd8, 0xb1, 0x74, 0x4f, 0xa2, 0x84, 0xdb, 0xfc, 0x71, 0x10, 0x19, 0x51, 0x6e, + 0x30, 0x1d, 0x32, 0x9a, 0x2c, 0xa0, 0xab, 0x3b, 0x26, 0x86, 0x9a, 0xd0, 0x69, 0xbf, 0xaf, 0xad, + 0xb2, 0xb3, 0x65, 0xc0, 0x6f, 0xf6, 0xc3, 0xda, 0x2e, 0x94, 0x63, 0xd5, 0x40, 0x9b, 0xd3, 0xb1, + 0x0c, 0xe1, 0xf9, 0x01, 0xb1, 0xfa, 0x68, 0x42, 0x02, 0x48, 0x89, 0xeb, 0xa8, 0x87, 0x84, 0xab, + 0x32, 0x10, 0x83, 0x62, 0xfd, 0xaf, 0x5e, 0x83, 0x72, 0xec, 0x43, 0x46, 0x53, 0x43, 0x1b, 0x4b, + 0x1a, 0x4f, 0x27, 0x92, 0xc6, 0xe3, 0xc9, 0xf0, 0x99, 0x64, 0x32, 0x7c, 0xe2, 0x24, 0x6e, 0x76, + 0xf2, 0x24, 0xee, 0x1d, 0x80, 0xa1, 0x63, 0x50, 0x18, 0xa4, 0x21, 0x33, 0xcd, 0x32, 0x5a, 0x0c, + 0x42, 0x5b, 0x20, 0xaa, 0x2b, 0xcb, 0x6a, 0x0b, 0x44, 0x75, 0x23, 0x9d, 0x4a, 0x18, 0x59, 0xe7, + 0x5d, 0x67, 0x37, 0xfc, 0x54, 0x54, 0x78, 0xc3, 0x58, 0x12, 0xce, 0x37, 0x26, 0xbf, 0xde, 0xf4, + 0xa5, 0x8b, 0xbf, 0xde, 0x14, 0x7c, 0x13, 0x4b, 0x5d, 0xcf, 0x11, 0x7e, 0xcc, 0x89, 0x3f, 0x86, + 0xb2, 0xee, 0xfb, 0x7a, 0x6f, 0x30, 0x54, 0x61, 0x8b, 0xcc, 0x8c, 0xd4, 0xda, 0x38, 0xa3, 0x46, + 0x88, 0xad, 0xc5, 0x29, 0xf9, 0x3a, 0x94, 0x5c, 0x35, 0x8e, 0x41, 0x76, 0xef, 0x2b, 0x97, 0xb0, + 0x09, 0xc6, 0xdc, 0xd3, 0x22, 0xb2, 0xf0, 0xfb, 0x21, 0x10, 0xfb, 0x7e, 0x08, 0x1d, 0xbb, 0x26, + 0x31, 0x44, 0x0f, 0x50, 0x5d, 0x8f, 0x1a, 0x07, 0x61, 0x6f, 0x0f, 0x74, 0x4f, 0xdd, 0x48, 0xae, + 0x8e, 0x87, 0xc5, 0x20, 0x94, 0x4d, 0x7b, 0x6e, 0xf7, 0x54, 0x22, 0x5c, 0x51, 0x53, 0x25, 0x84, + 0x8f, 0x4c, 0xdb, 0x16, 0x86, 0x3a, 0xf7, 0xa1, 0x4a, 0xfc, 0x55, 0x58, 0x4a, 0xca, 0x25, 0xc5, + 0x23, 0x8a, 0xda, 0x04, 0x94, 0x7f, 0x10, 0x66, 0x7b, 0xaf, 0xcc, 0xb4, 0x3d, 0x66, 0x74, 0x7f, + 0x22, 0x01, 0xbc, 0xf6, 0x4b, 0x29, 0x58, 0x4a, 0x8e, 0xcb, 0x1f, 0xc6, 0x55, 0xfd, 0x5f, 0x8f, + 0xae, 0xea, 0xff, 0x1c, 0xd7, 0xde, 0xff, 0x62, 0x0a, 0x20, 0x1a, 0x72, 0xec, 0x33, 0x79, 0x33, + 0x78, 0xb0, 0x6d, 0x21, 0x4b, 0x7c, 0x3b, 0x71, 0xcf, 0xe2, 0x5b, 0x73, 0xc9, 0x4f, 0xec, 0x6f, + 0xec, 0xec, 0xf3, 0x7d, 0x58, 0x4a, 0xc2, 0xe9, 0xcc, 0x78, 0x6b, 0xa7, 0x29, 0x13, 0x0b, 0x5a, + 0xbb, 0x8d, 0xc7, 0x4d, 0x75, 0xbf, 0x4b, 0x6b, 0xef, 0x09, 0x4b, 0xd7, 0xfe, 0x67, 0x0a, 0x4a, + 0xa1, 0x34, 0xf1, 0x8f, 0xe3, 0x62, 0x28, 0x8d, 0x8a, 0x87, 0xf3, 0x88, 0x61, 0xf4, 0xaf, 0x69, + 0xfb, 0xee, 0x79, 0x4c, 0x2a, 0x6b, 0x0e, 0x2c, 0x25, 0x1f, 0xce, 0x58, 0x6b, 0x1e, 0x27, 0xd7, + 0x9a, 0x37, 0xe7, 0x7a, 0x65, 0xb0, 0xb5, 0xb4, 0x63, 0x7a, 0xbe, 0x5a, 0x86, 0xde, 0x4d, 0x3f, + 0x4a, 0xd5, 0xee, 0xc2, 0x62, 0xfc, 0xd1, 0x8c, 0xbb, 0x9f, 0x7e, 0x3e, 0x03, 0x8b, 0x71, 0x91, + 0xe2, 0x8d, 0x98, 0xdc, 0x94, 0xa7, 0x8c, 0x91, 0x8b, 0x24, 0x31, 0xf1, 0xe1, 0xaf, 0x86, 0xfa, + 0x98, 0x55, 0xfa, 0x4a, 0x2c, 0x12, 0x9f, 0xb5, 0xda, 0x84, 0x1c, 0x9a, 0x3d, 0x86, 0xb2, 0x82, + 0x5f, 0x9f, 0x93, 0x07, 0x5d, 0xbc, 0xbe, 0xbd, 0xa0, 0x49, 0x62, 0x7e, 0x00, 0x65, 0x79, 0x48, + 0x83, 0xee, 0xfc, 0x57, 0x87, 0xc9, 0x1e, 0xcc, 0xcb, 0x2b, 0xa2, 0xdc, 0x5e, 0xd0, 0xe2, 0x8c, + 0xf8, 0x27, 0xb0, 0xa8, 0xb4, 0x9e, 0x64, 0x2c, 0xaf, 0x80, 0x79, 0x38, 0x27, 0xe3, 0xdd, 0x18, + 0xe9, 0xf6, 0x82, 0x96, 0x60, 0x15, 0xbb, 0x34, 0xa7, 0xf6, 0xef, 0x52, 0xc0, 0x26, 0xfb, 0xf8, + 0x8f, 0xdc, 0xb4, 0xbe, 0xe4, 0x03, 0x1c, 0x1c, 0xb2, 0x96, 0x6e, 0xf7, 0x83, 0x94, 0x49, 0xfc, + 0x5f, 0xfb, 0xa7, 0x13, 0x6d, 0xa2, 0x6d, 0xb2, 0xe9, 0x4f, 0x07, 0xa4, 0x66, 0x7e, 0x3a, 0x60, + 0x7e, 0xd5, 0x30, 0xf9, 0x0a, 0xba, 0x1e, 0x21, 0xa6, 0x1a, 0xbe, 0x86, 0xa6, 0x9f, 0x84, 0xc4, + 0xbe, 0x35, 0x71, 0xc1, 0xc7, 0x00, 0x16, 0xa3, 0x6f, 0xd3, 0xb0, 0x4c, 0xcd, 0x83, 0x95, 0x29, + 0x89, 0xfb, 0xc3, 0xbe, 0xeb, 0xac, 0xf6, 0x19, 0xdc, 0xba, 0x40, 0x34, 0x2f, 0x39, 0x04, 0xd1, + 0x0c, 0x45, 0xe9, 0x8a, 0x33, 0x11, 0xc7, 0x58, 0x0b, 0xc5, 0xf0, 0x97, 0x53, 0x50, 0xbd, 0x48, + 0x7c, 0x93, 0x5f, 0xa2, 0x4c, 0x4d, 0x7e, 0x89, 0x72, 0xae, 0x63, 0x18, 0xf1, 0x7a, 0x66, 0x3e, + 0x7f, 0x3d, 0xeb, 0xbf, 0x9c, 0x86, 0x32, 0x5d, 0x34, 0x3b, 0x40, 0x73, 0x6e, 0xea, 0x8c, 0x5f, + 0x6a, 0xfa, 0x8c, 0xdf, 0x23, 0xc8, 0x93, 0xaa, 0xbc, 0x68, 0x67, 0x3c, 0xc6, 0x4d, 0x59, 0xf8, + 0x0a, 0xbf, 0xf6, 0xcd, 0xe0, 0xcc, 0x58, 0x70, 0x29, 0x47, 0x6a, 0x66, 0xae, 0x6b, 0x9c, 0x41, + 0xec, 0xec, 0xd4, 0xd5, 0xdc, 0x89, 0x3d, 0x95, 0x44, 0x76, 0x1d, 0x58, 0x8c, 0x9f, 0xbc, 0x71, + 0x79, 0x61, 0x02, 0x2a, 0x6f, 0xd5, 0x4c, 0x51, 0x02, 0x53, 0x04, 0x55, 0xb7, 0x57, 0x3e, 0xf5, + 0x84, 0xcb, 0xd2, 0xab, 0xff, 0x32, 0x0b, 0x4b, 0xc9, 0x23, 0x5d, 0x74, 0x49, 0x98, 0x3c, 0x4e, + 0xd8, 0xb6, 0x8c, 0xd8, 0x05, 0x21, 0x8c, 0x2f, 0x43, 0x59, 0x65, 0x27, 0x10, 0x60, 0x85, 0x52, + 0xd3, 0x9c, 0xa1, 0x60, 0x77, 0xe3, 0x1f, 0xa0, 0x7a, 0x03, 0x27, 0x91, 0xbc, 0xa7, 0x8d, 0x8d, + 0x78, 0x49, 0x4d, 0xa2, 0x1f, 0x4f, 0xf3, 0x4a, 0xec, 0x9a, 0x8a, 0x6f, 0xa5, 0xf9, 0x75, 0x58, + 0x5e, 0x1f, 0xdb, 0x86, 0x25, 0x8c, 0x10, 0xfa, 0x4b, 0x71, 0x68, 0x78, 0xcf, 0xc4, 0x8f, 0xa3, + 0xcf, 0x57, 0xea, 0x8c, 0x8f, 0xd4, 0xcc, 0xfc, 0x93, 0x59, 0x7e, 0x13, 0x56, 0x14, 0x56, 0x74, + 0x0c, 0x9b, 0xfd, 0xa9, 0x2c, 0xbf, 0x06, 0x4b, 0x0d, 0xd9, 0xe9, 0xaa, 0xa2, 0xec, 0x4f, 0x67, + 0xb1, 0x0a, 0x74, 0xf3, 0xe8, 0x9f, 0x21, 0x3e, 0xe1, 0x1d, 0x4a, 0xec, 0x27, 0xb2, 0x7c, 0x19, + 0xa0, 0xd3, 0x0d, 0x5f, 0xf4, 0x53, 0x59, 0x5e, 0x86, 0x7c, 0xa7, 0x4b, 0xdc, 0x7e, 0x26, 0xcb, + 0x6f, 0x00, 0x8b, 0x9e, 0xaa, 0x83, 0xee, 0x7f, 0x41, 0x56, 0x26, 0x3c, 0xb9, 0xfe, 0xb3, 0x59, + 0x6c, 0x57, 0xb0, 0xae, 0xb2, 0xbf, 0x98, 0xe5, 0x0c, 0xca, 0xb1, 0x14, 0x48, 0xf6, 0x97, 0xb2, + 0x9c, 0x43, 0x65, 0x37, 0x71, 0x02, 0xfd, 0x27, 0xe9, 0xcd, 0x5b, 0xe1, 0x35, 0x50, 0xec, 0xe7, + 0xb2, 0xfc, 0x16, 0xf0, 0x78, 0xda, 0xb7, 0x7a, 0xf0, 0x97, 0x89, 0x5a, 0x3a, 0x90, 0x9e, 0x82, + 0xfd, 0x7c, 0x96, 0xbf, 0x04, 0xd7, 0x71, 0x1e, 0x48, 0x40, 0xec, 0x64, 0xfc, 0x5f, 0xa1, 0xae, + 0xd9, 0x88, 0x8e, 0xc6, 0x2b, 0x92, 0x6f, 0x11, 0x9b, 0x60, 0x58, 0x25, 0xec, 0x97, 0xa8, 0x81, + 0x9b, 0xe1, 0xd1, 0x77, 0x05, 0xfe, 0xeb, 0x59, 0x1c, 0x83, 0x6e, 0x70, 0xe7, 0x92, 0x82, 0xfe, + 0x8d, 0x2c, 0x7f, 0x19, 0x6e, 0x86, 0xd0, 0x03, 0xd3, 0xf5, 0xc7, 0xba, 0xa5, 0x1e, 0xfe, 0xcd, + 0xec, 0xea, 0x6f, 0x50, 0x2a, 0x70, 0xfc, 0xb4, 0x28, 0x6a, 0x4a, 0xcb, 0xb1, 0xfb, 0xbe, 0xfc, + 0x98, 0x58, 0x05, 0x4a, 0xde, 0xc0, 0x71, 0x7d, 0x2a, 0x92, 0x2b, 0x6e, 0xd3, 0x4d, 0xab, 0xf2, + 0xe6, 0x13, 0x19, 0x85, 0x95, 0xb1, 0x03, 0x5f, 0xef, 0xb3, 0x72, 0x78, 0x51, 0x40, 0x36, 0xbc, + 0xcc, 0x80, 0x6e, 0x7c, 0x0d, 0x2e, 0xc9, 0x64, 0x79, 0x44, 0x1d, 0xbb, 0x96, 0xbc, 0xd4, 0x40, + 0x0c, 0x75, 0xd3, 0x92, 0x5f, 0x0d, 0x1a, 0x0d, 0x1c, 0x5b, 0xdd, 0x6a, 0x20, 0xe8, 0x03, 0x42, + 0x10, 0x3b, 0x9b, 0x6b, 0x60, 0x3d, 0xc2, 0xe3, 0x67, 0x8c, 0x2e, 0x97, 0x1e, 0xea, 0x23, 0x76, + 0xbc, 0xfa, 0xd7, 0x52, 0xb0, 0x18, 0xdc, 0x61, 0x6a, 0xf6, 0x4d, 0x5b, 0xde, 0x8f, 0x10, 0x7c, + 0xab, 0xad, 0x67, 0x99, 0xa3, 0xe0, 0xdb, 0x47, 0xcb, 0x50, 0x36, 0x5c, 0xbd, 0xdf, 0xb0, 0x8d, + 0x4d, 0xd7, 0x19, 0xc9, 0xfa, 0xcb, 0x53, 0x03, 0xf2, 0x5e, 0x86, 0xe7, 0xe2, 0x08, 0xd1, 0x47, + 0xc2, 0x65, 0x59, 0x3a, 0x00, 0x3c, 0xd0, 0x5d, 0xd3, 0xee, 0x37, 0xcf, 0x7c, 0x61, 0x7b, 0xf2, + 0x7e, 0x86, 0x32, 0x14, 0xc6, 0x9e, 0xe8, 0xe9, 0x9e, 0x60, 0x79, 0x2c, 0x1c, 0x8d, 0x4d, 0xcb, + 0x37, 0x6d, 0xf9, 0xc9, 0xa1, 0xf0, 0x02, 0x86, 0x22, 0xd6, 0x4e, 0x1f, 0x99, 0xac, 0xb4, 0xfa, + 0x4f, 0x52, 0x50, 0xa6, 0x8e, 0x8f, 0xf2, 0x62, 0x13, 0xe1, 0x99, 0x9d, 0xf0, 0x13, 0x32, 0x79, + 0x48, 0xb7, 0x4f, 0x64, 0x5e, 0xac, 0x92, 0x39, 0x79, 0xa7, 0xa0, 0xfc, 0x9a, 0x0c, 0x4a, 0xce, + 0x0d, 0x4d, 0x0c, 0x1d, 0x5f, 0x3c, 0xd3, 0x4d, 0x3f, 0x7e, 0x17, 0x52, 0x8e, 0xaf, 0x40, 0x45, + 0x3e, 0x0a, 0x2e, 0x3f, 0xca, 0x53, 0x98, 0x1b, 0x5f, 0x1b, 0x40, 0x0a, 0xd8, 0x7a, 0x82, 0xa8, + 0xb8, 0x77, 0x31, 0x44, 0xf9, 0xc8, 0x31, 0x6d, 0x7c, 0x1b, 0x5d, 0x80, 0xd9, 0x91, 0x5b, 0x6b, + 0x43, 0xe7, 0x14, 0x41, 0xb0, 0xfa, 0x09, 0xdc, 0x9c, 0x9d, 0x16, 0x2c, 0xaf, 0xc6, 0xa4, 0x0f, + 0x1e, 0x52, 0x48, 0x48, 0x6e, 0xb5, 0xc9, 0x25, 0x97, 0x82, 0x63, 0x32, 0x22, 0xb4, 0xe7, 0xc4, + 0x68, 0xd4, 0x77, 0x78, 0x8c, 0xa1, 0x69, 0xb3, 0xec, 0xea, 0x3b, 0x00, 0x51, 0xd2, 0x9a, 0xfc, + 0x50, 0x02, 0xc9, 0x15, 0x99, 0xf5, 0x8f, 0xc7, 0xc2, 0x53, 0x69, 0x04, 0xcf, 0x4c, 0x7f, 0xe0, + 0x8c, 0x83, 0xec, 0x79, 0x96, 0x5e, 0xed, 0x25, 0xb2, 0xc1, 0xa3, 0x8e, 0x0d, 0x1a, 0xb2, 0x10, + 0xbb, 0x3d, 0x2a, 0x25, 0xf3, 0x8c, 0xe9, 0x53, 0xe6, 0xd2, 0x08, 0x50, 0x59, 0xd8, 0x86, 0x0c, + 0x1d, 0x85, 0x4d, 0xcd, 0xca, 0xcf, 0x3a, 0xd8, 0x3d, 0x61, 0x09, 0x83, 0xe5, 0x56, 0x1f, 0xc1, + 0xb2, 0xea, 0x2e, 0x5c, 0xbc, 0x83, 0xdb, 0x97, 0xf6, 0xe5, 0x26, 0x9d, 0xca, 0x35, 0x16, 0xae, + 0xe7, 0xd8, 0x74, 0x0f, 0x34, 0x40, 0xbe, 0x43, 0x9b, 0x89, 0x2c, 0xbd, 0xda, 0x54, 0x1d, 0xad, + 0xd2, 0x06, 0x13, 0xdf, 0x28, 0xda, 0xd4, 0x7d, 0x5d, 0xa1, 0xfb, 0xae, 0xd0, 0xd5, 0x05, 0x92, + 0x38, 0xed, 0x65, 0x75, 0xda, 0xb6, 0xe8, 0x3a, 0x6d, 0x5b, 0xb0, 0xec, 0xaa, 0xaf, 0x94, 0x55, + 0xb0, 0x6a, 0x84, 0x85, 0x48, 0x7e, 0xe2, 0xd0, 0xf0, 0xab, 0x11, 0xe1, 0x28, 0x22, 0x14, 0xe7, + 0xbd, 0x1c, 0x88, 0x10, 0xa4, 0x5e, 0x76, 0x43, 0xdd, 0xab, 0x83, 0xa0, 0xd8, 0x5b, 0x37, 0xa0, + 0x44, 0x06, 0xd3, 0x13, 0xd3, 0x36, 0x70, 0x1c, 0xd6, 0xd5, 0xb5, 0x26, 0xf4, 0xb5, 0x80, 0x53, + 0x1a, 0xe0, 0xa2, 0xfc, 0xee, 0x1c, 0x4b, 0xf3, 0x9b, 0xc0, 0x1b, 0x63, 0xdf, 0x19, 0xea, 0x74, + 0x6d, 0xa4, 0x75, 0x2e, 0xbf, 0x51, 0x98, 0x59, 0xfd, 0x00, 0xb8, 0xcc, 0x6b, 0x33, 0xc4, 0x99, + 0x69, 0xf7, 0xc3, 0x8b, 0x6c, 0x81, 0xae, 0xb0, 0x36, 0xc4, 0x59, 0x70, 0x0f, 0x58, 0x50, 0x08, + 0x2e, 0xd2, 0xde, 0x72, 0xc6, 0x36, 0x76, 0xe1, 0x01, 0x5c, 0x97, 0x93, 0x06, 0xfb, 0x94, 0xee, + 0x24, 0xbc, 0x70, 0x8b, 0x47, 0xde, 0x03, 0xe6, 0x8f, 0xbd, 0x10, 0x97, 0xa5, 0xb0, 0x62, 0x61, + 0x9e, 0x4a, 0x04, 0x4f, 0xaf, 0xd6, 0xe1, 0xda, 0x8c, 0x64, 0x21, 0x5a, 0x03, 0xe5, 0x66, 0x0f, + 0x5b, 0x58, 0x7d, 0x1f, 0x56, 0xa4, 0xd6, 0xde, 0x93, 0x77, 0xc2, 0x05, 0x83, 0xf8, 0xac, 0xb5, + 0xd5, 0x92, 0xe3, 0xbe, 0xd1, 0xdc, 0xd9, 0x79, 0xba, 0xd3, 0xd0, 0x64, 0x5f, 0xef, 0xb5, 0xbb, + 0x87, 0x1b, 0xed, 0xbd, 0xbd, 0xe6, 0x46, 0xb7, 0xb9, 0xc9, 0xd2, 0xab, 0x06, 0x40, 0xe7, 0xdc, + 0xee, 0xa9, 0x1a, 0xe3, 0x10, 0x85, 0xa5, 0x0e, 0xc5, 0x19, 0xe4, 0x77, 0x1f, 0x92, 0x50, 0xa9, + 0x02, 0xb0, 0x2d, 0x21, 0x58, 0xce, 0xfb, 0x74, 0x92, 0xc3, 0xc7, 0x63, 0x31, 0xa6, 0x2e, 0xf6, + 0xa0, 0x84, 0x50, 0x42, 0xa2, 0x6e, 0x09, 0x0a, 0x7b, 0x63, 0xfa, 0xee, 0xc8, 0x5d, 0xb8, 0x1d, + 0x82, 0x5a, 0x76, 0xcf, 0x19, 0x8e, 0x74, 0xdf, 0x3c, 0xb2, 0xc4, 0x81, 0x70, 0x3d, 0x79, 0x57, + 0xda, 0x4b, 0x70, 0x23, 0x22, 0x92, 0x4d, 0x55, 0x1f, 0xae, 0xa2, 0xee, 0x0b, 0x1e, 0xb5, 0x4f, + 0x91, 0xe2, 0x33, 0x61, 0xb0, 0xec, 0xea, 0xbb, 0x70, 0x2b, 0x58, 0xf1, 0xb1, 0xb3, 0xf6, 0x5d, + 0x71, 0x6c, 0x5a, 0x56, 0x90, 0xb2, 0x1f, 0x3b, 0x69, 0xbf, 0xe5, 0x3a, 0xc3, 0x38, 0x26, 0x4b, + 0xad, 0xaf, 0xfe, 0xeb, 0xef, 0xdc, 0x49, 0x7d, 0xfb, 0x3b, 0x77, 0x52, 0xff, 0xe9, 0x3b, 0x77, + 0x52, 0x3f, 0xf3, 0xdd, 0x3b, 0x0b, 0xdf, 0xfe, 0xee, 0x9d, 0x85, 0xdf, 0xfa, 0xee, 0x9d, 0x85, + 0x4f, 0xd9, 0xe8, 0xa4, 0x7f, 0xdf, 0x32, 0x8f, 0xee, 0x8f, 0x8e, 0xee, 0x93, 0xc1, 0x75, 0x94, + 0x27, 0x13, 0xea, 0xe1, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xaa, 0x9a, 0x02, 0x91, 0x4d, 0x82, + 0x00, 0x00, } func (m *SmartBlockSnapshotBase) Marshal() (dAtA []byte, err error) { diff --git a/pkg/lib/pb/model/protos/models.proto b/pkg/lib/pb/model/protos/models.proto index ac0daf1ffb..f766dd66d3 100644 --- a/pkg/lib/pb/model/protos/models.proto +++ b/pkg/lib/pb/model/protos/models.proto @@ -1231,6 +1231,11 @@ message Export { DOT = 3; SVG = 4; GRAPH_JSON = 5; + // AnyBlockJSON is the native AnyBlock JSON bundle (pkg/lib/anyblockjson + // SPEC.md): a directory of `.anyblock.json` documents beside an + // index.json and properties.json. Additive — existing values keep + // their numbers, so a client that does not know it is unaffected. + AnyBlockJSON = 6; } } diff --git a/space/internal/components/migration/systemobjectreviser/systemobjectreviser.go b/space/internal/components/migration/systemobjectreviser/systemobjectreviser.go index d474101de5..de742d3692 100644 --- a/space/internal/components/migration/systemobjectreviser/systemobjectreviser.go +++ b/space/internal/components/migration/systemobjectreviser/systemobjectreviser.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/commonspace/object/acl/list" "github.com/samber/lo" "go.uber.org/zap" @@ -48,11 +49,34 @@ var ( bundle.RelationKeyIconName, bundle.RelationKeyPluralName, } + + // nonSystemRelationFilterKeys lists the only details the reviser is allowed to touch on + // bundled NON-system relations. Kept deliberately narrow: non-system relations are fully + // user-modifiable, so the reviser only records the bundle revision and — guarded by + // previousBundledRelationNames — propagates bundled renames. + nonSystemRelationFilterKeys = []domain.RelationKey{ + bundle.RelationKeyRevision, + bundle.RelationKeyName, + } ) +// previousBundledRelationNames maps a bundled NON-system relation key to every display name +// it had in earlier releases. The reviser applies the current bundled name to an installed +// relation only if its local name still equals one of these previous bundled names; any other +// local name is a user's own rename and is kept. Whenever a bundled non-system relation is +// renamed, append its OLD name here and bump the relation's revision in relations.json — +// without both, the rename never reaches existing spaces. System relations do not need +// entries: users cannot rename them, so the system path applies names unconditionally. +var previousBundledRelationNames = map[domain.RelationKey][]string{ + bundle.RelationKeyAudioGenre: {"Genre"}, + bundle.RelationKeyHeaderRelationsLayout: {"Header relations layout"}, +} + // Migration SystemObjectReviser performs revision of all system object types and relations, so after Migration // objects installed in space should correspond to bundled objects from library. // To modify relations of system objects relation revision should be incremented in types.json or relations.json +// Bundled non-system relations are also reachable, but only for revision and name +// (see nonSystemRelationFilterKeys and previousBundledRelationNames). // For more info see 'System Objects Update' section of docs/Flow.md type Migration struct{} @@ -73,6 +97,25 @@ func (m Migration) Run(ctx context.Context, log logger.CtxLogger, store dependen } toMigrate++ if e != nil { + // A space this account may only READ answers the same way for + // every object in it — the verdict is the ACL's, not the + // object's — so one refusal ends the pass. Without this the + // migration attempted every pending object on every space load, + // logged an error for each, and left each one carrying the new + // value in memory until it was evicted: an unwritable change is + // applied to the loaded document before the push is refused, so + // a reader of a shared space saw a value that would never + // persist. Measured on a subscribed space: twelve failed writes + // and thirteen documents whose two consecutive exports differed. + // + // Not an error to report. A reader cannot revise the system + // objects of a space they do not own, and saying so once per + // load is the whole of what is true. + if errors.Is(e, list.ErrInsufficientPermissions) { + log.Debug("skipping system object revision: this account may only read the space", + zap.String("space", space.Id())) + return toMigrate, migrated, nil + } err = errors.Join(err, fmt.Errorf("failed to revise object: %w", e)) } else { migrated++ @@ -122,15 +165,19 @@ func reviseObject(ctx context.Context, log logger.CtxLogger, space dependencies. if bundleObject == nil { return false, nil } - details := buildDiffDetails(bundleObject, localObject, isSystem) + details := buildDiffDetails(bundleObject, localObject, uk, isSystem) - recRelsDetails, err := checkRecommendedRelations(ctx, space, bundleObject, localObject, uk) - if err != nil { - log.Error("failed to check recommended relations", zap.Error(err)) - } + // non-system relations are user-modifiable, so only the narrow filtered diff + // (revision + guarded name) may be applied to them + if isSystem || uk.SmartblockType() != coresb.SmartBlockTypeRelation { + recRelsDetails, err := checkRecommendedRelations(ctx, space, bundleObject, localObject, uk) + if err != nil { + log.Error("failed to check recommended relations", zap.Error(err)) + } - for _, recRelsDetail := range recRelsDetails { - details.Set(recRelsDetail.Key, recRelsDetail.Value) + for _, recRelsDetail := range recRelsDetails { + details.Set(recRelsDetail.Key, recRelsDetail.Value) + } } if isSystem { @@ -161,8 +208,10 @@ func reviseObject(ctx context.Context, log logger.CtxLogger, space dependencies. } // getBundleObjectRevision returns the revision of the bundled counterpart without building -// its details. Mirrors getBundleObjectDetails: ok is false for non-bundled types and -// non-system relations, which are not revisable. +// its details. Mirrors getBundleObjectDetails: ok is false for non-bundled objects, which +// are not revisable. Bundled non-system relations answer too, so that a bundled rename with +// a revision bump reaches them; almost all of them have revision 0, so the caller's +// revision guard short-circuits before any details are built. func getBundleObjectRevision(uk domain.UniqueKey) (revision int64, ok bool) { switch uk.SmartblockType() { case coresb.SmartBlockTypeObjectType: @@ -172,16 +221,17 @@ func getBundleObjectRevision(uk domain.UniqueKey) (revision int64, ok bool) { } return objectType.Revision, true case coresb.SmartBlockTypeRelation: - if !isSystemRelation(uk) { + relation, err := bundle.GetRelation(domain.RelationKey(uk.InternalKey())) + if err != nil { return 0, false } - return bundle.MustGetRelation(domain.RelationKey(uk.InternalKey())).Revision, true + return relation.Revision, true default: return 0, false } } -// getBundleObjectDetails returns nil if the object with provided unique key is not either system relation or bundled type +// getBundleObjectDetails returns nil if the object with provided unique key is not a bundled type or relation func getBundleObjectDetails(uk domain.UniqueKey) (details *domain.Details, isSystem bool) { switch uk.SmartblockType() { case coresb.SmartBlockTypeObjectType: @@ -193,34 +243,52 @@ func getBundleObjectDetails(uk domain.UniqueKey) (details *domain.Details, isSys } return (&relationutils.ObjectType{ObjectType: objectType}).BundledTypeDetails(), isSystemType(uk) case coresb.SmartBlockTypeRelation: - if !isSystemRelation(uk) { - // non system relation, no need to revise + relation, err := bundle.GetRelation(domain.RelationKey(uk.InternalKey())) + if err != nil { + // not bundled relation, no need to revise return nil, false } - relationKey := domain.RelationKey(uk.InternalKey()) - relation := bundle.MustGetRelation(relationKey) - return (&relationutils.Relation{Relation: relation}).ToDetails(), true + return (&relationutils.Relation{Relation: relation}).ToDetails(), isSystemRelation(uk) default: return nil, false } } -func buildDiffDetails(origin, current *domain.Details, isSystem bool) *domain.Details { - // non-system bundled types are going to update only icons and plural names for now - filterKeys := customObjectFilterKeys - if isSystem { - filterKeys = systemObjectFilterKeys +func buildDiffDetails(origin, current *domain.Details, uk domain.UniqueKey, isSystem bool) *domain.Details { + isNonSystemRelation := !isSystem && uk.SmartblockType() == coresb.SmartBlockTypeRelation + + filterKeys := systemObjectFilterKeys + if isNonSystemRelation { + // non-system bundled relations only record the revision and, guardedly, bundled renames + filterKeys = nonSystemRelationFilterKeys + } else if !isSystem { + // non-system bundled types are going to update only icons and plural names for now + filterKeys = customObjectFilterKeys } diff, _ := domain.StructDiff(current, origin) diff = diff.CopyOnlyKeys(filterKeys...) - if cannotApplyPluralName(isSystem, current, origin) { + if isNonSystemRelation { + if !canApplyBundledRelationName(domain.RelationKey(uk.InternalKey()), current.GetString(bundle.RelationKeyName)) { + diff.Delete(bundle.RelationKeyName) + } + } else if cannotApplyPluralName(isSystem, current, origin) { diff.Delete(bundle.RelationKeyName) diff.Delete(bundle.RelationKeyPluralName) } return diff } +// canApplyBundledRelationName reports whether the bundled name may overwrite the local one: +// only when the local name is still a previous bundled name from previousBundledRelationNames +// (or is empty). Any other local name is the user's own rename and must be kept. +func canApplyBundledRelationName(key domain.RelationKey, currentName string) bool { + if currentName == "" { + return true + } + return lo.Contains(previousBundledRelationNames[key], currentName) +} + func cannotApplyPluralName(isSystem bool, current, origin *domain.Details) bool { // we cannot set plural name to custom types with custom name return !isSystem && current.GetString(bundle.RelationKeyName) != origin.GetString(bundle.RelationKeyName) diff --git a/space/internal/components/migration/systemobjectreviser/systemobjectreviser_test.go b/space/internal/components/migration/systemobjectreviser/systemobjectreviser_test.go index 461c57909a..a99b17e909 100644 --- a/space/internal/components/migration/systemobjectreviser/systemobjectreviser_test.go +++ b/space/internal/components/migration/systemobjectreviser/systemobjectreviser_test.go @@ -2,14 +2,22 @@ package systemobjectreviser import ( "context" + "fmt" "testing" "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/commonspace/object/acl/list" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/anyproto/anytype-heart/core/block/editor/smartblock" + "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/core/relationutils" "github.com/anyproto/anytype-heart/pkg/lib/bundle" + coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/localstore/addr" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" @@ -51,6 +59,51 @@ func TestMigration_Run(t *testing.T) { assert.Equal(t, 1, migrated) assert.Equal(t, 1, toMigrate) }) + t.Run("a space this account may only read is skipped after the first refusal", func(t *testing.T) { + // given — three revisable relations in a space the ACL will not let + // this account write + store := objectstore.NewStoreFixture(t) + var objects []objectstore.TestObject + for i, key := range []domain.RelationKey{ + bundle.RelationKeyBacklinks, bundle.RelationKeyRelationKey, bundle.RelationKeyRelationOptionColor, + } { + objects = append(objects, objectstore.TestObject{ + bundle.RelationKeySpaceId: domain.String("space1"), + bundle.RelationKeyRelationFormat: domain.Int64(int64(model.RelationFormat_object)), + bundle.RelationKeyResolvedLayout: domain.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeyId: domain.String(fmt.Sprintf("id%d", i)), + bundle.RelationKeyIsHidden: domain.Bool(true), + bundle.RelationKeyRevision: domain.Int64(0), + bundle.RelationKeyUniqueKey: domain.String(key.URL()), + bundle.RelationKeySourceObject: domain.String(key.BundledURL()), + }) + } + store.AddObjects(t, "space1", objects) + + fixer := &Migration{} + ctx := context.Background() + log := logger.NewNamed("test") + + spc := mock_space.NewMockSpace(t) + spc.EXPECT().Id().Return("space1").Maybe() + spc.EXPECT().DeriveObjectID(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key domain.UniqueKey) (string, error) { + return key.Marshal(), nil + }).Maybe() + // the ACL refuses every write, and the migration must ask exactly ONCE: + // a refusal is the space's answer, not the object's, and each attempt + // leaves the loaded document carrying a value that will never persist + spc.EXPECT().DoCtx(ctx, mock.Anything, mock.Anything). + Return(list.ErrInsufficientPermissions).Times(1) + + // when + toMigrate, migrated, err := fixer.Run(ctx, log, store.SpaceIndex("space1"), spc) + + // then — reported as a skip, not a failure: a reader cannot revise + // the system objects of a space they do not own + assert.NoError(t, err) + assert.Equal(t, 0, migrated) + assert.Equal(t, 1, toMigrate, "the pass stops at the first refusal") + }) } func TestReviseSystemObject(t *testing.T) { @@ -211,12 +264,12 @@ func TestReviseSystemObject(t *testing.T) { assert.False(t, toRevise) }) - t.Run("non system relation is not updated", func(t *testing.T) { - // given + t.Run("non system relation without newer bundle revision is not updated", func(t *testing.T) { + // given bundle audioLyrics revision = 0, so the local object is already up to date rel := domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ bundle.RelationKeyRevision: domain.Int64(1), - bundle.RelationKeySourceObject: domain.String("_brlyrics"), - bundle.RelationKeyUniqueKey: domain.String("rel-lyrics"), + bundle.RelationKeySourceObject: domain.String("_braudioLyrics"), + bundle.RelationKeyUniqueKey: domain.String("rel-audioLyrics"), }) space := mock_space.NewMockSpace(t) // if unexpected space.Do will be called, test will fail @@ -322,7 +375,7 @@ func TestBuildDiffDetails(t *testing.T) { bundle.RelationKeyName: domain.String("Page"), }), domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ bundle.RelationKeyName: domain.String("Page"), - }), true) + }), domain.MustUniqueKey(coresb.SmartBlockTypeObjectType, "page"), true) assert.Equal(t, "Pages", diff.GetString(bundle.RelationKeyPluralName)) }) @@ -333,7 +386,7 @@ func TestBuildDiffDetails(t *testing.T) { bundle.RelationKeyName: domain.String("Project"), }), domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ bundle.RelationKeyName: domain.String("Project"), - }), false) + }), domain.MustUniqueKey(coresb.SmartBlockTypeObjectType, "project"), false) assert.Equal(t, "Projects", diff.GetString(bundle.RelationKeyPluralName)) }) @@ -344,9 +397,142 @@ func TestBuildDiffDetails(t *testing.T) { bundle.RelationKeyName: domain.String("Project"), }), domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ bundle.RelationKeyName: domain.String("Проект"), - }), false) + }), domain.MustUniqueKey(coresb.SmartBlockTypeObjectType, "project"), false) assert.False(t, diff.Has(bundle.RelationKeyPluralName)) assert.False(t, diff.Has(bundle.RelationKeyName)) }) } + +func TestReviseNonSystemBundledRelation(t *testing.T) { + ctx := context.Background() + log := logger.NewNamed("test") + + newSpaceApplyingTo := func(t *testing.T, sb *smarttest.SmartTest) *mock_space.MockSpace { + spc := mock_space.NewMockSpace(t) + spc.EXPECT().Id().Return("space1").Maybe() + spc.EXPECT().DeriveObjectID(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key domain.UniqueKey) (string, error) { + return key.Marshal(), nil + }).Maybe() + spc.EXPECT().DoCtx(mock.Anything, sb.Id(), mock.Anything).RunAndReturn( + func(_ context.Context, _ string, apply func(smartblock.SmartBlock) error) error { + return apply(sb) + }).Times(1) + return spc + } + + t.Run("relation still carrying the previous bundled name gets the new bundled name", func(t *testing.T) { + // given bundle audioGenre was renamed "Genre" -> "Audio genre" with revision 1 + rel := domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyId: domain.String("audioGenreId"), + bundle.RelationKeyName: domain.String("Genre"), + bundle.RelationKeySourceObject: domain.String(bundle.RelationKeyAudioGenre.BundledURL()), + bundle.RelationKeyUniqueKey: domain.String(bundle.RelationKeyAudioGenre.URL()), + }) + sb := smarttest.New("audioGenreId") + sb.Doc.(*state.State).SetDetail(bundle.RelationKeyName, domain.String("Genre")) + space := newSpaceApplyingTo(t, sb) + want := map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("Audio genre"), + bundle.RelationKeyRevision: domain.Int64(bundle.MustGetRelation(bundle.RelationKeyAudioGenre).Revision), + } + + // when + toRevise, err := reviseObject(ctx, log, space, rel) + + // then + require.NoError(t, err) + assert.True(t, toRevise) + for key, value := range want { + assert.Equal(t, value, sb.Details().Get(key), key) + } + // nothing beyond name and revision is applied to a non-system relation + assert.False(t, sb.Details().Has(bundle.RelationKeyRecommendedFeaturedRelations)) + }) + + t.Run("relation renamed by the user keeps the user's name", func(t *testing.T) { + // given the local name matches neither the previous nor the current bundled name + rel := domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyId: domain.String("audioGenreId"), + bundle.RelationKeyName: domain.String("My genre"), + bundle.RelationKeySourceObject: domain.String(bundle.RelationKeyAudioGenre.BundledURL()), + bundle.RelationKeyUniqueKey: domain.String(bundle.RelationKeyAudioGenre.URL()), + }) + sb := smarttest.New("audioGenreId") + sb.Doc.(*state.State).SetDetail(bundle.RelationKeyName, domain.String("My genre")) + space := newSpaceApplyingTo(t, sb) + want := map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("My genre"), + bundle.RelationKeyRevision: domain.Int64(bundle.MustGetRelation(bundle.RelationKeyAudioGenre).Revision), + } + + // when + toRevise, err := reviseObject(ctx, log, space, rel) + + // then + require.NoError(t, err) + assert.True(t, toRevise) + for key, value := range want { + assert.Equal(t, value, sb.Details().Get(key), key) + } + }) + + t.Run("relation with recorded bundle revision is not revised again", func(t *testing.T) { + // given + rel := domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyId: domain.String("audioGenreId"), + bundle.RelationKeyName: domain.String("My genre"), + bundle.RelationKeyRevision: domain.Int64(bundle.MustGetRelation(bundle.RelationKeyAudioGenre).Revision), + bundle.RelationKeySourceObject: domain.String(bundle.RelationKeyAudioGenre.BundledURL()), + bundle.RelationKeyUniqueKey: domain.String(bundle.RelationKeyAudioGenre.URL()), + }) + space := mock_space.NewMockSpace(t) // if unexpected space.Do will be called, test will fail + + // when + toRevise, err := reviseObject(ctx, log, space, rel) + + // then + assert.NoError(t, err) + assert.False(t, toRevise) + }) + + t.Run("bundled name is applied only via the previous-names table", func(t *testing.T) { + assert.True(t, canApplyBundledRelationName(bundle.RelationKeyAudioGenre, "Genre")) + assert.True(t, canApplyBundledRelationName(bundle.RelationKeyAudioGenre, "")) + assert.False(t, canApplyBundledRelationName(bundle.RelationKeyAudioGenre, "Genre 2")) + assert.True(t, canApplyBundledRelationName(bundle.RelationKeyHeaderRelationsLayout, "Header relations layout")) + // a relation with no recorded rename history never gets its name overwritten + assert.False(t, canApplyBundledRelationName(bundle.RelationKeyAudioLyrics, "Lyrics")) + }) + + t.Run("previous-names table agrees with the bundle", func(t *testing.T) { + for key, previousNames := range previousBundledRelationNames { + relation, err := bundle.GetRelation(key) + require.NoError(t, err, key) + // without a revision bump the rename never reaches existing spaces + assert.GreaterOrEqual(t, relation.Revision, int64(1), key) + // the table is only for non-system relations: the system path applies names unconditionally + assert.False(t, bundle.IsSystemRelation(key), key) + assert.NotContains(t, previousNames, relation.Name, key) + } + }) + + t.Run("only revision and name are revisable on non-system relations", func(t *testing.T) { + // given a local object diverging from the bundle in name, hidden flag and readonly value + diff := buildDiffDetails( + (&relationutils.Relation{Relation: bundle.MustGetRelation(bundle.RelationKeyAudioGenre)}).ToDetails(), + domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyName: domain.String("Genre"), + bundle.RelationKeyIsHidden: domain.Bool(true), + bundle.RelationKeyRelationReadonlyValue: domain.Bool(true), + }), + domain.MustUniqueKey(coresb.SmartBlockTypeRelation, bundle.RelationKeyAudioGenre.String()), + false) + + // then only name and revision made it into the diff + assert.Equal(t, "Audio genre", diff.GetString(bundle.RelationKeyName)) + assert.Equal(t, bundle.MustGetRelation(bundle.RelationKeyAudioGenre).Revision, diff.GetInt64(bundle.RelationKeyRevision)) + assert.False(t, diff.Has(bundle.RelationKeyIsHidden)) + assert.False(t, diff.Has(bundle.RelationKeyRelationReadonlyValue)) + }) +} diff --git a/util/builtinobjects/builtinobjects.go b/util/builtinobjects/builtinobjects.go index 87461a62bb..8ed1e6df5d 100644 --- a/util/builtinobjects/builtinobjects.go +++ b/util/builtinobjects/builtinobjects.go @@ -228,8 +228,13 @@ func (b *builtinObjects) CreateObjectsForExperience(ctx context.Context, spaceId if err != nil { log.Warnf("failed to get profile object: %v", err) } + // isBundle: true — this branch only runs for isNewSpace, so the + // profile's name and icon are this space's own identity, not an + // overwrite of one the user already chose (setWorkspaceSettings skips + // both under isBundle=false, which is why an experience-installed + // space always lost its avatar; see SPEC.md §2c) // TODO: GO-2627 Home page handling should be moved to importer - b.setWorkspaceSettings(profile, spaceId, false) + b.setWorkspaceSettings(profile, spaceId, true) removeFunc() } else if importFormat == model.Import_Markdown { // try to read manifest.json from archive diff --git a/util/constant/constant.go b/util/constant/constant.go index 37a76ee5ec..35f7e84e2d 100644 --- a/util/constant/constant.go +++ b/util/constant/constant.go @@ -1,6 +1,9 @@ package constant -import "math/rand" +import ( + "math/rand" + "slices" +) const ProfileFile = "profile" @@ -34,3 +37,11 @@ var colors = []OptionColor{ func RandomOptionColor() OptionColor { return colors[rand.Intn(len(colors))] } + +// OptionColors is the palette in canonical order. Callers that assign colors +// deliberately rather than at random — an AnyBlock JSON bundle declaring a +// select vocabulary, say — cycle it so a vocabulary that names no colors +// still ends up with distinct ones. +func OptionColors() []OptionColor { + return slices.Clone(colors) +}