From ec5be1e828ed3d1b9aa01b4e811f2c5b9b46c0c3 Mon Sep 17 00:00:00 2001 From: Fei Xu Date: Thu, 22 Oct 2026 16:26:05 +0000 Subject: [PATCH 1/2] feat(semantic): write entities, fields and field expressions into the second-generation catalog types With KC_V2_ASPECTS on, every entity gets an entry, abstract ones included. Its semantic-entity aspect holds the source and keys as written, its direct parents and one record per named profile. Its schema lists its own and inherited fields with their logical types, and a field's instructions, custom ai_context members, third-party blocks and examples go in aspects at the field's path. Field expressions go in sql-expressions, with the profile that supplied each one. An entity entry that would hold 100 instances of overview, guidelines or semantic-extensions fails the push before any write. --- .../semantic/deploy_knowledge_catalog.ts | 62 ++- .../mdcode/src/libts/semantic/kc_actions.ts | 5 +- .../mdcode/src/libts/semantic/kc_entries.ts | 15 +- .../src/libts/semantic/knowledge_catalog.ts | 239 ++++++++--- .../src/libts/semantic/resolve_profiles.ts | 8 + .../mdcode/src/libts/semantic/transpile.ts | 4 +- toolbox/mdcode/src/tool/commands.ts | 10 +- .../tests/libts/semantic/actions.test.ts | 16 +- .../semantic/deploy_knowledge_catalog.test.ts | 59 ++- .../tests/libts/semantic/kc_entries.test.ts | 12 +- .../libts/semantic/knowledge_catalog.test.ts | 396 +++++++++++++++++- 11 files changed, 703 insertions(+), 123 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 32e2e083f..5e314414d 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -23,7 +23,7 @@ // entry group to guard against foreign models (unless `--force-remove` is // set), and checks that any required custom entry/aspect types exist in // the target project. With the second-generation aspects, it also checks -// every aspect's size. +// every aspect's size and each entry's instance count per aspect type. // * Apply creates or updates each model's entries in dependency order: the // `semantic-model` anchor first, then independent child entries // concurrently, and finally dependent child entries (`semantic-metric` and @@ -196,7 +196,8 @@ function sleep(ms: number): Promise { * `opts.fileCopyTypeChecked` is set. * 5. Returns the emitted resources, entry-group listing, warnings, and plan * as `prepared` for `applyKnowledgeCatalog`. - * With `opts.v2Aspects` set, step 2 also checks every aspect's size. + * With `opts.v2Aspects` set, step 2 also checks every aspect's size and each + * entry's instance count per aspect type. */ export async function preflightKnowledgeCatalog( models: LoadedModel[], ctx: context.ApiContext, opts: KcDeployOptions): @@ -324,8 +325,8 @@ export async function deployEmittedModels( * The method performs the following checks: * 1. Validates locally that `emitted` contains a single model, that each * file copy fits in one aspect and, with `opts.v2Aspects` set, that every - * aspect fits. Builds the dry-run plan and returns early when - * `opts.validateOnly` is set. + * aspect size and instance count is within its limit. Builds the + * dry-run plan and returns early when `opts.validateOnly` is set. * 2. Lists the destination entry group and validates that it contains no * foreign models unless `--force-remove` is set. * 3. Validates that every custom type used by the model exists in the @@ -516,8 +517,9 @@ function fileCopySizeError(data: Record): string|undefined { /** * The emitter's warnings for `models` with v2Aspects on, and why push's - * catalog leg cannot write them: an emit error or an aspect too large to - * store. Calls no service. + * catalog leg cannot write them: an emit error, an aspect too large to + * store, or an entry with too many instances of one aspect type. Calls no + * service. */ export function checkCatalogAspects( models: LoadedModel[], @@ -531,7 +533,8 @@ export function checkCatalogAspects( // Why push cannot write what `model` emits, or undefined when it can. function emittedAspectsError(model: EmittedModel): string|undefined { for (const entry of model.resources.entries) { - const error = aspectDataError(entry.aspects, `entry '${entryId(entry)}'`); + const error = aspectDataError(entry.aspects, `entry '${entryId(entry)}'`) ?? + instanceCountError(entry); if (error) return error; } for (const link of model.resources.entryLinks) { @@ -541,6 +544,33 @@ function emittedAspectsError(model: EmittedModel): string|undefined { return undefined; } +// Most aspects Dataplex returns for one entry in one read. +const MAX_ASPECTS_PER_READ = 100; + +// The aspect types a field can carry at its path, each with what fills it. +const FIELD_ASPECT_CONTENT: Record = { + overview: 'examples', + guidelines: 'instructions or custom ai_context members', + 'semantic-extensions': 'custom_extensions from vendors other than GOOGLE', +}; + +// Why kcmd could not read back every instance of one aspect type `entry` +// carries, or undefined when it could. +function instanceCountError(entry: Entry): string|undefined { + for (const [type, content] of Object.entries(FIELD_ASPECT_CONTENT)) { + const count = Object.keys(entry.aspects ?? {}) + .filter(key => aspectTypeIdOfKey(key) === type) + .length; + if (count < MAX_ASPECTS_PER_READ) continue; + return `entry '${entryId(entry)}' would hold ${count} ${type} aspects, ` + + `and Knowledge Catalog returns at most ${MAX_ASPECTS_PER_READ} in ` + + `one read, so kcmd could not read them all back; give ${content} ` + + `to fewer than ${MAX_ASPECTS_PER_READ} of the entity and its ` + + `fields, inherited fields included.`; + } + return undefined; +} + // Why Knowledge Catalog cannot store one of `aspects`, which `owner` carries, // or undefined when each fits. function aspectDataError( @@ -1607,12 +1637,12 @@ const KEPT_SOURCE_FIELDS = * name. Passing `entryAspectKeys(entry, opts)` with `deleteMissingAspects: * true` makes a re-push converge: a still-present optional aspect is refreshed, * and a removed one (such as `guidelines` when `ai_context.instructions` is - * deleted, `sql-expressions`, or a field's `guidelines`, matched by its `@*` - * key under `v2Aspects`) is deleted so a later `pull` does not bring it back. - * Push owns the optional aspects `entryAspectKeys` lists, so it removes any - * that the model does not declare even if someone added them in the console. - * Every other aspect another tool attached is left alone, since `aspectKeys` - * names only the types kcmd writes. + * deleted, `sql-expressions`, or a field's `guidelines` or + * `semantic-extensions`, matched by its `@*` key under `v2Aspects`) is deleted + * so a later `pull` does not bring it back. Push owns the optional aspects + * `entryAspectKeys` lists, so it removes any that the model does not declare + * even if someone added them in the console. Every other aspect another tool + * attached is left alone, since `aspectKeys` names only the types kcmd writes. * * The update mask replaces the whole entrySource, so the update carries over * fields from `source`, the entrySource the entry had before this push. Of @@ -1641,11 +1671,11 @@ async function updateEntry( if (Object.keys(merged).length) entry = {...entry, entrySource: merged}; // We cannot simply pass `Object.keys(entry.aspects ?? {})` here because // `entry.aspects` only contains the aspects emitted in *this* push. If an - // optional aspect (such as `guidelines`, `sql-expressions`, or a column's + // optional aspect (such as `guidelines`, `sql-expressions`, or a field's // `guidelines@Schema.`) was present on a previous push and removed in // this push, it is absent from `entry.aspects` -- and Dataplex only deletes a - // removed aspect when its key (or `..guidelines@*` for - // field-level `guidelines`) is still listed in `aspectKeys`. + // removed aspect when its key (or its `@*` key, for a field's instance) is + // still listed in `aspectKeys`. const aspectKeys = entryAspectKeys(entry, opts); const upd = await cat.updateEntry( entry, ['entry_source', 'aspects'], [...aspectKeys], diff --git a/toolbox/mdcode/src/libts/semantic/kc_actions.ts b/toolbox/mdcode/src/libts/semantic/kc_actions.ts index 4751aa95a..fa3f533ff 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_actions.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_actions.ts @@ -71,7 +71,7 @@ export interface ActionEmitContext { // emitted (knowledge_catalog.claim). claim(entryId: string, label: string): boolean; // Names of the entities this push actually publishes an entry for. An - // abstract entity, or one a binding profile pruned, is absent, so a + // abstract entity under the first-generation types is absent, so a // parameter typed by it would name an entry that does not exist. publishedEntities: Set; } @@ -92,7 +92,8 @@ export function actionOwnedPrefix(modelId: string): string { // it will. Three ways to lose it, and naming which one is the difference // between an author deleting the projection and an author binding a column: // -// - the concept is not published at all (pruned, or abstract); +// - the concept is not published at all (an abstract entity under the +// first-generation types); // - the concept is published but the FIELD is not -- `pruneUnavailable` // drops an unbound field while keeping the entity whose key still binds, // and a projected parameter no longer prunes its action, so the entry goes diff --git a/toolbox/mdcode/src/libts/semantic/kc_entries.ts b/toolbox/mdcode/src/libts/semantic/kc_entries.ts index a9eacd359..bb6398b5b 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_entries.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_entries.ts @@ -269,9 +269,10 @@ function systemTypeProjectAndLocation( * so Dataplex removes the aspect when the object no longer has content * for it. * * Adds `..sql-expressions` on entity and metric entries. - * * Adds `..guidelines@*` on entity entries so Dataplex clears - * field-level `guidelines` aspects for columns whose instructions or - * fields were removed. + * * Adds `..guidelines@*` and `..semantic-extensions@*` + * on entity entries, so Dataplex removes a field's instance the entry no + * longer carries. These wildcards also write the field-path instances + * `entry` carries, so those keys are left out. */ export function entryAspectKeys( entry: Entry, opts?: SystemTypeOptions): Set { @@ -290,7 +291,13 @@ export function entryAspectKeys( keys.add(`${proj}.${loc}.sql-expressions`); } if (isEntityEntry(entry)) { - keys.add(`${proj}.${loc}.guidelines@*`); + for (const type of ['guidelines', 'semantic-extensions']) { + const prefix = `${proj}.${loc}.${type}@`; + for (const key of keys) { + if (key.startsWith(prefix)) keys.delete(key); + } + keys.add(`${prefix}*`); + } } return keys; } diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index 7775ff32f..a4804ac92 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -52,11 +52,14 @@ // KcGenerateOptions.emitExpressions (off by default) and omitted above. // // With the second-generation types (v2Aspects on), the model entry carries -// semantic-model and each metric entry carries semantic-metric and -// sql-expressions. Model, metric and relationship entries also carry -// guidelines, semantic-extensions and overview when the object has content -// for them. `guidelines` holds the object's instructions and, in `data`, its -// custom `ai_context` members. +// semantic-model, each entity entry carries semantic-entity and schema, and +// each metric entry carries semantic-metric and sql-expressions. An entity +// entry has sql-expressions too when one of its fields has an expression. +// Model, entity, metric and relationship entries also carry guidelines, +// semantic-extensions and overview when the object has content for them. A +// field has no entry, so its guidelines, semantic-extensions and overview go +// on its entity's entry at the field's path. `guidelines` holds the object's +// instructions and, in `data`, its custom `ai_context` members. // // Relationships become `schema-join` entry links between the two entity entries. // schema-join is a built-in, undirected entry link type in `dataplex-types/global` @@ -86,8 +89,7 @@ import {customAspectKey, customAspectTypeName, customEntryTypeName, FILE_COPY_TY import {linkSlug, Namer, ownedEntryIdPrefixes, semanticLinkId} from './kc_ids'; import {asResourceUri, GOOGLE_VERSION, isAcceptedSource} from './loader'; import {nonEmpty} from './osi_converter'; -import {InheritanceError, resolveInheritance} from './resolve_inheritance'; -import {DEFAULT_PROFILE} from './resolve_profiles'; +import {DEFAULT_PROFILE, resolvedEntities, statesKey, withoutExclusions} from './resolve_profiles'; export interface KcGenerateOptions { project: string; // project the entries are created in (destination) @@ -100,8 +102,8 @@ export interface KcGenerateOptions { // role) and `semantic-metric.expression`. Off by default so a push matches // the live types; flip on once the templates gain these fields. Enabling it // against today's types fails the push with an ASPECT_*_PARSING_FAILURE on - // the unknown property. With v2Aspects on, `semantic-metric` carries no - // expression. + // the unknown property. With v2Aspects on it changes nothing: `schema` has + // no `semantics` block and `semantic-metric` carries no expression. emitExpressions?: boolean; // Emit the second-generation built-in aspect fields and entry-id layout // (selected via KC_V2_ASPECTS=1). Off by default (absent means false), which @@ -168,9 +170,10 @@ export function generateCatalogResources( 'preserves them'); } - // With the first-generation aspects, a subtype field that only rebinds an - // inherited field is published with the definition it inherits. - const inherited = v2 ? undefined : inheritedFields(model); + // Each entity's fields with inheritance resolved: the second-generation + // schema lists them, and the first-generation one takes an inherited + // definition from them. + const resolved = resolvedFields(model); // Entry ids must be unique within the entry group. Two source names that // normalize to the same id (see slug), or exact duplicates the loader did not @@ -204,14 +207,9 @@ export function generateCatalogResources( // actually emitted (not skipped for a duplicate id) are recorded. const entityEntryName = new Map(); for (const entity of entities) { - // An abstract entity is a conceptual (table-less) supertype: it has no - // physical resource to catalog. The Knowledge Catalog leg does not model - // inheritance (that is BigQuery-only today, via resolveInheritance), so - // rather than publish a malformed entry -- an empty linked resource and no - // key -- skip it with a warning. Its concrete subtypes are published - // normally, and any edge naming it as an endpoint is dropped downstream - // (its name never enters `entityEntryName`). - if (entity.abstract) { + // The first-generation types do not model inheritance, so an abstract + // entity is skipped with a warning and an edge naming it gets no link. + if (entity.abstract && !v2) { warnings.push( `entity '${entity.name}' is abstract (no physical table); skipped ` + `for Knowledge Catalog (KC does not yet model class hierarchies)`); @@ -221,7 +219,8 @@ export function generateCatalogResources( if (!claim(seen, entityId, 'entry', `entity '${entity.name}'`, warnings)) continue; entityEntryName.set(entity.name, names.entry(entityId)); - if (!entity.keys || !entity.keys.length) { + const fields = resolved.get(entity)!; + if (v2 ? lacksKeys(model, entity) : !entity.keys || !entity.keys.length) { warnings.push( `entity '${entity.name}': no keys declared in the source model`); } @@ -230,16 +229,15 @@ export function generateCatalogResources( entryType: names.typeName('entry', 'semantic-entity'), parentEntry: modelEntryName, entrySource: source(entity.name, entity.description), - // required_aspects: semantic-entity AND the built-in schema; plus the - // built-in guidelines aspect when the entity carries ai_context - // instructions. - aspects: aspectsFor( - names, { - 'semantic-entity': entityAspectData(entity), - 'schema': schemaAspectData( - entity, emitExpr, inherited?.get(entity.name)), - }, - entity.aiContext), + // semantic-entity and schema are required. The file header lists the + // optional aspects. + aspects: v2 ? aspectMap(names, entityAspectsV2(model, entity, fields)) : + aspectsFor( + names, { + 'semantic-entity': entityAspectData(entity), + 'schema': schemaAspectData(entity, emitExpr, fields), + }, + entity.aiContext), }); } @@ -282,18 +280,21 @@ export function generateCatalogResources( } } + // The second-generation schema keeps a field a profile excludes, so + // actionEntries reads the model without exclusions. + const actionModel = v2 ? withoutExclusions(model) : model; // One entry per action, alongside the entities and metrics. Actions have no // built-in system type, so they reference the custom pair declared in // `kc_custom_types.ts`, and `kc_actions.ts` fills the aspect; it returns // nothing when the model declares no actions. - entries.push(...actionEntries(model, modelId, { + entries.push(...actionEntries(actionModel, modelId, { project: opts.project, entry: (id: string) => names.entry(id), anchor: modelEntryName, claim: (id: string, label: string) => claim(seen, id, 'entry', label, warnings), - // Populated by the entity loop above, so it excludes the abstract and - // unavailable entities that loop skipped. + // The entities the loop above gave an entry. With the first-generation + // types an abstract entity has none. publishedEntities: new Set(entityEntryName.keys()), }, warnings)); @@ -651,7 +652,7 @@ function hasThirdPartyExtensions(model: SemanticModel): boolean { } // One warning for each custom `ai_context` member that `guidelines` leaves -// out, on the model, a metric or a relationship. +// out, on the model, an entity, a field, a metric or a relationship. function unstorableMemberWarnings(model: SemanticModel): string[] { const warnings: string[] = []; const check = (object: string, ai: AiContext|undefined) => { @@ -667,6 +668,12 @@ function unstorableMemberWarnings(model: SemanticModel): string[] { } }; check(`model '${model.name}'`, model.aiContext); + for (const entity of model.entities ?? []) { + check(`entity '${entity.name}'`, entity.aiContext); + for (const field of entity.fields ?? []) { + check(`field '${entity.name}.${field.name}'`, field.aiContext); + } + } for (const metric of model.metrics ?? []) { check(`metric '${metric.name}'`, metric.aiContext); } @@ -691,24 +698,24 @@ function unstorableValue(value: unknown): string|undefined { return Object.values(value).map(unstorableValue).find(Boolean); } -// Each entity's fields with inheritance resolved, by entity and field name, or -// undefined when no entity extends another or inheritance cannot be resolved. -// A resolved field keeps what the entity's own declaration sets and takes the -// rest from the ancestor that defines it. -function inheritedFields(model: SemanticModel): - Map>|undefined { - if (!(model.entities ?? []).some(e => e.extends?.length)) return undefined; - let resolved: SemanticModel; - try { - resolved = resolveInheritance(model).model; - } catch (err) { - // Push validation reports it. The schema aspect keeps what each entity - // declares. - if (err instanceof InheritanceError) return undefined; - throw err; - } - return new Map((resolved.entities ?? []).map( - e => [e.name, new Map((e.fields ?? []).map(f => [f.name, f]))])); +// Each entity's fields with inheritance resolved and every profile exclusion +// ignored. A concrete entity has its own fields, then the ones it inherits, and +// an abstract entity has its own. +function resolvedFields(model: SemanticModel): Map { + const entities = model.entities ?? []; + const resolved = resolvedEntities(withoutExclusions(model)); + // resolvedEntities keeps the entities in order. + return new Map( + entities.map((e, i) => [e, e.abstract ? e.fields : resolved[i].fields])); +} + +// True when `entity` is concrete and states no primary key and no unique key, +// in its own bindings or in any named profile. +function lacksKeys(model: SemanticModel, entity: Entity): boolean { + const bindings = namedProfiles(model).flatMap( + p => p.entities.filter(b => b.name === entity.name)); + return !entity.abstract && !statesKey(entity.keys, entity.uniqueKeys) && + !bindings.some(b => statesKey(b.primaryKey, b.uniqueKeys)); } @@ -762,7 +769,7 @@ function entityAspectData(entity: Entity): Record { // it inherits from there. function schemaAspectData( entity: Entity, emitExpressions: boolean, - inherited?: Map): Record { + inherited: Field[]): Record { // Key column lists ride the schema aspect verbatim -- they need NOT be // declared fields (the closed template accepts and round-trips columns absent // from fields[], live-verified). Keep only non-empty strings, matching the @@ -779,7 +786,7 @@ function schemaAspectData( // A subtype's redeclaration of an inherited field only rebinds it, so its // definition is the inherited one. const withDefinition = (f: Field): Field => { - const from = inherited?.get(f.name); + const from = inherited.find(g => g.name === f.name); return { ...f, type: f.type ?? from?.type, @@ -863,15 +870,16 @@ function metricAspectData( } // The guidelines, semantic-extensions and overview aspects of an object's -// entry, each left out when the object has nothing for it. +// entry, each left out when the object has nothing for it. For a field's +// aspects, `at` is `@Schema.`, and each key ends in it. function objectAspectsV2( - object: {aiContext?: AiContext; customExtensions?: CustomExtension[]}): - Record { + object: {aiContext?: AiContext; customExtensions?: CustomExtension[]}, + at = ''): Record { const examples = object.aiContext?.examples ?? []; return compact({ - guidelines: guidelinesAspectDataV2(object.aiContext), - 'semantic-extensions': extensionsAspectData(object.customExtensions), - overview: examples.length ? + [`guidelines${at}`]: guidelinesAspectDataV2(object.aiContext), + [`semantic-extensions${at}`]: extensionsAspectData(object.customExtensions), + [`overview${at}`]: examples.length ? {examples: examples.map(content => ({content}))} : undefined, }); @@ -917,6 +925,104 @@ function modelAspectDataV2(model: SemanticModel): Record { }); } +// The aspects of an entity entry under the second-generation types, with +// `fields` the fields its schema lists, each with its own aspects at its path. +function entityAspectsV2( + model: SemanticModel, entity: Entity, + fields: Field[]): Record> { + return compact({ + 'semantic-entity': entityAspectDataV2(model, entity), + 'schema': schemaAspectDataV2(fields), + ...objectAspectsV2(entity), + ...Object.assign( + {}, ...fields.map(f => objectAspectsV2(f, `@Schema.${f.name}`))), + 'sql-expressions': fieldExpressionsData(model, entity), + }); +} + +// The built-in schema aspect under the second-generation types, one record per +// field. `is_time` is written only when the document declares it. +function schemaAspectDataV2(fields: Field[]): Record { + return { + fields: fields.map(f => { + const type = f.type ?? 'Opaque'; + const annotations = compact({ + label: f.label || undefined, + is_time: f.dimension?.isTime?.toString(), + }); + return compact({ + name: f.name, + description: f.description, + dataType: type, + metadataType: columnMetadataType(type), + semantic: f.dimension ? 'DIMENSION' : undefined, + synonyms: nonEmpty(f.aiContext?.synonyms), + annotations: Object.keys(annotations).length ? annotations : undefined, + }); + }), + }; +} + +// semantic-entity under the second-generation types: the entity's source and +// keys as written, its direct parents, whether it is abstract, its synonyms, +// and one record per named profile that lists it, holding what that profile +// states. Anything empty is left out. +function entityAspectDataV2( + model: SemanticModel, entity: Entity): Record { + const profiles = namedProfiles(model).flatMap(p => { + const bound = p.entities.find(b => b.name === entity.name); + if (!bound) return []; + return [compact({ + name: p.name, + source: bound.source ? {resource: bound.source} : undefined, + excludedFields: nonEmpty(bound.fieldsExclude), + ...keysData(bound.primaryKey, bound.uniqueKeys), + })]; + }); + return compact({ + source: entity.authoredSource ? {resource: entity.authoredSource} : + undefined, + ...keysData(entity.keys, entity.uniqueKeys), + profiles: nonEmpty(profiles), + extends: nonEmpty(entity.extends), + abstract: entity.abstract || undefined, + synonyms: nonEmpty(entity.aiContext?.synonyms), + }); +} + +// The `primaryKey` and `uniqueConstraints` of semantic-entity or a profile +// record, each undefined when empty. A unique key with no column is left out. +function keysData( + primaryKey: readonly string[]|undefined, + uniqueKeys: readonly string[][]|undefined): Record { + return { + primaryKey: primaryKey?.length ? {fields: [...primaryKey]} : undefined, + uniqueConstraints: nonEmpty( + uniqueKeys?.filter(k => k.length).map(k => ({fields: [...k]}))), + }; +} + +// The sql-expressions aspect of an entity entry: the records of the entity's +// own fields, then those of each named profile that lists it, with the +// profile's name in `qualifier`. Returns undefined when there is no record. +function fieldExpressionsData( + model: SemanticModel, entity: Entity): Record|undefined { + const records = + (fields: ReadonlyArray>, + qualifier?: string) => + fields.flatMap( + f => expressionRecords(f).map( + r => compact({name: f.name, qualifier, ...r}))); + const fieldExpressions = [ + ...records(entity.fields), + ...namedProfiles(model).flatMap( + p => records( + p.entities.find(b => b.name === entity.name)?.fields ?? [], + p.name)), + ]; + return fieldExpressions.length ? {fieldExpressions} : undefined; +} + // The aspects of a metric entry under the second-generation types. function metricAspectsV2( model: SemanticModel, metric: Metric): Record { @@ -1041,14 +1147,15 @@ function resourcePath(dataSource: string): string { // Naming and small helpers. // --------------------------------------------------------------------------- -// Wraps aspect data (keyed by bare type id) as the client's aspect map: each -// key is the `project.location.type` reference form, each value the -// fully-qualified aspectType plus its data. +// Wraps aspect data (keyed by bare type id, with an optional `@`) as the +// client's aspect map: each key is the `project.location.type` reference form +// with the same path, each value the fully-qualified aspectType plus its data. function aspectMap(names: Namer, byType: Record>): Record { const out: Record = {}; - for (const [type, data] of Object.entries(byType)) { - out[names.aspectRef(type)] = { + for (const [key, data] of Object.entries(byType)) { + const type = key.split('@')[0]; + out[names.aspectRef(type) + key.slice(type.length)] = { aspectType: names.typeName('aspect', type), data }; diff --git a/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts b/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts index a1a8a52ae..62cc9b0a4 100644 --- a/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts +++ b/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts @@ -858,6 +858,14 @@ export function pruneUnavailable( return {model: clone, report}; } +// `model` without the exclusions a profile marked on it. +export function withoutExclusions(model: SemanticModel): SemanticModel { + const out = structuredClone(model); + for (const e of out.entities ?? []) delete e.excludedFields; + delete out.excludedMetrics; + return out; +} + // Each entity resolved for inheritance, or as declared when the hierarchy is // broken, which push reports elsewhere. `extends` is expanded to every // ancestor. diff --git a/toolbox/mdcode/src/libts/semantic/transpile.ts b/toolbox/mdcode/src/libts/semantic/transpile.ts index 00f086224..c5c132136 100644 --- a/toolbox/mdcode/src/libts/semantic/transpile.ts +++ b/toolbox/mdcode/src/libts/semantic/transpile.ts @@ -18,8 +18,8 @@ // in GoogleSQL/ANSI is unaffected and its goldens are stable. The BigQuery // generator reads `expression`, so filling it gives the graph the transpiled // GoogleSQL. The Knowledge Catalog emitter reads it too, except that with -// KC_V2_ASPECTS on it stores each metric's dialect texts as written and -// leaves out the translation. +// KC_V2_ASPECTS on it stores each field's and metric's dialect texts as +// written and leaves out the translation. // // Design: // - Target-agnostic and non-mutating: `transpileModel(model, {target, diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 6bdb060bc..6d98bfe63 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -23,7 +23,7 @@ import {FILE_COPY_TYPE_ID, FileCopy, provisionCustomTypes} from '../libts/semant import {deploymentProfileError, LoadedModel, loadSemanticModels, SemanticModelError, setsAKeyTwice, writtenSource} from '../libts/semantic/loader'; import {serializeModel} from '../libts/semantic/osi_converter'; import * as pullKc from '../libts/semantic/pull_kc'; -import {applyProfileExclusions, AvailabilityReport, CloudSqlLoad, DeclaredProfile, declaredProfiles, DEFAULT_PROFILE, DocumentProfile, expressionSources, inheritanceErrorOf, isLegacyWrapper, isProfileFileForm, loadProfileFile, modelFileAloneMessage, ProfileExclusion, profileErrorMessage, profileForDocument, profileSourceMessage, pruneUnavailable, reservedProfileNameError, sourcesDialect, unknownProfileMessage, validateProfileCompleteness, validateProfileConsistency,} from '../libts/semantic/resolve_profiles'; +import {applyProfileExclusions, AvailabilityReport, CloudSqlLoad, DeclaredProfile, declaredProfiles, DEFAULT_PROFILE, DocumentProfile, expressionSources, inheritanceErrorOf, isLegacyWrapper, isProfileFileForm, loadProfileFile, modelFileAloneMessage, ProfileExclusion, profileErrorMessage, profileForDocument, profileSourceMessage, pruneUnavailable, reservedProfileNameError, sourcesDialect, unknownProfileMessage, validateProfileCompleteness, validateProfileConsistency, withoutExclusions,} from '../libts/semantic/resolve_profiles'; import {createSemanticRuntimes, undeclaredProfileError} from '../libts/semantic/runtime/runtime'; import {profileTargets, storeLine} from '../libts/semantic/runtime/store'; import {generateSkill, SkillPackage} from '../libts/semantic/skills'; @@ -438,14 +438,6 @@ function withProfileLabel( list.map(m => ({...m, document: `${m.document}, profile '${profileName}'`})); } -// `model` without the exclusions a profile marked on it. -function withoutExclusions(model: SemanticModel): SemanticModel { - const out = structuredClone(model); - for (const e of out.entities ?? []) delete e.excludedFields; - delete out.excludedMetrics; - return out; -} - // The fields a profile marked as excluded on `model`. function fieldExclusionsOf(model: SemanticModel): ProfileExclusion[] { return (model.entities ?? []).flatMap( diff --git a/toolbox/mdcode/tests/libts/semantic/actions.test.ts b/toolbox/mdcode/tests/libts/semantic/actions.test.ts index 2c619f883..0a6bba12c 100644 --- a/toolbox/mdcode/tests/libts/semantic/actions.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/actions.test.ts @@ -940,13 +940,12 @@ describe('Knowledge Catalog round trip across executor kinds', () => { describe('actions projecting concepts the push does not publish', () => { test( - 'a parameter projected from an abstract entity publishes, and warns', + 'a parameter projected from an abstract entity publishes, and warns ' + + 'only with the first-generation types', () => { - // An abstract entity is a table-less supertype, so the Knowledge - // Catalog leg skips it. The parameter still publishes whole -- its type - // and wording were resolved at load and travel in the aspect -- but the - // catalog then names a concept it has no entry for, which is what the - // warning is about. + // With the first-generation types the Knowledge Catalog leg skips an + // abstract entity. The published parameter then names a concept with + // no entry, which draws the warning. const {models} = fromDocument( { version: '0.2.0.dev0/google', @@ -985,6 +984,11 @@ describe('actions projecting concepts the push does not publish', () => { w => w.includes('parameter \'who\'') && w.includes('does not publish'))) .toBe(true); + // The second-generation types give the entity an entry, so the + // parameter draws no warning. + expect(generateCatalogResources(models[0], {...OPTS, v2Aspects: true}) + .warnings.filter(w => w.includes('does not publish'))) + .toEqual([]); }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts index 3d4ef1520..2197dea56 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -16,7 +16,7 @@ import {ApiResult} from '../../../src/libts/gcp/api'; import {ApiContext} from '../../../src/libts/gcp/context'; import {fixProject, ResourceManagerClient} from '../../../src/libts/gcp/crm'; import {CatalogClient} from '../../../src/libts/gcp/dataplex'; -import {applyKnowledgeCatalog, checkFileCopies, deployEmittedModels, deployKnowledgeCatalog, EmittedModel, MAX_ASPECT_DATA_BYTES, preflightKnowledgeCatalog,} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {applyKnowledgeCatalog, checkCatalogAspects, checkFileCopies, deployEmittedModels, deployKnowledgeCatalog, EmittedModel, MAX_ASPECT_DATA_BYTES, preflightKnowledgeCatalog,} from '../../../src/libts/semantic/deploy_knowledge_catalog'; import {semanticLinkId} from '../../../src/libts/semantic/kc_ids'; import {loadSemanticModels} from '../../../src/libts/semantic/loader'; @@ -2709,11 +2709,6 @@ describe('deployKnowledgeCatalog: aspect removal on update', () => { expect(metricUpdateCall).toBeDefined(); expect(actionUpdateCall).toBeDefined(); - expect( - entityUpdateCall![0] - .aspects?.['dataplex-types.global.sql-expressions']) - .toBeUndefined(); - expect(modelUpdateCall![2]).toEqual([ 'dataplex-types.global.semantic-model', 'dataplex-types.global.guidelines', @@ -2725,10 +2720,11 @@ describe('deployKnowledgeCatalog: aspect removal on update', () => { expect(entityUpdateCall![2]).toEqual([ 'dataplex-types.global.semantic-entity', 'dataplex-types.global.schema', + 'dataplex-types.global.sql-expressions', 'dataplex-types.global.guidelines', 'dataplex-types.global.semantic-extensions', - 'dataplex-types.global.sql-expressions', 'dataplex-types.global.guidelines@*', + 'dataplex-types.global.semantic-extensions@*', ]); expect(entityUpdateCall![3]).toBe(true); @@ -4183,7 +4179,7 @@ describe('deployKnowledgeCatalog: the size of a V2 aspect', () => { ['a schema aspect over 120 KiB and under 2 MiB fits', ['x'], V2_OPTS, 'd'.repeat(LIMIT)], ['a schema aspect over 2 MiB fails', - ['x', tooLarge('schema', 'entry \'sales/entities/orders\'', 2_097_352, + ['x', tooLarge('schema', 'entry \'sales/entities/orders\'', 2_097_313, 2 * 1024 * 1024)], {...V2_OPTS, validateOnly: true}, 'd'.repeat(2 * 1024 * 1024)], ['with the flag off, only file copies are checked', ['x'.repeat(LIMIT)], @@ -4200,6 +4196,53 @@ describe('deployKnowledgeCatalog: the size of a V2 aspect', () => { }); } + // OSSIE with n more fields on orders, f0 to f, each with the + // ai_context `member`. + const wide = (n: number, member = 'examples: [e]') => + OSSIE.replace('- name: o_orderkey\n', Array.from( + {length: n}, (_, i) => `- { name: f${i}, expression: f${i}, ` + + `ai_context: {${member}} }\n `).join('') + + '- name: o_orderkey\n'); + // Expects a push of `text` to fail before any write, with the same error + // as the check kcmd profiles runs, and returns the error. + async function refusal(text: string) { + const {list, create} = stubClient(); + const docs = models([{name: 'sales.yaml', text}]); + const result = await deployKnowledgeCatalog(docs, CTX, V2_OPTS); + expect(result.success).toBe(false); + expect(list).not.toHaveBeenCalled(); + expect(create).not.toHaveBeenCalled(); + expect(checkCatalogAspects(docs, V2_OPTS).error).toBe(result.details); + return result.details; + } + + test('an entry that would hold 100 instances of one aspect type fails ' + + 'before any write, in push and in the check kcmd profiles runs', + async () => { + expect(await refusal(wide(100))) + .toBe( + 'entry \'sales/entities/orders\' would hold 100 overview ' + + 'aspects, and Knowledge Catalog returns at most 100 in one ' + + 'read, so kcmd could not read them all back; give examples ' + + 'to fewer than 100 of the entity and its fields, inherited ' + + 'fields included.'); + expect(checkCatalogAspects(models([{name: 's.yaml', text: wide(99)}]), + V2_OPTS).error) + .toBeUndefined(); + // Orders has 99 fields with instructions, and big_orders inherits + // them and adds one. + expect(await refusal(wide(99, 'instructions: i') + .replace(' metrics:', ` - name: big_orders + extends: [orders] + source: bigquery.googleapis.com/projects/demo/datasets/sales/tables/big + primary_key: [o_orderkey] + fields: [{ name: g, expression: g, ai_context: {instructions: g} }] + metrics:`))) + .toStartWith( + 'entry \'sales/entities/big_orders\' would hold 100 ' + + 'guidelines aspects, '); + }); + test('a semantic-link aspect is checked too', async () => { stubClient(); const relationships = ['r'.repeat(LIMIT)]; diff --git a/toolbox/mdcode/tests/libts/semantic/kc_entries.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_entries.test.ts index 50fa9fcb7..904479070 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_entries.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_entries.test.ts @@ -348,7 +348,7 @@ describe('entryAspectKeys', () => { test( 'with v2Aspects on, entryAspectKeys lists the V2 keys of each entry ' + - 'kind', + 'kind, with a field\'s keys as wildcards except its overview', () => { const entity = makeEntry('sales/entities/orders', ENTITY_TYPE, { 'dataplex-types.global.semantic-entity': {data: {}}, @@ -364,6 +364,12 @@ describe('entryAspectKeys', () => { 'sales/relationships/orders_to_customer', RELATIONSHIP_TYPE, { 'dataplex-types.global.semantic-relationship': {data: {}}, }); + const withFields = makeEntry('sales/entities/orders', ENTITY_TYPE, { + ...entity.aspects, + ...Object.fromEntries( + ['guidelines', 'semantic-extensions', 'overview'].map( + t => [`dataplex-types.global.${t}@Schema.id`, {data: {}}])), + }); const entityKeys = entryAspectKeys(entity, {v2Aspects: true}); const metricKeys = entryAspectKeys(metric, {v2Aspects: true}); @@ -376,6 +382,7 @@ describe('entryAspectKeys', () => { 'dataplex-types.global.semantic-extensions', 'dataplex-types.global.sql-expressions', 'dataplex-types.global.guidelines@*', + 'dataplex-types.global.semantic-extensions@*', ])); expect(metricKeys).toEqual(new Set([ 'dataplex-types.global.semantic-metric', @@ -397,6 +404,8 @@ describe('entryAspectKeys', () => { 'dataplex-types.global.semantic-extensions', 'dataplex-types.global.overview', ])); + expect(entryAspectKeys(withFields, {v2Aspects: true})).toEqual(new Set( + [...entityKeys, 'dataplex-types.global.overview@Schema.id'])); }); test( @@ -420,6 +429,7 @@ describe('entryAspectKeys', () => { 'staging-types.us.semantic-extensions', 'staging-types.us.sql-expressions', 'staging-types.us.guidelines@*', + 'staging-types.us.semantic-extensions@*', ])); }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts index 99372e1b8..3969d225f 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -70,10 +70,26 @@ const load = (text: string): SemanticModel => // Loads the profile `name` whose other keys are the YAML `body`. const PROFILE = (name: string, body: string) => loadProfileFile(`name: ${name}\n${body}`, name); +// Loads the profile `name`. Each binding is an entity name followed by the +// other keys of its entry, in YAML flow style. +const ENTITIES = (name: string, ...bindings: string[]) => PROFILE( + name, `entities: [${bindings.map(b => `{name: ${b}}`).join(', ')}]`); // The data of a guidelines aspect and of an overview aspect. const GUIDE = (instructions: string) => ({instructions, userManaged: true}); const EXAMPLES = (...texts: string[]) => ({examples: texts.map(content => ({content}))}); +// An actions block with an MCP action for each key, whose parameters are YAML +// flow mappings. +const ACTIONS = (actions: Record) => ' actions:\n' + + Object.entries(actions) + .map(([name, params]) => ` - name: ${name} + executor: + mcp: + server: //agentregistry.googleapis.com/x/mcpServers/shop + tool: ${name.toLowerCase()} + parameters: +${params.map(p => ` - ${p}\n`).join('')}`) + .join(''); // A YAML entity over table p.d. with an `id` field and the given key. const ENTITY_YAML = (name: string, key = '[id]') => ` - name: ${name} source: ${TABLE(name)} @@ -977,10 +993,8 @@ describe('a purely logical model (no physical binding) emits cleanly', () => { describe('abstract entities are skipped for Knowledge Catalog', () => { - // The KC leg does not model inheritance (that is BigQuery-only today), and an - // abstract entity has no physical table, so it must not be published as an - // entry with an empty linked resource. It is skipped with a warning; its - // concrete subtype is published normally. + // With the first-generation types an abstract entity has no table to link, so + // it is skipped with a warning. Its concrete subtype is published normally. function withAbstract(): SemanticModel { return { name: 'm', @@ -1288,9 +1302,8 @@ ${customerFields}`; annotations: {label: 'Party ID'}, }; - function customerField( - model: SemanticModel, name: string, v2Aspects = false): any { - const {entries} = generateCatalogResources(model, {...OPTS, v2Aspects}); + function customerField(model: SemanticModel, name: string): any { + const {entries} = generateCatalogResources(model, OPTS); const customer = entries.find(e => e.entrySource?.displayName === 'Customer')!; return customer.aspects!['dataplex-types.global.schema'].data!.fields.find( (f: any) => f.name === name); @@ -1302,8 +1315,6 @@ ${customerFields}`; MODEL_FILE(' - { name: id, expression: cust_id }\n'), {bindingOptional: true}); expect(customerField(models[0], 'id')).toEqual(ANCESTOR); - // The second-generation layout lists inherited fields another way. - expect(customerField(models[0], 'id', true).dataType).toBe('STRING'); }); // Push rejects a subtype that sets a definition on an inherited field, so @@ -1777,3 +1788,370 @@ ${ENTITY_YAML('addresses')} ]); }); }); + + +describe('with KC_V2_ASPECTS on, entities and their fields', () => { + const AT = (type: string, field: string) => `${KEY(type)}@Schema.${field}`; + const MODEL_FILE = `version: "0.2.0.dev0/google" +semantic_model: + - name: sales + entities: + - name: party + abstract: true + ai_context: {synonyms: [counterparty]} + fields: + - name: id + datatype: Integer + description: The party id + label: Party ID + ai_context: + instructions: Never show raw ids. + synonyms: [party_key] + examples: [Party 7] + custom: {pii: true} + custom_extensions: + - {vendor_name: DBT, data: '{"tests": ["not_null"]}'} + - { name: name, datatype: String } + - name: person + abstract: true + extends: [party] + fields: [{ name: birth_date, datatype: Date }] + - name: customers + extends: [person] + source: ${TABLE('customers')} + primary_key: [id] + unique_keys: [[tenant, email], ['\`legacy id\`']] + ai_context: + instructions: Use for customer analytics. + synonyms: [buyers, clients] + examples: [Who bought the most?] + custom: {tier: gold} + custom_extensions: + - {vendor_name: DBT, data: '{"materialized": "table"}'} + fields: + - { name: id, expression: cust_id } + - { name: signup, datatype: Date, expression: signup, + dimension: {is_time: true} } + - { name: region, expression: region, dimension: {} } + - { name: active, datatype: Boolean, expression: active, + dimension: {is_time: false} } + - { name: email, expression: email } + - name: orders + source: ${TABLE('orders')} + primary_key: [order_id] + fields: [{name: order_id, expression: order_id}] + - name: events + source: ${TABLE('events')} + fields: [{name: at, expression: at}] + - name: visits + source: ${TABLE('visits')} + unique_keys: [[visit_no]] + fields: [{name: visit_no, expression: visit_no}] +`; + const NO_KEYS = (entity: string) => + `entity '${entity}': no keys declared in the source model`; + const ID = { + name: 'id', description: 'The party id', dataType: 'Integer', + metadataType: 'NUMBER', synonyms: ['party_key'], + annotations: {label: 'Party ID'}, + }; + const NAME = {name: 'name', dataType: 'String', metadataType: 'STRING'}; + const BIRTH = + {name: 'birth_date', dataType: 'Date', metadataType: 'DATETIME'}; + const DBT = {vendorName: 'DBT', data: '{"tests": ["not_null"]}'}; + const loaded = () => load(MODEL_FILE); + // The entry of entity `name`, and the data of its semantic-entity and schema + // aspects. + const entity = (model: SemanticModel, name: string) => + entry(model, `sales/entities/${name}`); + const entityData = (model: SemanticModel, name: string) => + data(model, `sales/entities/${name}`, 'semantic-entity'); + const schema = (model: SemanticModel, name: string, opts = V2) => + data(model, `sales/entities/${name}`, 'schema', opts); + const names = (aspect: any) => aspect.fields.map((f: any) => f.name); + + test('a concrete entity\'s semantic-entity holds its source as written, ' + + 'its keys, direct parents and synonyms', + () => { + const model = loaded(); + expect(entityData(model, 'customers')).toEqual({ + source: {resource: TABLE('customers')}, + primaryKey: {fields: ['id']}, + uniqueConstraints: [ + {fields: ['tenant', 'email']}, + {fields: ['`legacy id`']}, + ], + extends: ['person'], + synonyms: ['buyers', 'clients'], + }); + model.entities.find(e => e.name === 'customers')!.extends = + ['person', 'party']; + expect(entityData(model, 'customers').extends) + .toEqual(['person', 'party']); + }); + + test('an abstract entity gets an entry, with abstract set and no source ' + + 'or missing-key warning', + () => { + const model = loaded(); + const {warnings} = generateCatalogResources(model, V2); + expect(entityData(model, 'party')) + .toEqual({abstract: true, synonyms: ['counterparty']}); + expect(entityData(model, 'person')).toEqual({ + extends: ['party'], + abstract: true, + }); + expect(warnings.filter(w => w.includes('abstract'))).toEqual([]); + expect(warnings.filter(w => w.includes('no keys'))) + .toEqual([NO_KEYS('events')]); + }); + + test('the missing-key warning counts unique keys and keys in a named ' + + 'profile, and with the flag off only a primary key', + () => { + const cases: [string, string, SemanticModel['profiles'], boolean][] = [ + ['only unique keys', 'visits', undefined, false], + ['a primary key in a profile', 'events', [ + ENTITIES('ops', 'orders'), + ENTITIES('prod', 'events, primary_key: [event_id]'), + ], false], + ['a unique key in a profile', 'events', + [ENTITIES('prod', 'events, unique_keys: [[at, kind]]')], false], + ['a profile with no key', 'events', [ENTITIES( + 'prod', `events, source: ${TABLE('events_prod')}`, + 'visits, primary_key: [visit_no]')], true], + ]; + for (const [name, entityName, profiles, warns] of cases) { + const model = loaded(); + model.profiles = profiles; + expect(generateCatalogResources(model, V2).warnings.filter( + w => w === NO_KEYS(entityName)), name) + .toEqual(warns ? [NO_KEYS(entityName)] : []); + } + expect(generateCatalogResources(loaded(), OPTS).warnings) + .toContain(NO_KEYS('visits')); + }); + + test('a profile record holds what its profile states: the source as ' + + 'written, excludedFields and keys, leaving out a unique key with ' + + 'no column', + () => { + const model = loaded(); + model.profiles = [ + ENTITIES( + 'prod', + `customers, source: ${TABLE('customers_prod')}, ` + + 'primary_key: [cust_key], fields_exclude: [email], ' + + 'unique_keys: [[tenant_id, mail]]'), + ENTITIES('ops', 'orders'), + ENTITIES('dev', `customers, source: ${TABLE('customers_dev')}`), + ]; + expect(entityData(model, 'customers').profiles).toEqual([ + { + name: 'prod', + source: {resource: TABLE('customers_prod')}, + excludedFields: ['email'], + primaryKey: {fields: ['cust_key']}, + uniqueConstraints: [{fields: ['tenant_id', 'mail']}], + }, + {name: 'dev', source: {resource: TABLE('customers_dev')}}, + ]); + expect(entityData(model, 'orders').profiles).toEqual([{name: 'ops'}]); + expect(entityData(model, 'events').profiles).toBeUndefined(); + const uri = TABLE('orders_prod'); + model.profiles = [ENTITIES( + 'prod', `orders, source: ${uri}`, + 'events, unique_keys: [[], [at]]')]; + expect(entityData(model, 'orders').profiles) + .toEqual([{name: 'prod', source: {resource: uri}}]); + expect(entityData(model, 'events').profiles).toEqual([ + {name: 'prod', uniqueConstraints: [{fields: ['at']}]}, + ]); + }); + + test('a concrete entity lists its own fields, then the inherited ones ' + + 'nearest ancestor first, each with its resolved definition and ' + + 'no keys or semantics block, even under emitExpressions, and ' + + 'an abstract entity lists its own fields only', + () => { + for (const opts of [V2, V2_EXPR]) { + const aspect = schema(loaded(), 'customers', opts); + expect(Object.keys(aspect)).toEqual(['fields']); + expect(aspect.fields).toEqual([ + ID, + {name: 'signup', dataType: 'Date', metadataType: 'DATETIME', + semantic: 'DIMENSION', annotations: {is_time: 'true'}}, + {name: 'region', dataType: 'Opaque', metadataType: 'OTHER', + semantic: 'DIMENSION'}, + {name: 'active', dataType: 'Boolean', metadataType: 'BOOLEAN', + semantic: 'DIMENSION', annotations: {is_time: 'false'}}, + {name: 'email', dataType: 'Opaque', metadataType: 'OTHER'}, + BIRTH, NAME, + ]); + } + const model = loaded(); + expect(schema(model, 'party').fields).toEqual([ID, NAME]); + expect(schema(model, 'person').fields).toEqual([BIRTH]); + }); + + test('a field a profile excludes, its own, inherited or on an entity ' + + 'with no parent, is still listed, so an action projecting it ' + + 'does not warn', + () => { + const applied = applyProfileFile( + load(MODEL_FILE + ACTIONS({ + Notify: ['{name: who, concept: customers, field: name}'], + })), + 'name: prod\nentities: [{name: customers, ' + + 'fields_exclude: [signup, name]}]', + 'prod'); + if ('error' in applied) throw new Error(applied.error); + const {model} = applied; + const customers = model.entities.find(e => e.name === 'customers')!; + expect(customers.excludedFields).toEqual(['signup', 'name']); + expect(names(schema(model, 'customers'))).toEqual([ + 'id', 'signup', 'region', 'active', 'email', 'birth_date', 'name', + ]); + const who = (opts: KcGenerateOptions) => + generateCatalogResources(model, opts) + .warnings.filter(w => w.includes('parameter \'who\'')); + expect(who(V2)).toEqual([]); + // With the flag off the schema leaves the field out, so it warns. + expect(who(OPTS)).not.toEqual([]); + const flat = modelWithField(undefined); + flat.entities[0].fields.push({name: 'secret'}); + flat.entities[0].excludedFields = ['secret']; + expect(names(data(flat, 'm/entities/e', 'schema'))) + .toEqual(['f', 'secret']); + }); + + test('an entity carries its instructions, custom members, third-party ' + + 'blocks and examples at entry level, and a field with them gets ' + + 'the matching aspect at its path, apart from the entity\'s own', + () => { + const aspects = entity(loaded(), 'customers').aspects!; + const ID_ASPECTS: Record = { + guidelines: {...GUIDE('Never show raw ids.'), data: {pii: true}}, + 'semantic-extensions': {customExtensions: [DBT]}, + overview: EXAMPLES('Party 7'), + }; + expect(aspects[KEY('guidelines')].data).toEqual( + {...GUIDE('Use for customer analytics.'), data: {tier: 'gold'}}); + for (const [type, want] of Object.entries(ID_ASPECTS)) { + expect(aspects[AT(type, 'id')], type).toEqual( + {aspectType: `${DATAPLEX}/aspectTypes/${type}`, data: want}); + } + const keys = (model: SemanticModel, name: string) => + Object.keys(entity(model, name).aspects!); + const HEAD = [KEY('semantic-entity'), KEY('schema')]; + const OWN = ['guidelines', 'semantic-extensions', 'overview'].map(KEY); + const FIELD = Object.keys(ID_ASPECTS).map(type => AT(type, 'id')); + const SQL = KEY('sql-expressions'); + expect(Object.keys(aspects)) + .toEqual([...HEAD, ...OWN, ...FIELD, SQL]); + expect(keys(loaded(), 'orders')).toEqual([...HEAD, SQL]); + // The abstract entity that declares the field carries them too. + expect(keys(loaded(), 'party')).toEqual([...HEAD, ...FIELD]); + }); + + test('an inherited field\'s unstorable member warns once, under the ' + + 'entity that declares it', + () => { + const model = load(MODEL_FILE.replace('{pii: true}', '{pii: .inf}')); + expect(generateCatalogResources(model, V2).warnings.filter( + w => w.includes('ai_context member'))) + .toEqual([`field 'party.id': ai_context member 'pii' is not ` + + `pushed: its value holds .inf, which Knowledge ` + + `Catalog cannot store as written`]); + }); +}); + + +describe('with KC_V2_ASPECTS on, field expressions', () => { + // A sql-expressions record, and an aspect of such records. `qualifier` names + // the profile a record comes from and is unset for the entity's own expression. + const record = (qualifier?: string) => + (name: string, sql: string, dialect?: string) => + ({name, qualifier, dialect, sql}); + const [own, dev, prod, qa] = + [record(), record('dev'), record('prod'), record('qa')]; + const SQL_ASPECT = (...fieldExpressions: object[]) => + ({aspectType: `${DATAPLEX}/aspectTypes/sql-expressions`, + data: {fieldExpressions}}); + const MODEL_FILE = `version: "0.2.0.dev0/google" +semantic_model: + - name: sales + entities: + - name: customers + source: ${TABLE('customers')} + primary_key: [id] + fields: + - { name: id, expression: id } + - { name: email, expression: LOWER(email) } + - name: score + expression: {dialects: [ + {dialect: BIGQUERY, expression: 'SAFE_DIVIDE(a, b)'}, + {dialect: ANSI_SQL, expression: a / b}]} + - { name: note } + - name: vip_customers + extends: [customers] + source: ${TABLE('vip_customers')} + primary_key: [id] + fields: [{ name: id, expression: vip_id }] + - name: events + source: ${TABLE('events')} + primary_key: [at] + fields: [{ name: at }] +`; + const CUSTOMERS = [ + own('id', 'id'), own('email', 'LOWER(email)'), + own('score', 'SAFE_DIVIDE(a, b)', 'BIGQUERY'), + own('score', 'a / b', 'ANSI_SQL'), + ]; + + const loaded = () => load(MODEL_FILE); + // Each entity entry's sql-expressions aspect, or null, by entity name. + const sqlExpressions = (model: SemanticModel) => + Object.fromEntries(generateCatalogResources(model, V2) + .entries.filter(e => e.entryType.endsWith('/semantic-entity')) + .map(e => [e.entrySource!.displayName, + e.aspects![KEY('sql-expressions')] ?? null])); + + test('an entity\'s own field expressions become records in field order, ' + + 'then each named profile that binds a field adds records in ' + + 'declaration order with its name in qualifier, and an entity with ' + + 'none gets no sql-expressions aspect', + () => { + const model = loaded(); + expect(sqlExpressions(model)).toEqual({ + customers: SQL_ASPECT(...CUSTOMERS), + vip_customers: SQL_ASPECT(own('id', 'vip_id')), + events: null, + }); + model.profiles = [ + ENTITIES( + 'prod', + 'customers, fields: [{name: email, expression: mail}, ' + + '{name: score, expression: {dialects: [' + + '{dialect: SPANNER, expression: "a / NULLIF(b, 0)"}, ' + + '{dialect: ANSI_SQL, expression: a / b}]}}]', + 'vip_customers, fields: [{name: email, expression: vip_mail}]'), + ENTITIES('ops', `customers, source: ${TABLE('customers_ops')}`), + ENTITIES( + 'dev', + 'customers, fields: [{name: email, expression: mail}, ' + + '{name: id, expression: dev_id}]'), + ENTITIES('qa', 'customers, fields: [{name: id, expression: qa_id}]'), + ]; + expect(sqlExpressions(model)).toEqual({ + customers: SQL_ASPECT( + ...CUSTOMERS, prod('email', 'mail'), + prod('score', 'a / NULLIF(b, 0)', 'SPANNER'), + prod('score', 'a / b', 'ANSI_SQL'), dev('email', 'mail'), + dev('id', 'dev_id'), qa('id', 'qa_id')), + vip_customers: + SQL_ASPECT(own('id', 'vip_id'), prod('email', 'vip_mail')), + events: null, + }); + }); +}); From a6ec7cac2378e21c8a67644e6dfd8f0bc80bb535 Mon Sep 17 00:00:00 2001 From: Fei Xu Date: Fri, 23 Oct 2026 08:08:42 +0000 Subject: [PATCH 2/2] feat(semantic): add extends links and entity overview sidecars With KC_V2_ASPECTS on, an entity's schema lists only the fields it declares, and a field's aspects sit on the entity that declares it. A reference link runs from each entity to each parent it extends, with an id that starts with semantic-extends-. Push owns those links as it owns the semantic-relationship- ones. Push reads an entity's overview sidecar, /entities/.overview.md. An entity's update lists overview and overview@*, so push removes an overview the workspace no longer gives. With the flag off, the schema aspect is built as before. Every rule here was settled by Fei Xu with Dmitry Lychagin, the author of the mapping. --- .../mdcode/docs/semantic-model/reference.md | 55 ++- .../src/libts/layouts/semantic-model.ts | 14 +- .../semantic/deploy_knowledge_catalog.ts | 33 +- .../mdcode/src/libts/semantic/kc_actions.ts | 4 +- .../mdcode/src/libts/semantic/kc_entries.ts | 32 +- toolbox/mdcode/src/libts/semantic/kc_ids.ts | 2 +- .../src/libts/semantic/knowledge_catalog.ts | 94 ++-- .../src/libts/semantic/resolve_profiles.ts | 2 +- .../semantic/deploy_knowledge_catalog.test.ts | 65 +-- .../tests/libts/semantic/kc_entries.test.ts | 20 +- .../tests/libts/semantic/kc_ids.test.ts | 2 + .../libts/semantic/knowledge_catalog.test.ts | 463 +++++++----------- .../semantic/semantic_model_layout.test.ts | 8 +- 13 files changed, 342 insertions(+), 452 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model/reference.md b/toolbox/mdcode/docs/semantic-model/reference.md index 72c0eae91..58ac34275 100644 --- a/toolbox/mdcode/docs/semantic-model/reference.md +++ b/toolbox/mdcode/docs/semantic-model/reference.md @@ -357,10 +357,10 @@ the shared label's signature is present on every subtype table. An abstract entity that no concrete entity extends has nothing to attach to and is dropped with a warning. `abstract` is an explicit marker: a non-abstract entity with no `source` is treated as a binding error and fails the push, never -silently dropped as if it were table-less. The Knowledge Catalog leg does not -model inheritance today, so an abstract entity has no physical resource to -catalog and is skipped there (with a warning); its concrete subtypes are -published normally. +silently dropped as if it were table-less. With the first-generation types, the +Knowledge Catalog leg skips an abstract entity with a warning and publishes its +concrete subtypes. With `KC_V2_ASPECTS=1`, every entity gets an entry, and a +`reference` link runs from each entity to each parent it extends. A supertype **may** instead be concrete — carry its own `source` and key. It then becomes both its own node table and a label on its subtypes. Every subtype table @@ -518,27 +518,35 @@ its own. Entry ids are ``, `/entities/`, A `semantic-link` joins each ordered pair of related entities. Built-in `reference` links run from each relationship's entry to the entries of its -source and target entities. An unbound relationship gets its `reference` links -too. Their ids start with `semantic-relationship-`, followed by the first 32 -hex characters of a SHA-256 hash of `//source` or -`//target`. Push owns only the `reference` links with that -prefix. It deletes the ones the model no longer has and leaves every other -`reference` link alone. +source and target entities, and from each entity's entry to the entry of each +parent in its `extends`. An unbound relationship gets its `reference` links +too. Their ids start with `semantic-relationship-` or `semantic-extends-`, +followed by the first 32 hex characters of a SHA-256 hash. The hash covers +`//source` or `//target` for a +relationship, and `//` for a parent. Push owns only the +`reference` links with those prefixes. It deletes the ones the model no longer +has and leaves every other `reference` link alone. + +An entity's `schema` aspect lists only the fields it declares. A field it +inherits and only rebinds is a `fieldExpressions[]` record in its +`sql-expressions` aspect, with nothing in `schema`. A field's `guidelines`, +`overview` and `semantic-extensions` aspects sit at `Schema.` on the +entity that declares the field. `semantic-metric.entity` holds the metric's entity, declared or derived. `excludedInProfiles` follows the order of the model entry's `profiles` list. -Push writes the whole `overview` aspect of the model's entry and of each metric -and relationship entry from the workspace, and never reads it back. `examples` -come from the model file's `ai_context.examples`. `content`, `contentType` and -`links` come from the entry's sidecar, the file `.overview.md` in the -directory that holds `.yaml`. For example, the model's sidecar is -`.overview.md` and a metric's is `/metrics/.overview.md`. -The sidecar's frontmatter holds `contentType` and `links`, and its body is -`content`. `contentType` is `MARKDOWN` or `HTML`. Each link has a `url` and an -optional `title`. A malformed sidecar fails the push before anything is -written. Push warns about a sidecar that names no entry of the model. An -overview edited in the console does not survive a push. +Push writes each entry's whole `overview` aspect from the workspace and never +reads it back. `examples` come from the model file's `ai_context.examples`. +`content`, `contentType` and `links` come from the entry's sidecar. The +sidecar is the file `.overview.md` in the directory that holds +`.yaml`, so the model's sidecar is `.overview.md` and an +entity's is `/entities/.overview.md`. The sidecar's frontmatter holds +`contentType` and `links`, and its body is `content`. `contentType` is +`MARKDOWN` or `HTML`. Each link has a `url` and an optional `title`. A +malformed sidecar fails the push before anything is written. Push warns about +a sidecar that names no entry of the model. A field's overview holds its +examples alone. An overview edited in the console does not survive a push. ## Validation @@ -779,13 +787,14 @@ and each aspect type attached, so a push needs, on the destination entry group: * `dataplex.entryLinks.create` and `dataplex.entryLinks.delete` — the `schema-join` links, when the model has relationships * `dataplex.entryGroups.useSchemaAspect` — every entity carries the built-in - `schema` aspect (its fields, keys, unique keys, and labels) + `schema` aspect (its fields, keys, unique keys, and labels; with + `KC_V2_ASPECTS=1`, the fields it declares and no keys) * `dataplex.entryGroups.useGuidelinesAspect` — when the model, an entity, or a metric carries `ai_context.instructions` * `dataplex.entryGroups.useSchemaJoinAspect` and `dataplex.entryGroups.useSchemaJoinEntryLink` — when the model has relationships * `dataplex.entryGroups.useReferenceEntryLink` — with `KC_V2_ASPECTS=1`, when - the model has relationships + the model has relationships or an entity extends another * the `useAspect` permission for the `semantic-model`, `semantic-entity`, and `semantic-metric` aspect types the push attaches — i.e. `dataplex.entryGroups.useSemanticModelAspect`, `useSemanticEntityAspect`, and diff --git a/toolbox/mdcode/src/libts/layouts/semantic-model.ts b/toolbox/mdcode/src/libts/layouts/semantic-model.ts index 7ac3e864a..8bb81020e 100644 --- a/toolbox/mdcode/src/libts/layouts/semantic-model.ts +++ b/toolbox/mdcode/src/libts/layouts/semantic-model.ts @@ -2,10 +2,10 @@ // // The whole model definition is a single Apache Ossie document per model, // located at `catalog/EntryGroups//.yaml`, with its -// profile files beside it. The model's Knowledge Catalog entry and each metric -// and relationship entry may have an overview sidecar at its entry id under -// the same directory: `.overview.md` for the model and, for example, -// `/metrics/.overview.md` for a metric. +// profile files beside it. Each Knowledge Catalog entry of the model may have +// an overview sidecar at its entry id under the same directory: +// `.overview.md` for the model and, for example, +// `/entities/.overview.md` for an entity. // import * as glob from 'glob'; @@ -175,15 +175,15 @@ export class SemanticModelLayout implements CatalogLayout { ])); } - // Each overview sidecar on disk at a model, metric or relationship path, - // with the entry id its path names, in path order. + // Each overview sidecar on disk at a model, entity, metric or relationship + // path, with the entry id its path names, in path order. overviewSidecarPaths(): {entryId: string; path: string}[] { if (!this._entryGroup) return []; const dir = this._groupDir(); const files = glob.globSync( [ `*${OVERVIEW_SIDECAR}`, - `*/{metrics,relationships}/*${OVERVIEW_SIDECAR}`, + `*/{entities,metrics,relationships}/*${OVERVIEW_SIDECAR}`, ], {cwd: dir, nodir: true, posix: true}); return files.sort().map(f => ({ diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 5e314414d..943d5dd63 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -31,11 +31,12 @@ // * Relationship edges are published as entry links between two entity // entries after that model's entries exist: a `schema-join` link per // relationship, or with v2Aspects a `semantic-link` per ordered entity -// pair and a `reference` link from each relationship entry to each of its -// entities. A re-push first deletes any orphaned links (no longer emitted) -// and stale links (whose type or endpoint references changed), then -// creates new or recreated links and updates unchanged links' aspects in -// place. The caller additionally needs +// pair, a `reference` link from each relationship entry to each of its +// entities, and one from each entity to each parent it extends. A re-push +// first deletes any orphaned links (no longer emitted) and stale links +// (whose type or endpoint references changed), then creates new or +// recreated links and updates unchanged links' aspects in place. The +// caller additionally needs // `dataplex.entryGroups.useSchemaJoinEntryLink` and `useSchemaJoinAspect`, // and with v2Aspects `useReferenceEntryLink`, on the destination entry // group. @@ -565,8 +566,7 @@ function instanceCountError(entry: Entry): string|undefined { return `entry '${entryId(entry)}' would hold ${count} ${type} aspects, ` + `and Knowledge Catalog returns at most ${MAX_ASPECTS_PER_READ} in ` + `one read, so kcmd could not read them all back; give ${content} ` + - `to fewer than ${MAX_ASPECTS_PER_READ} of the entity and its ` + - `fields, inherited fields included.`; + `to fewer than ${MAX_ASPECTS_PER_READ} of the entity and its fields.`; } return undefined; } @@ -1637,12 +1637,12 @@ const KEPT_SOURCE_FIELDS = * name. Passing `entryAspectKeys(entry, opts)` with `deleteMissingAspects: * true` makes a re-push converge: a still-present optional aspect is refreshed, * and a removed one (such as `guidelines` when `ai_context.instructions` is - * deleted, `sql-expressions`, or a field's `guidelines` or - * `semantic-extensions`, matched by its `@*` key under `v2Aspects`) is deleted - * so a later `pull` does not bring it back. Push owns the optional aspects - * `entryAspectKeys` lists, so it removes any that the model does not declare - * even if someone added them in the console. Every other aspect another tool - * attached is left alone, since `aspectKeys` names only the types kcmd writes. + * deleted, `sql-expressions`, or a field's instance, matched by its `@*` key + * under `v2Aspects`) is deleted so a later `pull` does not bring it back. Push + * owns the optional aspects `entryAspectKeys` lists, so it removes any that the + * model does not declare even if someone added them in the console. Every other + * aspect another tool attached is left alone, since `aspectKeys` names only the + * types kcmd writes. * * The update mask replaces the whole entrySource, so the update carries over * fields from `source`, the entrySource the entry had before this push. Of @@ -1669,13 +1669,6 @@ async function updateEntry( ...(Object.keys(labels).length ? {labels} : {}), }; if (Object.keys(merged).length) entry = {...entry, entrySource: merged}; - // We cannot simply pass `Object.keys(entry.aspects ?? {})` here because - // `entry.aspects` only contains the aspects emitted in *this* push. If an - // optional aspect (such as `guidelines`, `sql-expressions`, or a field's - // `guidelines@Schema.`) was present on a previous push and removed in - // this push, it is absent from `entry.aspects` -- and Dataplex only deletes a - // removed aspect when its key (or its `@*` key, for a field's instance) is - // still listed in `aspectKeys`. const aspectKeys = entryAspectKeys(entry, opts); const upd = await cat.updateEntry( entry, ['entry_source', 'aspects'], [...aspectKeys], diff --git a/toolbox/mdcode/src/libts/semantic/kc_actions.ts b/toolbox/mdcode/src/libts/semantic/kc_actions.ts index fa3f533ff..33bcd96e1 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_actions.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_actions.ts @@ -145,8 +145,8 @@ export function actionEntries( const relationshipNames = new Set((model.relationships ?? []).map(r => r.name)); - // The fields this push actually writes into a `schema` aspect, which is - // what a later pull reads back. Asked of the loader's own resolver so the + // The fields of each concept with inheritance resolved, which a later pull + // rebuilds from the catalog. Asked of the loader's own resolver so the // answer accounts for `extends`; a model that will not resolve returns null // and the field half of the check is skipped, since an emitter is not the // place to discover that a model does not load. diff --git a/toolbox/mdcode/src/libts/semantic/kc_entries.ts b/toolbox/mdcode/src/libts/semantic/kc_entries.ts index bb6398b5b..7cbd4efc5 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_entries.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_entries.ts @@ -150,7 +150,7 @@ export function isDependentEntry(entry: Entry): boolean { /** * Returns true when `link` is of a kind push writes for a model: `schema-join` * in V1, and in V2 `semantic-link` or a `reference` link whose ID starts with - * `semantic-relationship-`. + * `semantic-relationship-` or `semantic-extends-`. * * Includes both V1 and V2 regardless of `v2Aspects`, so a model pushed with V1 * and then V2 still discovers and cleans up its old `schema-join` links @@ -160,7 +160,7 @@ export function isModelLink(link: EntryLink): boolean { const type = linkTypeId(link); return type === 'schema-join' || type === 'semantic-link' || (type === 'reference' && - /^semantic-relationship-/.test(linkId(link))); + /^semantic-(relationship|extends)-/.test(linkId(link))); } // --------------------------------------------------------------------------- @@ -262,41 +262,33 @@ function systemTypeProjectAndLocation( * * Includes every aspect key present on `entry.aspects` plus the optional * `..guidelines` aspect. When `opts?.v2Aspects` is true: - * * Adds `..semantic-extensions` on model, entity, metric and - * relationship entries, so Dataplex removes the aspect when the object no - * longer has a third-party extension block. - * * Adds `..overview` on model, metric and relationship entries, - * so Dataplex removes the aspect when the object no longer has content - * for it. + * * Adds `..semantic-extensions` and `..overview` on + * model, entity, metric and relationship entries, so Dataplex removes the + * aspect when the object no longer has content for it. * * Adds `..sql-expressions` on entity and metric entries. - * * Adds `..guidelines@*` and `..semantic-extensions@*` - * on entity entries, so Dataplex removes a field's instance the entry no - * longer carries. These wildcards also write the field-path instances - * `entry` carries, so those keys are left out. + * * Adds `..@*` for guidelines, semantic-extensions and + * overview on entity entries, so Dataplex writes each field's instance + * `entry` carries and removes the others. A field's own key is left out. */ export function entryAspectKeys( entry: Entry, opts?: SystemTypeOptions): Set { const {proj, loc} = systemTypeProjectAndLocation(opts); const keys = new Set([ - ...Object.keys(entry.aspects ?? {}), + ...Object.keys(entry.aspects ?? {}).filter(key => !key.includes('@')), `${proj}.${loc}.guidelines`, ]); if (!opts?.v2Aspects) return keys; if (isSemanticObjectEntry(entry)) { keys.add(`${proj}.${loc}.semantic-extensions`); - if (!isEntityEntry(entry)) keys.add(`${proj}.${loc}.overview`); + keys.add(`${proj}.${loc}.overview`); } if (isEntityEntry(entry) || isMetricEntry(entry)) { keys.add(`${proj}.${loc}.sql-expressions`); } if (isEntityEntry(entry)) { - for (const type of ['guidelines', 'semantic-extensions']) { - const prefix = `${proj}.${loc}.${type}@`; - for (const key of keys) { - if (key.startsWith(prefix)) keys.delete(key); - } - keys.add(`${prefix}*`); + for (const type of ['guidelines', 'semantic-extensions', 'overview']) { + keys.add(`${proj}.${loc}.${type}@*`); } } return keys; diff --git a/toolbox/mdcode/src/libts/semantic/kc_ids.ts b/toolbox/mdcode/src/libts/semantic/kc_ids.ts index e6655a8da..607f05edb 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_ids.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_ids.ts @@ -150,7 +150,7 @@ export function relationshipLinkId(modelName: string, relName: string): string { * releases, so that push finds the links it wrote. */ export function semanticLinkId( - kind: 'link'|'relationship', ...parts: string[]): string { + kind: 'link'|'relationship'|'extends', ...parts: string[]): string { const digest = createHash('sha256').update(parts.join('/'), 'utf8').digest('hex'); return `semantic-${kind}-${digest.slice(0, 32)}`; diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index a4804ac92..1af33b576 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -75,7 +75,8 @@ // `/relationships/`, that carries semantic-relationship. A // `semantic-link` entry link joins each ordered pair of related entities and // lists their relationships in declaration order. A `reference` link runs -// from each relationship entry to each of its two entities. +// from each relationship entry to each of its two entities, and from each +// entity entry to the entry of each parent it extends. // import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; @@ -89,7 +90,8 @@ import {customAspectKey, customAspectTypeName, customEntryTypeName, FILE_COPY_TY import {linkSlug, Namer, ownedEntryIdPrefixes, semanticLinkId} from './kc_ids'; import {asResourceUri, GOOGLE_VERSION, isAcceptedSource} from './loader'; import {nonEmpty} from './osi_converter'; -import {DEFAULT_PROFILE, resolvedEntities, statesKey, withoutExclusions} from './resolve_profiles'; +import {InheritanceError, resolveInheritance} from './resolve_inheritance'; +import {DEFAULT_PROFILE, inheritedNamesOf, statesKey, withoutExclusions} from './resolve_profiles'; export interface KcGenerateOptions { project: string; // project the entries are created in (destination) @@ -139,10 +141,10 @@ export interface KcResources { * Generates the Knowledge Catalog resources for a semantic model. * * Returns the entries (model anchor first, then entities, metrics and, with - * v2Aspects, relationships), the relationships' entry links, and any warnings - * collected while mapping the IR (missing keys, un-typed metrics, skipped M:N - * relationships). The resources reference the built-in system types; they do not - * create them. + * v2Aspects, relationships), the entry links of the relationships and, with + * v2Aspects, of each entity's parents, and any warnings collected while + * mapping the IR (missing keys, un-typed metrics, skipped M:N relationships). + * The resources reference the built-in system types; they do not create them. */ export function generateCatalogResources( model: SemanticModel, opts: KcGenerateOptions): KcResources { @@ -170,10 +172,9 @@ export function generateCatalogResources( 'preserves them'); } - // Each entity's fields with inheritance resolved: the second-generation - // schema lists them, and the first-generation one takes an inherited - // definition from them. - const resolved = resolvedFields(model); + // With the first-generation aspects, a subtype field that only rebinds an + // inherited field is published with the definition it inherits. + const inherited = v2 ? undefined : inheritedFields(model); // Entry ids must be unique within the entry group. Two source names that // normalize to the same id (see slug), or exact duplicates the loader did not @@ -219,7 +220,6 @@ export function generateCatalogResources( if (!claim(seen, entityId, 'entry', `entity '${entity.name}'`, warnings)) continue; entityEntryName.set(entity.name, names.entry(entityId)); - const fields = resolved.get(entity)!; if (v2 ? lacksKeys(model, entity) : !entity.keys || !entity.keys.length) { warnings.push( `entity '${entity.name}': no keys declared in the source model`); @@ -231,11 +231,12 @@ export function generateCatalogResources( entrySource: source(entity.name, entity.description), // semantic-entity and schema are required. The file header lists the // optional aspects. - aspects: v2 ? aspectMap(names, entityAspectsV2(model, entity, fields)) : + aspects: v2 ? aspectMap(names, entityAspectsV2(model, entity)) : aspectsFor( names, { 'semantic-entity': entityAspectData(entity), - 'schema': schemaAspectData(entity, emitExpr, fields), + 'schema': schemaAspectData( + entity, emitExpr, inherited?.get(entity.name)), }, entity.aiContext), }); @@ -343,7 +344,11 @@ export function generateCatalogResources( } } - const entryLinks = v2Relationships?.links ?? + const entryLinks = v2Relationships ? + [ + ...v2Relationships.links, + ...extendsLinks(names, model, entityEntryName), + ] : schemaJoinLinks(names, model, entityEntryName, warnings); return { @@ -453,7 +458,7 @@ function relationshipResourcesV2( const pairs = new Map(); // Push never reaches a name with no entry: validation rejects an unknown - // entity, and a relationship on an entity that is not a concrete leaf. + // entity, and every entity has an entry under these types. for (const rel of model.relationships ?? []) { if (rel.association) { warnings.push(`relationship '${rel.name}': ${ASSOCIATION_REASON}`); @@ -492,6 +497,19 @@ function relationshipResourcesV2( return {entries, links: [...links, ...references]}; } +// The reference link from each entity entry to the entry of each parent it +// extends. +function extendsLinks( + names: Namer, model: SemanticModel, + entityEntryName: ReadonlyMap): EntryLink[] { + return (model.entities ?? []).flatMap( + e => (e.extends ?? []).map( + parent => directedLink( + names, 'reference', + semanticLinkId('extends', model.name, e.name, parent), + entityEntryName.get(e.name)!, entityEntryName.get(parent)!))); +} + // A link of type `type` with ID `id` from the entry `from` to the entry `to`. function directedLink( names: Namer, type: string, id: string, from: string, @@ -698,15 +716,31 @@ function unstorableValue(value: unknown): string|undefined { return Object.values(value).map(unstorableValue).find(Boolean); } -// Each entity's fields with inheritance resolved and every profile exclusion -// ignored. A concrete entity has its own fields, then the ones it inherits, and -// an abstract entity has its own. -function resolvedFields(model: SemanticModel): Map { - const entities = model.entities ?? []; - const resolved = resolvedEntities(withoutExclusions(model)); - // resolvedEntities keeps the entities in order. - return new Map( - entities.map((e, i) => [e, e.abstract ? e.fields : resolved[i].fields])); +// Each entity's fields with inheritance resolved, by entity and field name, or +// undefined when no entity extends another or inheritance cannot be resolved. +// A resolved field keeps what the entity's own declaration sets and takes the +// rest from the ancestor that defines it. +function inheritedFields(model: SemanticModel): + Map>|undefined { + if (!(model.entities ?? []).some(e => e.extends?.length)) return undefined; + let resolved: SemanticModel; + try { + resolved = resolveInheritance(model).model; + } catch (err) { + // Push validation reports it. The schema aspect keeps what each entity + // declares. + if (err instanceof InheritanceError) return undefined; + throw err; + } + return new Map((resolved.entities ?? []).map( + e => [e.name, new Map((e.fields ?? []).map(f => [f.name, f]))])); +} + +// The fields `entity` declares, leaving out each line that only rebinds a +// field it inherits. +function declaredFields(model: SemanticModel, entity: Entity): Field[] { + const inherited = inheritedNamesOf(model, entity.name); + return entity.fields.filter(f => !inherited.has(f.name)); } // True when `entity` is concrete and states no primary key and no unique key, @@ -769,7 +803,7 @@ function entityAspectData(entity: Entity): Record { // it inherits from there. function schemaAspectData( entity: Entity, emitExpressions: boolean, - inherited: Field[]): Record { + inherited?: Map): Record { // Key column lists ride the schema aspect verbatim -- they need NOT be // declared fields (the closed template accepts and round-trips columns absent // from fields[], live-verified). Keep only non-empty strings, matching the @@ -786,7 +820,7 @@ function schemaAspectData( // A subtype's redeclaration of an inherited field only rebinds it, so its // definition is the inherited one. const withDefinition = (f: Field): Field => { - const from = inherited.find(g => g.name === f.name); + const from = inherited?.get(f.name); return { ...f, type: f.type ?? from?.type, @@ -925,11 +959,11 @@ function modelAspectDataV2(model: SemanticModel): Record { }); } -// The aspects of an entity entry under the second-generation types, with -// `fields` the fields its schema lists, each with its own aspects at its path. +// The aspects of an entity entry under the second-generation types. Its schema +// lists the fields it declares, each with its own aspects at its path. function entityAspectsV2( - model: SemanticModel, entity: Entity, - fields: Field[]): Record> { + model: SemanticModel, entity: Entity): Record> { + const fields = declaredFields(model, entity); return compact({ 'semantic-entity': entityAspectDataV2(model, entity), 'schema': schemaAspectDataV2(fields), diff --git a/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts b/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts index 62cc9b0a4..ea4fa9033 100644 --- a/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts +++ b/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts @@ -883,7 +883,7 @@ export function resolvedEntities(model: SemanticModel): Entity[] { // The names of the fields `entityName` inherits from every ancestor its // `extends` reaches. A caller asking for many entities passes `byName`, the // model's entities by name, built once. -function inheritedNamesOf( +export function inheritedNamesOf( model: SemanticModel, entityName: string, byName = new Map((model.entities ?? []).map(e => [e.name, e]))): Set { diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts index 2197dea56..13fcbc5b6 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -2723,8 +2723,10 @@ describe('deployKnowledgeCatalog: aspect removal on update', () => { 'dataplex-types.global.sql-expressions', 'dataplex-types.global.guidelines', 'dataplex-types.global.semantic-extensions', + 'dataplex-types.global.overview', 'dataplex-types.global.guidelines@*', 'dataplex-types.global.semantic-extensions@*', + 'dataplex-types.global.overview@*', ]); expect(entityUpdateCall![3]).toBe(true); @@ -4196,51 +4198,24 @@ describe('deployKnowledgeCatalog: the size of a V2 aspect', () => { }); } - // OSSIE with n more fields on orders, f0 to f, each with the - // ai_context `member`. - const wide = (n: number, member = 'examples: [e]') => - OSSIE.replace('- name: o_orderkey\n', Array.from( - {length: n}, (_, i) => `- { name: f${i}, expression: f${i}, ` + - `ai_context: {${member}} }\n `).join('') + - '- name: o_orderkey\n'); - // Expects a push of `text` to fail before any write, with the same error - // as the check kcmd profiles runs, and returns the error. - async function refusal(text: string) { - const {list, create} = stubClient(); - const docs = models([{name: 'sales.yaml', text}]); - const result = await deployKnowledgeCatalog(docs, CTX, V2_OPTS); - expect(result.success).toBe(false); - expect(list).not.toHaveBeenCalled(); - expect(create).not.toHaveBeenCalled(); - expect(checkCatalogAspects(docs, V2_OPTS).error).toBe(result.details); - return result.details; - } - - test('an entry that would hold 100 instances of one aspect type fails ' + - 'before any write, in push and in the check kcmd profiles runs', - async () => { - expect(await refusal(wide(100))) - .toBe( - 'entry \'sales/entities/orders\' would hold 100 overview ' + - 'aspects, and Knowledge Catalog returns at most 100 in one ' + - 'read, so kcmd could not read them all back; give examples ' + - 'to fewer than 100 of the entity and its fields, inherited ' + - 'fields included.'); - expect(checkCatalogAspects(models([{name: 's.yaml', text: wide(99)}]), - V2_OPTS).error) - .toBeUndefined(); - // Orders has 99 fields with instructions, and big_orders inherits - // them and adds one. - expect(await refusal(wide(99, 'instructions: i') - .replace(' metrics:', ` - name: big_orders - extends: [orders] - source: bigquery.googleapis.com/projects/demo/datasets/sales/tables/big - primary_key: [o_orderkey] - fields: [{ name: g, expression: g, ai_context: {instructions: g} }] - metrics:`))) - .toStartWith( - 'entry \'sales/entities/big_orders\' would hold 100 ' + - 'guidelines aspects, '); + // OSSIE with n more fields on orders, f0 to f, each with an example. + const wide = (n: number) => models([{ + name: 'sales.yaml', + text: OSSIE.replace('- name: o_orderkey\n', Array.from( + {length: n}, (_, i) => `- { name: f${i}, expression: f${i}, ` + + 'ai_context: {examples: [e]} }\n ').join('') + + '- name: o_orderkey\n'), + }]); + + test('an entry that would hold 100 instances of one aspect type fails, ' + + 'and one with 99 passes', + () => { + expect(checkCatalogAspects(wide(100), V2_OPTS).error).toBe( + 'entry \'sales/entities/orders\' would hold 100 overview ' + + 'aspects, and Knowledge Catalog returns at most 100 in one read, ' + + 'so kcmd could not read them all back; give examples to fewer ' + + 'than 100 of the entity and its fields.'); + expect(checkCatalogAspects(wide(99), V2_OPTS).error).toBeUndefined(); }); test('a semantic-link aspect is checked too', async () => { diff --git a/toolbox/mdcode/tests/libts/semantic/kc_entries.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_entries.test.ts index 904479070..617eb95df 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_entries.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_entries.test.ts @@ -163,11 +163,12 @@ describe('entry and link type predicates, anchorId, and custom types', () => { link('sales-orders-to-customer', 'schema-join'), link('sales-orders-to-customer', 'semantic-link'), link('semantic-relationship-1', 'reference'), + link('semantic-extends-1', 'reference'), link('semantic-link-1', 'reference'), link('sales-orders-to-customer', 'semantic-relationship'), link('sales-orders-to-customer', 'other-link'), ].map(isModelLink)) - .toEqual([true, true, true, false, false, false]); + .toEqual([true, true, true, true, false, false, false]); }); test( @@ -348,11 +349,14 @@ describe('entryAspectKeys', () => { test( 'with v2Aspects on, entryAspectKeys lists the V2 keys of each entry ' + - 'kind, with a field\'s keys as wildcards except its overview', + 'kind, with a field\'s keys as wildcards', () => { const entity = makeEntry('sales/entities/orders', ENTITY_TYPE, { 'dataplex-types.global.semantic-entity': {data: {}}, 'dataplex-types.global.schema': {data: {}}, + 'dataplex-types.global.guidelines@Schema.id': {data: {}}, + 'dataplex-types.global.semantic-extensions@Schema.id': {data: {}}, + 'dataplex-types.global.overview@Schema.id': {data: {}}, }); const metric = makeEntry('sales/metrics/revenue', METRIC_TYPE, { 'dataplex-types.global.semantic-metric': {data: {}}, @@ -364,12 +368,6 @@ describe('entryAspectKeys', () => { 'sales/relationships/orders_to_customer', RELATIONSHIP_TYPE, { 'dataplex-types.global.semantic-relationship': {data: {}}, }); - const withFields = makeEntry('sales/entities/orders', ENTITY_TYPE, { - ...entity.aspects, - ...Object.fromEntries( - ['guidelines', 'semantic-extensions', 'overview'].map( - t => [`dataplex-types.global.${t}@Schema.id`, {data: {}}])), - }); const entityKeys = entryAspectKeys(entity, {v2Aspects: true}); const metricKeys = entryAspectKeys(metric, {v2Aspects: true}); @@ -380,9 +378,11 @@ describe('entryAspectKeys', () => { 'dataplex-types.global.schema', 'dataplex-types.global.guidelines', 'dataplex-types.global.semantic-extensions', + 'dataplex-types.global.overview', 'dataplex-types.global.sql-expressions', 'dataplex-types.global.guidelines@*', 'dataplex-types.global.semantic-extensions@*', + 'dataplex-types.global.overview@*', ])); expect(metricKeys).toEqual(new Set([ 'dataplex-types.global.semantic-metric', @@ -404,8 +404,6 @@ describe('entryAspectKeys', () => { 'dataplex-types.global.semantic-extensions', 'dataplex-types.global.overview', ])); - expect(entryAspectKeys(withFields, {v2Aspects: true})).toEqual(new Set( - [...entityKeys, 'dataplex-types.global.overview@Schema.id'])); }); test( @@ -427,9 +425,11 @@ describe('entryAspectKeys', () => { 'staging-types.us.semantic-entity', 'staging-types.us.guidelines', 'staging-types.us.semantic-extensions', + 'staging-types.us.overview', 'staging-types.us.sql-expressions', 'staging-types.us.guidelines@*', 'staging-types.us.semantic-extensions@*', + 'staging-types.us.overview@*', ])); }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/kc_ids.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_ids.test.ts index 26b6e3196..6b3cd4027 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_ids.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_ids.test.ts @@ -88,9 +88,11 @@ describe('semanticLinkId', () => { expect([ semanticLinkId('link', 'sales', 'orders', 'customers'), semanticLinkId('relationship', 'sales', 'placed_by', 'source'), + semanticLinkId('extends', 'sales', 'vips', 'customers'), ]).toEqual([ 'semantic-link-e6b1c3bb1665161a428c2cd13a1cb443', 'semantic-relationship-7d24245079e3654a7a01f88f89d599a9', + 'semantic-extends-22fb71a480680b1a4d779ce39187b845', ]); }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts index 3969d225f..520ca3bd6 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -78,18 +78,6 @@ const ENTITIES = (name: string, ...bindings: string[]) => PROFILE( const GUIDE = (instructions: string) => ({instructions, userManaged: true}); const EXAMPLES = (...texts: string[]) => ({examples: texts.map(content => ({content}))}); -// An actions block with an MCP action for each key, whose parameters are YAML -// flow mappings. -const ACTIONS = (actions: Record) => ' actions:\n' + - Object.entries(actions) - .map(([name, params]) => ` - name: ${name} - executor: - mcp: - server: //agentregistry.googleapis.com/x/mcpServers/shop - tool: ${name.toLowerCase()} - parameters: -${params.map(p => ` - ${p}\n`).join('')}`) - .join(''); // A YAML entity over table p.d. with an `id` field and the given key. const ENTITY_YAML = (name: string, key = '[id]') => ` - name: ${name} source: ${TABLE(name)} @@ -1790,8 +1778,7 @@ ${ENTITY_YAML('addresses')} }); -describe('with KC_V2_ASPECTS on, entities and their fields', () => { - const AT = (type: string, field: string) => `${KEY(type)}@Schema.${field}`; +describe('with KC_V2_ASPECTS on, entities, fields and expressions', () => { const MODEL_FILE = `version: "0.2.0.dev0/google" semantic_model: - name: sales @@ -1804,11 +1791,8 @@ semantic_model: datatype: Integer description: The party id label: Party ID - ai_context: - instructions: Never show raw ids. - synonyms: [party_key] - examples: [Party 7] - custom: {pii: true} + ai_context: {instructions: Hide raw ids., synonyms: [party_key], + examples: [Party 7], custom: {pii: true, rank: .inf}} custom_extensions: - {vendor_name: DBT, data: '{"tests": ["not_null"]}'} - { name: name, datatype: String } @@ -1822,336 +1806,237 @@ semantic_model: primary_key: [id] unique_keys: [[tenant, email], ['\`legacy id\`']] ai_context: - instructions: Use for customer analytics. - synonyms: [buyers, clients] - examples: [Who bought the most?] - custom: {tier: gold} - custom_extensions: - - {vendor_name: DBT, data: '{"materialized": "table"}'} + {instructions: Use for analytics., synonyms: [buyers, clients], + examples: [Who bought the most?], custom: {tier: gold, rank: .nan}} + custom_extensions: [{vendor_name: DBT, data: '{"model": true}'}] fields: - { name: id, expression: cust_id } - - { name: signup, datatype: Date, expression: signup, - dimension: {is_time: true} } - - { name: region, expression: region, dimension: {} } - - { name: active, datatype: Boolean, expression: active, - dimension: {is_time: false} } - - { name: email, expression: email } - - name: orders - source: ${TABLE('orders')} - primary_key: [order_id] - fields: [{name: order_id, expression: order_id}] + - { name: signup, datatype: Date, dimension: {is_time: true} } + - {name: region, dimension: {}, expression: {dialects: [ + {dialect: BIGQUERY, expression: 'SAFE_DIVIDE(a, b)'}, + {dialect: ANSI_SQL, expression: a / b}]}} + - { name: active, datatype: Boolean, dimension: {is_time: false} } + - { name: email, expression: LOWER(email) } - name: events source: ${TABLE('events')} fields: [{name: at, expression: at}] - name: visits + extends: [events] source: ${TABLE('visits')} unique_keys: [[visit_no]] fields: [{name: visit_no, expression: visit_no}] + actions: + - name: Notify + executor: + mcp: {server: //agentregistry.googleapis.com/x/mcpServers/s, tool: n} + parameters: [{name: who, concept: customers, field: name}] `; + const loaded = () => load(MODEL_FILE); + // The data of aspect `type` on the entry of entity `name`. + const aspect = + (model: SemanticModel, name: string, type: string, opts = V2) => + data(model, `sales/entities/${name}`, type, opts); const NO_KEYS = (entity: string) => `entity '${entity}': no keys declared in the source model`; - const ID = { - name: 'id', description: 'The party id', dataType: 'Integer', - metadataType: 'NUMBER', synonyms: ['party_key'], - annotations: {label: 'Party ID'}, - }; - const NAME = {name: 'name', dataType: 'String', metadataType: 'STRING'}; - const BIRTH = - {name: 'birth_date', dataType: 'Date', metadataType: 'DATETIME'}; - const DBT = {vendorName: 'DBT', data: '{"tests": ["not_null"]}'}; - const loaded = () => load(MODEL_FILE); - // The entry of entity `name`, and the data of its semantic-entity and schema - // aspects. - const entity = (model: SemanticModel, name: string) => - entry(model, `sales/entities/${name}`); - const entityData = (model: SemanticModel, name: string) => - data(model, `sales/entities/${name}`, 'semantic-entity'); - const schema = (model: SemanticModel, name: string, opts = V2) => - data(model, `sales/entities/${name}`, 'schema', opts); - const names = (aspect: any) => aspect.fields.map((f: any) => f.name); - - test('a concrete entity\'s semantic-entity holds its source as written, ' + - 'its keys, direct parents and synonyms', + // A sql-expressions record; `qualifier` names the profile it comes from. + const record = (qualifier?: string) => + (name: string, sql: string, dialect?: string) => + ({name, qualifier, dialect, sql}); + const [own, prod, dev] = [record(), record('prod'), record('dev')]; + + test('semantic-entity holds the source and keys as written, the direct ' + + 'parents, abstract and synonyms, a reference link runs from each ' + + 'entity to each parent, and an abstract entity draws no warning', () => { const model = loaded(); - expect(entityData(model, 'customers')).toEqual({ - source: {resource: TABLE('customers')}, - primaryKey: {fields: ['id']}, - uniqueConstraints: [ - {fields: ['tenant', 'email']}, - {fields: ['`legacy id`']}, - ], - extends: ['person'], - synonyms: ['buyers', 'clients'], - }); + expect(['customers', 'party', 'person'].map( + name => aspect(model, name, 'semantic-entity'))) + .toEqual([ + { + source: {resource: TABLE('customers')}, + primaryKey: {fields: ['id']}, + uniqueConstraints: [ + {fields: ['tenant', 'email']}, {fields: ['`legacy id`']}, + ], + extends: ['person'], + synonyms: ['buyers', 'clients'], + }, + {abstract: true, synonyms: ['counterparty']}, + {extends: ['party'], abstract: true}, + ]); + const extendsLink = (child: string, parent: string) => DIRECTED( + 'reference', semanticLinkId('extends', 'sales', child, parent), + `sales/entities/${child}`, `sales/entities/${parent}`); model.entities.find(e => e.name === 'customers')!.extends = ['person', 'party']; - expect(entityData(model, 'customers').extends) + expect(aspect(model, 'customers', 'semantic-entity').extends) .toEqual(['person', 'party']); - }); - - test('an abstract entity gets an entry, with abstract set and no source ' + - 'or missing-key warning', - () => { - const model = loaded(); - const {warnings} = generateCatalogResources(model, V2); - expect(entityData(model, 'party')) - .toEqual({abstract: true, synonyms: ['counterparty']}); - expect(entityData(model, 'person')).toEqual({ - extends: ['party'], - abstract: true, - }); + const {entryLinks, warnings} = generateCatalogResources(model, V2); + expect(entryLinks).toEqual([ + extendsLink('person', 'party'), extendsLink('customers', 'person'), + extendsLink('customers', 'party'), extendsLink('visits', 'events'), + ]); expect(warnings.filter(w => w.includes('abstract'))).toEqual([]); - expect(warnings.filter(w => w.includes('no keys'))) - .toEqual([NO_KEYS('events')]); }); - test('the missing-key warning counts unique keys and keys in a named ' + - 'profile, and with the flag off only a primary key', + test('the missing-key warning skips abstract entities and counts unique ' + + 'keys and keys in a named profile, and with the flag off only a ' + + 'primary key', () => { - const cases: [string, string, SemanticModel['profiles'], boolean][] = [ - ['only unique keys', 'visits', undefined, false], - ['a primary key in a profile', 'events', [ - ENTITIES('ops', 'orders'), - ENTITIES('prod', 'events, primary_key: [event_id]'), - ], false], - ['a unique key in a profile', 'events', - [ENTITIES('prod', 'events, unique_keys: [[at, kind]]')], false], - ['a profile with no key', 'events', [ENTITIES( - 'prod', `events, source: ${TABLE('events_prod')}`, - 'visits, primary_key: [visit_no]')], true], - ]; - for (const [name, entityName, profiles, warns] of cases) { + for (const [name, profiles, warns] of [ + ['no profile', [], true], + ['a primary key in a profile', [ + ENTITIES('ops', 'visits'), + ENTITIES('prod', 'events, primary_key: [event_id]'), + ], false], + ['a unique key in a profile', + [ENTITIES('prod', 'events, unique_keys: [[at]]')], false], + ['a profile with no key', [ENTITIES( + 'prod', `events, source: ${TABLE('events_prod')}`, + 'visits, primary_key: [visit_no]')], true], + ] as [string, SemanticModel['profiles'], boolean][]) { const model = loaded(); model.profiles = profiles; expect(generateCatalogResources(model, V2).warnings.filter( - w => w === NO_KEYS(entityName)), name) - .toEqual(warns ? [NO_KEYS(entityName)] : []); + w => w.includes('no keys')), name) + .toEqual(warns ? [NO_KEYS('events')] : []); } expect(generateCatalogResources(loaded(), OPTS).warnings) .toContain(NO_KEYS('visits')); }); - test('a profile record holds what its profile states: the source as ' + - 'written, excludedFields and keys, leaving out a unique key with ' + - 'no column', + test('a named profile that lists an entity gets a record of the source, ' + + 'excludedFields and keys it states, without an empty unique key, ' + + 'and a qualified expression record for each field it binds, after ' + + 'the entity\'s own records', () => { const model = loaded(); + expect(aspect(model, 'visits', 'sql-expressions')) + .toEqual({fieldExpressions: [own('visit_no', 'visit_no')]}); model.profiles = [ ENTITIES( 'prod', `customers, source: ${TABLE('customers_prod')}, ` + - 'primary_key: [cust_key], fields_exclude: [email], ' + - 'unique_keys: [[tenant_id, mail]]'), - ENTITIES('ops', 'orders'), - ENTITIES('dev', `customers, source: ${TABLE('customers_dev')}`), + 'primary_key: [cust_key], fields_exclude: [active], ' + + 'unique_keys: [[tenant_id, mail], []], fields: [' + + '{name: email, expression: mail}, ' + + '{name: name, expression: full_name}, ' + + '{name: region, expression: {dialects: [' + + '{dialect: SPANNER, expression: "a / NULLIF(b, 0)"}, ' + + '{dialect: ANSI_SQL, expression: a / b}]}}]', + 'events, unique_keys: [[at]]'), + ENTITIES( + 'dev', + `customers, source: ${TABLE('customers_dev')}, fields: [` + + '{name: email, expression: mail}, ' + + '{name: id, expression: dev_id}]'), ]; - expect(entityData(model, 'customers').profiles).toEqual([ - { - name: 'prod', - source: {resource: TABLE('customers_prod')}, - excludedFields: ['email'], - primaryKey: {fields: ['cust_key']}, - uniqueConstraints: [{fields: ['tenant_id', 'mail']}], - }, - {name: 'dev', source: {resource: TABLE('customers_dev')}}, - ]); - expect(entityData(model, 'orders').profiles).toEqual([{name: 'ops'}]); - expect(entityData(model, 'events').profiles).toBeUndefined(); - const uri = TABLE('orders_prod'); - model.profiles = [ENTITIES( - 'prod', `orders, source: ${uri}`, - 'events, unique_keys: [[], [at]]')]; - expect(entityData(model, 'orders').profiles) - .toEqual([{name: 'prod', source: {resource: uri}}]); - expect(entityData(model, 'events').profiles).toEqual([ - {name: 'prod', uniqueConstraints: [{fields: ['at']}]}, - ]); - }); - - test('a concrete entity lists its own fields, then the inherited ones ' + - 'nearest ancestor first, each with its resolved definition and ' + - 'no keys or semantics block, even under emitExpressions, and ' + - 'an abstract entity lists its own fields only', - () => { - for (const opts of [V2, V2_EXPR]) { - const aspect = schema(loaded(), 'customers', opts); - expect(Object.keys(aspect)).toEqual(['fields']); - expect(aspect.fields).toEqual([ - ID, - {name: 'signup', dataType: 'Date', metadataType: 'DATETIME', - semantic: 'DIMENSION', annotations: {is_time: 'true'}}, - {name: 'region', dataType: 'Opaque', metadataType: 'OTHER', - semantic: 'DIMENSION'}, - {name: 'active', dataType: 'Boolean', metadataType: 'BOOLEAN', - semantic: 'DIMENSION', annotations: {is_time: 'false'}}, - {name: 'email', dataType: 'Opaque', metadataType: 'OTHER'}, - BIRTH, NAME, - ]); - } - const model = loaded(); - expect(schema(model, 'party').fields).toEqual([ID, NAME]); - expect(schema(model, 'person').fields).toEqual([BIRTH]); + expect(aspect(model, 'customers', 'semantic-entity').profiles) + .toEqual([ + { + name: 'prod', + source: {resource: TABLE('customers_prod')}, + excludedFields: ['active'], + primaryKey: {fields: ['cust_key']}, + uniqueConstraints: [{fields: ['tenant_id', 'mail']}], + }, + {name: 'dev', source: {resource: TABLE('customers_dev')}}, + ]); + expect(aspect(model, 'events', 'semantic-entity').profiles).toEqual( + [{name: 'prod', uniqueConstraints: [{fields: ['at']}]}]); + expect(aspect(model, 'visits', 'semantic-entity').profiles) + .toBeUndefined(); + expect(aspect(model, 'customers', 'sql-expressions').fieldExpressions) + .toEqual([ + own('id', 'cust_id'), + own('region', 'SAFE_DIVIDE(a, b)', 'BIGQUERY'), + own('region', 'a / b', 'ANSI_SQL'), own('email', 'LOWER(email)'), + prod('email', 'mail'), prod('name', 'full_name'), + prod('region', 'a / NULLIF(b, 0)', 'SPANNER'), + prod('region', 'a / b', 'ANSI_SQL'), dev('email', 'mail'), + dev('id', 'dev_id'), + ]); }); - test('a field a profile excludes, its own, inherited or on an entity ' + - 'with no parent, is still listed, so an action projecting it ' + - 'does not warn', + test('an entity\'s schema lists the fields it declares, leaving out one ' + + 'it only rebinds and keeping one a profile excludes, with no ' + + 'keys or semantics block, even under emitExpressions, and an ' + + 'action projecting an excluded field does not warn', () => { const applied = applyProfileFile( - load(MODEL_FILE + ACTIONS({ - Notify: ['{name: who, concept: customers, field: name}'], - })), + loaded(), 'name: prod\nentities: [{name: customers, ' + 'fields_exclude: [signup, name]}]', 'prod'); if ('error' in applied) throw new Error(applied.error); const {model} = applied; - const customers = model.entities.find(e => e.name === 'customers')!; - expect(customers.excludedFields).toEqual(['signup', 'name']); - expect(names(schema(model, 'customers'))).toEqual([ - 'id', 'signup', 'region', 'active', 'email', 'birth_date', 'name', + for (const opts of [V2, V2_EXPR]) { + expect(aspect(model, 'customers', 'schema', opts)).toEqual({ + fields: [ + {name: 'signup', dataType: 'Date', metadataType: 'DATETIME', + semantic: 'DIMENSION', annotations: {is_time: 'true'}}, + {name: 'region', dataType: 'Opaque', metadataType: 'OTHER', + semantic: 'DIMENSION'}, + {name: 'active', dataType: 'Boolean', metadataType: 'BOOLEAN', + semantic: 'DIMENSION', annotations: {is_time: 'false'}}, + {name: 'email', dataType: 'Opaque', metadataType: 'OTHER'}, + ], + }); + } + expect(aspect(model, 'party', 'schema').fields).toEqual([ + {name: 'id', description: 'The party id', dataType: 'Integer', + metadataType: 'NUMBER', synonyms: ['party_key'], + annotations: {label: 'Party ID'}}, + {name: 'name', dataType: 'String', metadataType: 'STRING'}, ]); + // An action projecting the excluded inherited field warns only with + // the flag off, whose schema leaves the field out. const who = (opts: KcGenerateOptions) => generateCatalogResources(model, opts) .warnings.filter(w => w.includes('parameter \'who\'')); expect(who(V2)).toEqual([]); - // With the flag off the schema leaves the field out, so it warns. expect(who(OPTS)).not.toEqual([]); - const flat = modelWithField(undefined); - flat.entities[0].fields.push({name: 'secret'}); - flat.entities[0].excludedFields = ['secret']; - expect(names(data(flat, 'm/entities/e', 'schema'))) - .toEqual(['f', 'secret']); }); - test('an entity carries its instructions, custom members, third-party ' + - 'blocks and examples at entry level, and a field with them gets ' + - 'the matching aspect at its path, apart from the entity\'s own', + test('an entity carries guidelines, semantic-extensions and an overview ' + + 'of its examples and sidecar at entry level, a field with them ' + + 'gets the matching aspect at its path on the entity that declares ' + + 'it, and an unstorable member warns once', () => { - const aspects = entity(loaded(), 'customers').aspects!; - const ID_ASPECTS: Record = { - guidelines: {...GUIDE('Never show raw ids.'), data: {pii: true}}, - 'semantic-extensions': {customExtensions: [DBT]}, + const AT = (type: string) => `${KEY(type)}@Schema.id`; + const model = loaded(); + const id = 'sales/entities/customers'; + const opts = {...V2, overviews: new Map([[id, {content: 'Buyers.'}]])}; + const customers = entry(model, id, opts).aspects!; + const party = entry(model, 'sales/entities/party').aspects!; + expect(customers[KEY('overview')].data).toEqual( + {content: 'Buyers.', ...EXAMPLES('Who bought the most?')}); + const FIELD: Record = { + guidelines: {...GUIDE('Hide raw ids.'), data: {pii: true}}, + 'semantic-extensions': { + customExtensions: [ + {vendorName: 'DBT', data: '{"tests": ["not_null"]}'}, + ], + }, overview: EXAMPLES('Party 7'), }; - expect(aspects[KEY('guidelines')].data).toEqual( - {...GUIDE('Use for customer analytics.'), data: {tier: 'gold'}}); - for (const [type, want] of Object.entries(ID_ASPECTS)) { - expect(aspects[AT(type, 'id')], type).toEqual( + for (const [type, want] of Object.entries(FIELD)) { + expect(party[AT(type)], type).toEqual( {aspectType: `${DATAPLEX}/aspectTypes/${type}`, data: want}); } - const keys = (model: SemanticModel, name: string) => - Object.keys(entity(model, name).aspects!); const HEAD = [KEY('semantic-entity'), KEY('schema')]; - const OWN = ['guidelines', 'semantic-extensions', 'overview'].map(KEY); - const FIELD = Object.keys(ID_ASPECTS).map(type => AT(type, 'id')); - const SQL = KEY('sql-expressions'); - expect(Object.keys(aspects)) - .toEqual([...HEAD, ...OWN, ...FIELD, SQL]); - expect(keys(loaded(), 'orders')).toEqual([...HEAD, SQL]); - // The abstract entity that declares the field carries them too. - expect(keys(loaded(), 'party')).toEqual([...HEAD, ...FIELD]); - }); - - test('an inherited field\'s unstorable member warns once, under the ' + - 'entity that declares it', - () => { - const model = load(MODEL_FILE.replace('{pii: true}', '{pii: .inf}')); + expect(Object.keys(customers)).toEqual([ + ...HEAD, ...Object.keys(FIELD).map(KEY), KEY('sql-expressions'), + ]); + expect(Object.keys(party)) + .toEqual([...HEAD, ...Object.keys(FIELD).map(AT)]); expect(generateCatalogResources(model, V2).warnings.filter( w => w.includes('ai_context member'))) - .toEqual([`field 'party.id': ai_context member 'pii' is not ` + - `pushed: its value holds .inf, which Knowledge ` + - `Catalog cannot store as written`]); - }); -}); - - -describe('with KC_V2_ASPECTS on, field expressions', () => { - // A sql-expressions record, and an aspect of such records. `qualifier` names - // the profile a record comes from and is unset for the entity's own expression. - const record = (qualifier?: string) => - (name: string, sql: string, dialect?: string) => - ({name, qualifier, dialect, sql}); - const [own, dev, prod, qa] = - [record(), record('dev'), record('prod'), record('qa')]; - const SQL_ASPECT = (...fieldExpressions: object[]) => - ({aspectType: `${DATAPLEX}/aspectTypes/sql-expressions`, - data: {fieldExpressions}}); - const MODEL_FILE = `version: "0.2.0.dev0/google" -semantic_model: - - name: sales - entities: - - name: customers - source: ${TABLE('customers')} - primary_key: [id] - fields: - - { name: id, expression: id } - - { name: email, expression: LOWER(email) } - - name: score - expression: {dialects: [ - {dialect: BIGQUERY, expression: 'SAFE_DIVIDE(a, b)'}, - {dialect: ANSI_SQL, expression: a / b}]} - - { name: note } - - name: vip_customers - extends: [customers] - source: ${TABLE('vip_customers')} - primary_key: [id] - fields: [{ name: id, expression: vip_id }] - - name: events - source: ${TABLE('events')} - primary_key: [at] - fields: [{ name: at }] -`; - const CUSTOMERS = [ - own('id', 'id'), own('email', 'LOWER(email)'), - own('score', 'SAFE_DIVIDE(a, b)', 'BIGQUERY'), - own('score', 'a / b', 'ANSI_SQL'), - ]; - - const loaded = () => load(MODEL_FILE); - // Each entity entry's sql-expressions aspect, or null, by entity name. - const sqlExpressions = (model: SemanticModel) => - Object.fromEntries(generateCatalogResources(model, V2) - .entries.filter(e => e.entryType.endsWith('/semantic-entity')) - .map(e => [e.entrySource!.displayName, - e.aspects![KEY('sql-expressions')] ?? null])); - - test('an entity\'s own field expressions become records in field order, ' + - 'then each named profile that binds a field adds records in ' + - 'declaration order with its name in qualifier, and an entity with ' + - 'none gets no sql-expressions aspect', - () => { - const model = loaded(); - expect(sqlExpressions(model)).toEqual({ - customers: SQL_ASPECT(...CUSTOMERS), - vip_customers: SQL_ASPECT(own('id', 'vip_id')), - events: null, - }); - model.profiles = [ - ENTITIES( - 'prod', - 'customers, fields: [{name: email, expression: mail}, ' + - '{name: score, expression: {dialects: [' + - '{dialect: SPANNER, expression: "a / NULLIF(b, 0)"}, ' + - '{dialect: ANSI_SQL, expression: a / b}]}}]', - 'vip_customers, fields: [{name: email, expression: vip_mail}]'), - ENTITIES('ops', `customers, source: ${TABLE('customers_ops')}`), - ENTITIES( - 'dev', - 'customers, fields: [{name: email, expression: mail}, ' + - '{name: id, expression: dev_id}]'), - ENTITIES('qa', 'customers, fields: [{name: id, expression: qa_id}]'), - ]; - expect(sqlExpressions(model)).toEqual({ - customers: SQL_ASPECT( - ...CUSTOMERS, prod('email', 'mail'), - prod('score', 'a / NULLIF(b, 0)', 'SPANNER'), - prod('score', 'a / b', 'ANSI_SQL'), dev('email', 'mail'), - dev('id', 'dev_id'), qa('id', 'qa_id')), - vip_customers: - SQL_ASPECT(own('id', 'vip_id'), prod('email', 'vip_mail')), - events: null, - }); + .toEqual([['field \'party.id\'', '.inf'], + ['entity \'customers\'', '.nan']].map( + ([object, holds]) => `${object}: ai_context ` + + `member 'rank' is not pushed: its value ` + + `holds ${holds}, which Knowledge Catalog ` + + `cannot store as written`)); }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts index 459f3b284..6513b4d45 100644 --- a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts @@ -518,8 +518,8 @@ describe('SemanticModelLayout overview sidecars', () => { } }); - test('overviewSidecarPaths finds the sidecars of model, metric and ' + - 'relationship entries, by entry id', + test('overviewSidecarPaths finds the sidecars of model, entity, metric ' + + 'and relationship entries, by entry id', async () => { const dir = path.join(catalogPath, 'EntryGroups', 'eg'); for (const id of ['retail', 'retail/metrics/m', @@ -529,8 +529,8 @@ describe('SemanticModelLayout overview sidecars', () => { } const l = await layout('eg'); expect(l.overviewSidecarPaths()) - .toEqual(['other', 'retail', 'retail/metrics/m', - 'retail/relationships/r'] + .toEqual(['other', 'retail', 'retail/entities/x', + 'retail/metrics/m', 'retail/relationships/r'] .map(entryId => ({ entryId, path: path.join(dir, `${entryId}.overview.md`),