feat(semantic): write entities, fields and field expressions into the second-generation catalog types

toolbox/mdcode/docs/semantic-model/reference.md

@@ -357,10 +357,10 @@ the shared label's signature is present on every subtype table. An abstract
 entity that no concrete entity extends has nothing to attach to and is dropped
 with a warning. `abstract` is an explicit marker: a non-abstract entity with no
 `source` is treated as a binding error and fails the push, never
-silently dropped as if it were table-less. The Knowledge Catalog leg does not
-model inheritance today, so an abstract entity has no physical resource to
-catalog and is skipped there (with a warning); its concrete subtypes are
-published normally.
+silently dropped as if it were table-less. With the first-generation types, the
+Knowledge Catalog leg skips an abstract entity with a warning and publishes its
+concrete subtypes. With `KC_V2_ASPECTS=1`, every entity gets an entry, and a
+`reference` link runs from each entity to each parent it extends.
 
 A supertype **may** instead be concrete — carry its own `source` and key. It then
 becomes both its own node table and a label on its subtypes. Every subtype table
@@ -518,27 +518,35 @@ its own. Entry ids are `<model>`, `<model>/entities/<entity>`,
 
 A `semantic-link` joins each ordered pair of related entities. Built-in
 `reference` links run from each relationship's entry to the entries of its
-source and target entities. An unbound relationship gets its `reference` links
-too. Their ids start with `semantic-relationship-`, followed by the first 32
-hex characters of a SHA-256 hash of `<model>/<relationship>/source` or
-`<model>/<relationship>/target`. Push owns only the `reference` links with that
-prefix. It deletes the ones the model no longer has and leaves every other
-`reference` link alone.
+source and target entities, and from each entity's entry to the entry of each
+parent in its `extends`. An unbound relationship gets its `reference` links
+too. Their ids start with `semantic-relationship-` or `semantic-extends-`,
+followed by the first 32 hex characters of a SHA-256 hash. The hash covers
+`<model>/<relationship>/source` or `<model>/<relationship>/target` for a
+relationship, and `<model>/<child>/<parent>` for a parent. Push owns only the
+`reference` links with those prefixes. It deletes the ones the model no longer
+has and leaves every other `reference` link alone.
+
+An entity's `schema` aspect lists only the fields it declares. A field it
+inherits and only rebinds is a `fieldExpressions[]` record in its
+`sql-expressions` aspect, with nothing in `schema`. A field's `guidelines`,
+`overview` and `semantic-extensions` aspects sit at `Schema.<field>` on the
+entity that declares the field.
 
 `semantic-metric.entity` holds the metric's entity, declared or derived.
 `excludedInProfiles` follows the order of the model entry's `profiles` list.
 
-Push writes the whole `overview` aspect of the model's entry and of each metric
-and relationship entry from the workspace, and never reads it back. `examples`
-come from the model file's `ai_context.examples`. `content`, `contentType` and
-`links` come from the entry's sidecar, the file `<entry-id>.overview.md` in the
-directory that holds `<model>.yaml`. For example, the model's sidecar is
-`<model>.overview.md` and a metric's is `<model>/metrics/<metric>.overview.md`.
-The sidecar's frontmatter holds `contentType` and `links`, and its body is
-`content`. `contentType` is `MARKDOWN` or `HTML`. Each link has a `url` and an
-optional `title`. A malformed sidecar fails the push before anything is
-written. Push warns about a sidecar that names no entry of the model. An
-overview edited in the console does not survive a push.
+Push writes each entry's whole `overview` aspect from the workspace and never
+reads it back. `examples` come from the model file's `ai_context.examples`.
+`content`, `contentType` and `links` come from the entry's sidecar. The
+sidecar is the file `<entry-id>.overview.md` in the directory that holds
+`<model>.yaml`, so the model's sidecar is `<model>.overview.md` and an
+entity's is `<model>/entities/<entity>.overview.md`. The sidecar's frontmatter holds
+`contentType` and `links`, and its body is `content`. `contentType` is
+`MARKDOWN` or `HTML`. Each link has a `url` and an optional `title`. A
+malformed sidecar fails the push before anything is written. Push warns about
+a sidecar that names no entry of the model. A field's overview holds its
+examples alone. An overview edited in the console does not survive a push.
 
 ## Validation
 
@@ -779,13 +787,14 @@ and each aspect type attached, so a push needs, on the destination entry group:
 * `dataplex.entryLinks.create` and `dataplex.entryLinks.delete` — the
   `schema-join` links, when the model has relationships
 * `dataplex.entryGroups.useSchemaAspect` — every entity carries the built-in
-  `schema` aspect (its fields, keys, unique keys, and labels)
+  `schema` aspect (its fields, keys, unique keys, and labels; with
+  `KC_V2_ASPECTS=1`, the fields it declares and no keys)
 * `dataplex.entryGroups.useGuidelinesAspect` — when the model, an entity, or a
   metric carries `ai_context.instructions`
 * `dataplex.entryGroups.useSchemaJoinAspect` and
   `dataplex.entryGroups.useSchemaJoinEntryLink` — when the model has relationships
 * `dataplex.entryGroups.useReferenceEntryLink` — with `KC_V2_ASPECTS=1`, when
-  the model has relationships
+  the model has relationships or an entity extends another
 * the `use<AspectType>Aspect` permission for the `semantic-model`,
   `semantic-entity`, and `semantic-metric` aspect types the push attaches — i.e.
   `dataplex.entryGroups.useSemanticModelAspect`, `useSemanticEntityAspect`, and

toolbox/mdcode/src/libts/layouts/semantic-model.ts

@@ -2,10 +2,10 @@
 //
 // The whole model definition is a single Apache Ossie document per model,
 // located at `catalog/EntryGroups/<entryGroupId>/<model>.yaml`, with its
-// profile files beside it. The model's Knowledge Catalog entry and each metric
-// and relationship entry may have an overview sidecar at its entry id under
-// the same directory: `<model>.overview.md` for the model and, for example,
-// `<model>/metrics/<metric>.overview.md` for a metric.
+// profile files beside it. Each Knowledge Catalog entry of the model may have
+// an overview sidecar at its entry id under the same directory:
+// `<model>.overview.md` for the model and, for example,
+// `<model>/entities/<entity>.overview.md` for an entity.
 //
 
 import * as glob from 'glob';
@@ -175,15 +175,15 @@ export class SemanticModelLayout implements CatalogLayout {
         ]));
   }
 
-  // Each overview sidecar on disk at a model, metric or relationship path,
-  // with the entry id its path names, in path order.
+  // Each overview sidecar on disk at a model, entity, metric or relationship
+  // path, with the entry id its path names, in path order.
   overviewSidecarPaths(): {entryId: string; path: string}[] {
     if (!this._entryGroup) return [];
     const dir = this._groupDir();
     const files = glob.globSync(
         [
           `*${OVERVIEW_SIDECAR}`,
-          `*/{metrics,relationships}/*${OVERVIEW_SIDECAR}`,
+          `*/{entities,metrics,relationships}/*${OVERVIEW_SIDECAR}`,
         ],
         {cwd: dir, nodir: true, posix: true});
     return files.sort().map(f => ({

toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts

@@ -23,19 +23,20 @@
 //     entry group to guard against foreign models (unless `--force-remove` is
 //     set), and checks that any required custom entry/aspect types exist in
 //     the target project. With the second-generation aspects, it also checks
-//     every aspect's size.
+//     every aspect's size and each entry's instance count per aspect type.
 //   * Apply creates or updates each model's entries in dependency order: the
 //     `semantic-model` anchor first, then independent child entries
 //     concurrently, and finally dependent child entries (`semantic-metric` and
 //     `semantic-explore`).
 //   * Relationship edges are published as entry links between two entity
 //     entries after that model's entries exist: a `schema-join` link per
 //     relationship, or with v2Aspects a `semantic-link` per ordered entity
-//     pair and a `reference` link from each relationship entry to each of its
-//     entities. A re-push first deletes any orphaned links (no longer emitted)
-//     and stale links (whose type or endpoint references changed), then
-//     creates new or recreated links and updates unchanged links' aspects in
-//     place. The caller additionally needs
+//     pair, a `reference` link from each relationship entry to each of its
+//     entities, and one from each entity to each parent it extends. A re-push
+//     first deletes any orphaned links (no longer emitted) and stale links
+//     (whose type or endpoint references changed), then creates new or
+//     recreated links and updates unchanged links' aspects in place. The
+//     caller additionally needs
 //     `dataplex.entryGroups.useSchemaJoinEntryLink` and `useSchemaJoinAspect`,
 //     and with v2Aspects `useReferenceEntryLink`, on the destination entry
 //     group.
@@ -196,7 +197,8 @@ function sleep(ms: number): Promise<void> {
  *      `opts.fileCopyTypeChecked` is set.
  *   5. Returns the emitted resources, entry-group listing, warnings, and plan
  *      as `prepared` for `applyKnowledgeCatalog`.
- * With `opts.v2Aspects` set, step 2 also checks every aspect's size.
+ * With `opts.v2Aspects` set, step 2 also checks every aspect's size and each
+ * entry's instance count per aspect type.
  */
 export async function preflightKnowledgeCatalog(
     models: LoadedModel[], ctx: context.ApiContext, opts: KcDeployOptions):
@@ -324,8 +326,8 @@ export async function deployEmittedModels(
  * The method performs the following checks:
  *   1. Validates locally that `emitted` contains a single model, that each
  *      file copy fits in one aspect and, with `opts.v2Aspects` set, that every
- *      aspect fits. Builds the dry-run plan and returns early when
- *      `opts.validateOnly` is set.
+ *      aspect size and instance count is within its limit. Builds the
+ *      dry-run plan and returns early when `opts.validateOnly` is set.
  *   2. Lists the destination entry group and validates that it contains no
  *      foreign models unless `--force-remove` is set.
  *   3. Validates that every custom type used by the model exists in the
@@ -516,8 +518,9 @@ function fileCopySizeError(data: Record<string, any>): string|undefined {
 
 /**
  * The emitter's warnings for `models` with v2Aspects on, and why push's
- * catalog leg cannot write them: an emit error or an aspect too large to
- * store. Calls no service.
+ * catalog leg cannot write them: an emit error, an aspect too large to
+ * store, or an entry with too many instances of one aspect type. Calls no
+ * service.
  */
 export function checkCatalogAspects(
     models: LoadedModel[],
@@ -531,7 +534,8 @@ export function checkCatalogAspects(
 // Why push cannot write what `model` emits, or undefined when it can.
 function emittedAspectsError(model: EmittedModel): string|undefined {
   for (const entry of model.resources.entries) {
-    const error = aspectDataError(entry.aspects, `entry '${entryId(entry)}'`);
+    const error = aspectDataError(entry.aspects, `entry '${entryId(entry)}'`) ??
+        instanceCountError(entry);
     if (error) return error;
   }
   for (const link of model.resources.entryLinks) {
@@ -541,6 +545,32 @@ function emittedAspectsError(model: EmittedModel): string|undefined {
   return undefined;
 }
 
+// Most aspects Dataplex returns for one entry in one read.
+const MAX_ASPECTS_PER_READ = 100;
+
+// The aspect types a field can carry at its path, each with what fills it.
+const FIELD_ASPECT_CONTENT: Record<string, string> = {
+  overview: 'examples',
+  guidelines: 'instructions or custom ai_context members',
+  'semantic-extensions': 'custom_extensions from vendors other than GOOGLE',
+};
+
+// Why kcmd could not read back every instance of one aspect type `entry`
+// carries, or undefined when it could.
+function instanceCountError(entry: Entry): string|undefined {
+  for (const [type, content] of Object.entries(FIELD_ASPECT_CONTENT)) {
+    const count = Object.keys(entry.aspects ?? {})
+                      .filter(key => aspectTypeIdOfKey(key) === type)
+                      .length;
+    if (count < MAX_ASPECTS_PER_READ) continue;
+    return `entry '${entryId(entry)}' would hold ${count} ${type} aspects, ` +
+        `and Knowledge Catalog returns at most ${MAX_ASPECTS_PER_READ} in ` +
+        `one read, so kcmd could not read them all back; give ${content} ` +
+        `to fewer than ${MAX_ASPECTS_PER_READ} of the entity and its fields.`;
+  }
+  return undefined;
+}
+
 // Why Knowledge Catalog cannot store one of `aspects`, which `owner` carries,
 // or undefined when each fits.
 function aspectDataError(
@@ -1607,12 +1637,12 @@ const KEPT_SOURCE_FIELDS =
  * name. Passing `entryAspectKeys(entry, opts)` with `deleteMissingAspects:
  * true` makes a re-push converge: a still-present optional aspect is refreshed,
  * and a removed one (such as `guidelines` when `ai_context.instructions` is
- * deleted, `sql-expressions`, or a field's `guidelines`, matched by its `@*`
- * key under `v2Aspects`) is deleted so a later `pull` does not bring it back.
- * Push owns the optional aspects `entryAspectKeys` lists, so it removes any
- * that the model does not declare even if someone added them in the console.
- * Every other aspect another tool attached is left alone, since `aspectKeys`
- * names only the types kcmd writes.
+ * deleted, `sql-expressions`, or a field's instance, matched by its `@*` key
+ * under `v2Aspects`) is deleted so a later `pull` does not bring it back. Push
+ * owns the optional aspects `entryAspectKeys` lists, so it removes any that the
+ * model does not declare even if someone added them in the console. Every other
+ * aspect another tool attached is left alone, since `aspectKeys` names only the
+ * types kcmd writes.
  *
  * The update mask replaces the whole entrySource, so the update carries over
  * fields from `source`, the entrySource the entry had before this push. Of
@@ -1639,13 +1669,6 @@ async function updateEntry(
     ...(Object.keys(labels).length ? {labels} : {}),
   };
   if (Object.keys(merged).length) entry = {...entry, entrySource: merged};
-  // We cannot simply pass `Object.keys(entry.aspects ?? {})` here because
-  // `entry.aspects` only contains the aspects emitted in *this* push. If an
-  // optional aspect (such as `guidelines`, `sql-expressions`, or a column's
-  // `guidelines@Schema.<field>`) was present on a previous push and removed in
-  // this push, it is absent from `entry.aspects` -- and Dataplex only deletes a
-  // removed aspect when its key (or `<project>.<location>.guidelines@*` for
-  // field-level `guidelines`) is still listed in `aspectKeys`.
   const aspectKeys = entryAspectKeys(entry, opts);
   const upd = await cat.updateEntry(
       entry, ['entry_source', 'aspects'], [...aspectKeys],

toolbox/mdcode/src/libts/semantic/kc_actions.ts

@@ -71,7 +71,7 @@ export interface ActionEmitContext {
   // emitted (knowledge_catalog.claim).
   claim(entryId: string, label: string): boolean;
   // Names of the entities this push actually publishes an entry for. An
-  // abstract entity, or one a binding profile pruned, is absent, so a
+  // abstract entity under the first-generation types is absent, so a
   // parameter typed by it would name an entry that does not exist.
   publishedEntities: Set<string>;
 }
@@ -92,7 +92,8 @@ export function actionOwnedPrefix(modelId: string): string {
 // it will. Three ways to lose it, and naming which one is the difference
 // between an author deleting the projection and an author binding a column:
 //
-//   - the concept is not published at all (pruned, or abstract);
+//   - the concept is not published at all (an abstract entity under the
+//     first-generation types);
 //   - the concept is published but the FIELD is not -- `pruneUnavailable`
 //     drops an unbound field while keeping the entity whose key still binds,
 //     and a projected parameter no longer prunes its action, so the entry goes
@@ -144,8 +145,8 @@ export function actionEntries(
   const relationshipNames =
       new Set((model.relationships ?? []).map(r => r.name));
 
-  // The fields this push actually writes into a `schema` aspect, which is
-  // what a later pull reads back. Asked of the loader's own resolver so the
+  // The fields of each concept with inheritance resolved, which a later pull
+  // rebuilds from the catalog. Asked of the loader's own resolver so the
   // answer accounts for `extends`; a model that will not resolve returns null
   // and the field half of the check is skipped, since an emitter is not the
   // place to discover that a model does not load.

toolbox/mdcode/src/libts/semantic/kc_entries.ts

@@ -150,7 +150,7 @@ export function isDependentEntry(entry: Entry): boolean {
 /**
  * Returns true when `link` is of a kind push writes for a model: `schema-join`
  * in V1, and in V2 `semantic-link` or a `reference` link whose ID starts with
- * `semantic-relationship-`.
+ * `semantic-relationship-` or `semantic-extends-`.
  *
  * Includes both V1 and V2 regardless of `v2Aspects`, so a model pushed with V1
  * and then V2 still discovers and cleans up its old `schema-join` links
@@ -160,7 +160,7 @@ export function isModelLink(link: EntryLink): boolean {
   const type = linkTypeId(link);
   return type === 'schema-join' || type === 'semantic-link' ||
       (type === 'reference' &&
-       /^semantic-relationship-/.test(linkId(link)));
+       /^semantic-(relationship|extends)-/.test(linkId(link)));
 }
 
 // ---------------------------------------------------------------------------
@@ -262,35 +262,34 @@ function systemTypeProjectAndLocation(
  *
  * Includes every aspect key present on `entry.aspects` plus the optional
  * `<proj>.<loc>.guidelines` aspect. When `opts?.v2Aspects` is true:
- *   * Adds `<proj>.<loc>.semantic-extensions` on model, entity, metric and
- *     relationship entries, so Dataplex removes the aspect when the object no
- *     longer has a third-party extension block.
- *   * Adds `<proj>.<loc>.overview` on model, metric and relationship entries,
- *     so Dataplex removes the aspect when the object no longer has content
- *     for it.
+ *   * Adds `<proj>.<loc>.semantic-extensions` and `<proj>.<loc>.overview` on
+ *     model, entity, metric and relationship entries, so Dataplex removes the
+ *     aspect when the object no longer has content for it.
  *   * Adds `<proj>.<loc>.sql-expressions` on entity and metric entries.
- *   * Adds `<proj>.<loc>.guidelines@*` on entity entries so Dataplex clears
- *     field-level `guidelines` aspects for columns whose instructions or
- *     fields were removed.
+ *   * Adds `<proj>.<loc>.<type>@*` for guidelines, semantic-extensions and
+ *     overview on entity entries, so Dataplex writes each field's instance
+ *     `entry` carries and removes the others. A field's own key is left out.
  */
 export function entryAspectKeys(
     entry: Entry, opts?: SystemTypeOptions): Set<string> {
   const {proj, loc} = systemTypeProjectAndLocation(opts);
   const keys = new Set([
-    ...Object.keys(entry.aspects ?? {}),
+    ...Object.keys(entry.aspects ?? {}).filter(key => !key.includes('@')),
     `${proj}.${loc}.guidelines`,
   ]);
   if (!opts?.v2Aspects) return keys;
 
   if (isSemanticObjectEntry(entry)) {
     keys.add(`${proj}.${loc}.semantic-extensions`);
-    if (!isEntityEntry(entry)) keys.add(`${proj}.${loc}.overview`);
+    keys.add(`${proj}.${loc}.overview`);
   }
   if (isEntityEntry(entry) || isMetricEntry(entry)) {
     keys.add(`${proj}.${loc}.sql-expressions`);
   }
   if (isEntityEntry(entry)) {
-    keys.add(`${proj}.${loc}.guidelines@*`);
+    for (const type of ['guidelines', 'semantic-extensions', 'overview']) {
+      keys.add(`${proj}.${loc}.${type}@*`);
+    }
   }
   return keys;
 }

toolbox/mdcode/src/libts/semantic/kc_ids.ts

@@ -150,7 +150,7 @@ export function relationshipLinkId(modelName: string, relName: string): string {
  * releases, so that push finds the links it wrote.
  */
 export function semanticLinkId(
-    kind: 'link'|'relationship', ...parts: string[]): string {
+    kind: 'link'|'relationship'|'extends', ...parts: string[]): string {
   const digest =
       createHash('sha256').update(parts.join('/'), 'utf8').digest('hex');
   return `semantic-${kind}-${digest.slice(0, 32)}`;