feat(semantic): rebuild second-generation models on pull and write them as files

toolbox/mdcode/docs/semantic-model/fidelity.md

@@ -314,6 +314,10 @@ model has no `kcmd-push-id` label and no stored copy, such as a model an older
 after the model. [Pull](README.md#pull) says when `pull` refuses the catalog
 instead.
 
+With `KC_V2_ASPECTS=1`, pull rebuilds the model file, profile files and overview
+sidecars from the metadata, and writes each metric's entity as `entity:`,
+declared or derived. See [Reference](reference.md#with-kc_v2_aspects1).
+
 `pull` returns what push wrote — the **Recovered by `pull`** column above is the
 summary. When the push deployed a graph, what push wrote was already pruned to
 the bound view (see [To Knowledge Catalog](#to-knowledge-catalog)), so pull

toolbox/mdcode/docs/semantic-model/reference.md

@@ -73,7 +73,7 @@ the same scope you authored under. See [Pull](README.md#pull) for behavior.
 | Flag | Effect |
 |------|--------|
 | `--dry-run` | Read the catalog and report what would be written, but write or delete no files. |
-| `--force-remove` | Replace a differently-named local model with the catalog's (see [Pull](README.md#pull)); without it, a pull that would leave the entry group holding two models fails. Also deletes a local profile file the catalog has no copy of, when the catalog holds copies of the model's files. |
+| `--force-remove` | Replace a differently-named local model with the catalog's (see [Pull](README.md#pull)); without it, a pull that would leave the entry group holding two models fails. Also deletes a local profile file the catalog has no copy of, when the catalog holds copies of the model's files. With `KC_V2_ASPECTS=1`, it also deletes a local profile file or overview sidecar the pull did not write. |
 
 ### profiles
 
@@ -514,7 +514,7 @@ never stored, see
 
 ### With `KC_V2_ASPECTS=1`
 
-With `KC_V2_ASPECTS=1`, push writes a newer set of Knowledge Catalog
+With `KC_V2_ASPECTS=1`, push and pull use a newer set of Knowledge Catalog
 types. Each relationship other than a many-to-many association has an entry of
 its own. Entry ids are `<model>`, `<model>/entities/<entity>`,
 `<model>/metrics/<metric>` and `<model>/relationships/<relationship>`.
@@ -528,16 +528,17 @@ followed by the first 32 hex characters of a SHA-256 hash. The hash covers
 `<model>/<relationship>/source` or `<model>/<relationship>/target` for a
 relationship, and `<model>/<child>/<parent>` for a parent. Push owns only the
 `reference` links with those prefixes. It deletes the ones the model no longer
-has and leaves every other `reference` link alone.
+has and leaves every other `reference` link alone. Pull reads no entry link.
 
 An entity's `schema` aspect lists only the fields it declares. A field it
 inherits and only rebinds is a `fieldExpressions[]` record in its
 `sql-expressions` aspect, with nothing in `schema`. A field's `guidelines`,
 `overview` and `semantic-extensions` aspects sit at `Schema.<field>` on the
 entity that declares the field.
 
-`semantic-metric.entity` holds the metric's entity, declared or derived.
-`excludedInProfiles` follows the order of the model entry's `profiles` list.
+`semantic-metric.entity` holds the metric's entity, declared or derived. Pull
+writes it back as `entity:`. `excludedInProfiles` follows the order of the
+model entry's `profiles` list.
 
 Push writes each entry's whole `overview` aspect from the workspace and never
 reads it back. `examples` come from the model file's `ai_context.examples`.
@@ -549,7 +550,15 @@ entity's is `<model>/entities/<entity>.overview.md`. The sidecar's frontmatter h
 `MARKDOWN` or `HTML`. Each link has a `url` and an optional `title`. A
 malformed sidecar fails the push before anything is written. Push warns about
 a sidecar that names no entry of the model. A field's overview holds its
-examples alone. An overview edited in the console does not survive a push.
+examples alone. An overview edited in the console survives a push only once
+`kcmd pull` writes the edit into the sidecar. Pull writes a sidecar for each
+entry whose overview has content, a content type or links. It warns about a
+local sidecar the catalog did not return, and `--force-remove` deletes it.
+
+Pull writes the model file, and one file per named profile of a Google-flavor
+model. A profile file keeps the actions it binds locally, since the catalog
+stores no profile's actions. A local profile file the pull did not write draws a
+warning, and `--force-remove` deletes it.
 
 ## Validation
 

toolbox/mdcode/src/libts/layouts/documents.ts

@@ -195,6 +195,11 @@ export function splitFrontmatter(
   };
 }
 
+// Markdown text with `metadata` as its YAML frontmatter and `body` after it.
+export function joinFrontmatter(metadata: object, body: string): string {
+  return `---\n${yaml.stringify(metadata).trim()}\n---\n${body}`;
+}
+
 export function parseMarkdown(content: string): { entry: md.Entry|null; body: string } {
   const {metadata, body} = splitFrontmatter(content);
   if (metadata === undefined) {
@@ -257,6 +262,5 @@ export function toMarkdown(entry: md.Entry, body: string): string {
     delete entryClone.resource.labels[tag];
   }
 
-  const frontmatter = yaml.stringify(metadata).trim();
-  return `---\n${frontmatter}\n---\n${body}`;
+  return joinFrontmatter(metadata, body);
 }

toolbox/mdcode/src/libts/layouts/semantic-model.ts

@@ -19,7 +19,7 @@ import {modelDocumentVersion} from '../semantic/loader';
 import {nameProblem} from '../semantic/names';
 import {YAML_OPTIONS} from '../semantic/yaml_options';
 
-import {splitFrontmatter} from './documents';
+import {joinFrontmatter, splitFrontmatter} from './documents';
 
 // Sidecar suffixes that are NOT model documents.
 const SIDECAR_SUFFIXES = ['.aspects.yaml', '.overview.yaml'];
@@ -552,6 +552,17 @@ export function parseOverviewSidecar(
   return {content: split.body.trim() || undefined, contentType, links};
 }
 
+// The text of the overview sidecar that holds `overview`.
+export function formatOverviewSidecar(
+    {content = '', ...rest}: OverviewSidecar): string {
+  // The body ends with a newline, which push trims.
+  const body = content && !content.endsWith('\n') ? `${content}\n` : content;
+  // Content whose first line is `---` would otherwise read back as frontmatter.
+  return Object.keys(rest).length || body.split(/\r?\n/, 1)[0] === '---' ?
+      joinFrontmatter(rest, body) :
+      body;
+}
+
 // The `name` a profile file declares at its top level, or undefined when it
 // declares none. A file that does not parse throws, naming the file.
 function parseProfileName(text: string, file: string): string|undefined {

toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts

@@ -546,7 +546,7 @@ function emittedAspectsError(model: EmittedModel): string|undefined {
 }
 
 // Most aspects Dataplex returns for one entry in one read.
-const MAX_ASPECTS_PER_READ = 100;
+export const MAX_ASPECTS_PER_READ = 100;
 
 // The aspect types a field can carry at its path, each with what fills it.
 const FIELD_ASPECT_CONTENT: Record<string, string> = {
@@ -556,7 +556,8 @@ const FIELD_ASPECT_CONTENT: Record<string, string> = {
 };
 
 // Why kcmd could not read back every instance of one aspect type `entry`
-// carries, or undefined when it could.
+// carries, or undefined when it could. The limit holds per type because pull
+// reads an entry again once per aspect type when one read returns 100.
 function instanceCountError(entry: Entry): string|undefined {
   for (const [type, content] of Object.entries(FIELD_ASPECT_CONTENT)) {
     const count = Object.keys(entry.aspects ?? {})