diff --git a/toolbox/mdcode/docs/semantic-model/fidelity.md b/toolbox/mdcode/docs/semantic-model/fidelity.md index 9aeec1859..fa7c54a98 100644 --- a/toolbox/mdcode/docs/semantic-model/fidelity.md +++ b/toolbox/mdcode/docs/semantic-model/fidelity.md @@ -314,6 +314,10 @@ model has no `kcmd-push-id` label and no stored copy, such as a model an older after the model. [Pull](README.md#pull) says when `pull` refuses the catalog instead. +With `KC_V2_ASPECTS=1`, pull rebuilds the model file, profile files and overview +sidecars from the metadata, and writes each metric's entity as `entity:`, +declared or derived. See [Reference](reference.md#with-kc_v2_aspects1). + `pull` returns what push wrote — the **Recovered by `pull`** column above is the summary. When the push deployed a graph, what push wrote was already pruned to the bound view (see [To Knowledge Catalog](#to-knowledge-catalog)), so pull diff --git a/toolbox/mdcode/docs/semantic-model/reference.md b/toolbox/mdcode/docs/semantic-model/reference.md index d84703a97..96cce3e71 100644 --- a/toolbox/mdcode/docs/semantic-model/reference.md +++ b/toolbox/mdcode/docs/semantic-model/reference.md @@ -73,7 +73,7 @@ the same scope you authored under. See [Pull](README.md#pull) for behavior. | Flag | Effect | |------|--------| | `--dry-run` | Read the catalog and report what would be written, but write or delete no files. | -| `--force-remove` | Replace a differently-named local model with the catalog's (see [Pull](README.md#pull)); without it, a pull that would leave the entry group holding two models fails. Also deletes a local profile file the catalog has no copy of, when the catalog holds copies of the model's files. | +| `--force-remove` | Replace a differently-named local model with the catalog's (see [Pull](README.md#pull)); without it, a pull that would leave the entry group holding two models fails. Also deletes a local profile file the catalog has no copy of, when the catalog holds copies of the model's files. With `KC_V2_ASPECTS=1`, it also deletes a local profile file or overview sidecar the pull did not write. | ### profiles @@ -514,7 +514,7 @@ never stored, see ### With `KC_V2_ASPECTS=1` -With `KC_V2_ASPECTS=1`, push writes a newer set of Knowledge Catalog +With `KC_V2_ASPECTS=1`, push and pull use a newer set of Knowledge Catalog types. Each relationship other than a many-to-many association has an entry of its own. Entry ids are ``, `/entities/`, `/metrics/` and `/relationships/`. @@ -528,7 +528,7 @@ followed by the first 32 hex characters of a SHA-256 hash. The hash covers `//source` or `//target` for a relationship, and `//` for a parent. Push owns only the `reference` links with those prefixes. It deletes the ones the model no longer -has and leaves every other `reference` link alone. +has and leaves every other `reference` link alone. Pull reads no entry link. An entity's `schema` aspect lists only the fields it declares. A field it inherits and only rebinds is a `fieldExpressions[]` record in its @@ -536,8 +536,9 @@ inherits and only rebinds is a `fieldExpressions[]` record in its `overview` and `semantic-extensions` aspects sit at `Schema.` on the entity that declares the field. -`semantic-metric.entity` holds the metric's entity, declared or derived. -`excludedInProfiles` follows the order of the model entry's `profiles` list. +`semantic-metric.entity` holds the metric's entity, declared or derived. Pull +writes it back as `entity:`. `excludedInProfiles` follows the order of the +model entry's `profiles` list. Push writes each entry's whole `overview` aspect from the workspace and never reads it back. `examples` come from the model file's `ai_context.examples`. @@ -549,7 +550,15 @@ entity's is `/entities/.overview.md`. The sidecar's frontmatter h `MARKDOWN` or `HTML`. Each link has a `url` and an optional `title`. A malformed sidecar fails the push before anything is written. Push warns about a sidecar that names no entry of the model. A field's overview holds its -examples alone. An overview edited in the console does not survive a push. +examples alone. An overview edited in the console survives a push only once +`kcmd pull` writes the edit into the sidecar. Pull writes a sidecar for each +entry whose overview has content, a content type or links. It warns about a +local sidecar the catalog did not return, and `--force-remove` deletes it. + +Pull writes the model file, and one file per named profile of a Google-flavor +model. A profile file keeps the actions it binds locally, since the catalog +stores no profile's actions. A local profile file the pull did not write draws a +warning, and `--force-remove` deletes it. ## Validation diff --git a/toolbox/mdcode/src/libts/layouts/documents.ts b/toolbox/mdcode/src/libts/layouts/documents.ts index 534861a68..d778ce369 100644 --- a/toolbox/mdcode/src/libts/layouts/documents.ts +++ b/toolbox/mdcode/src/libts/layouts/documents.ts @@ -195,6 +195,11 @@ export function splitFrontmatter( }; } +// Markdown text with `metadata` as its YAML frontmatter and `body` after it. +export function joinFrontmatter(metadata: object, body: string): string { + return `---\n${yaml.stringify(metadata).trim()}\n---\n${body}`; +} + export function parseMarkdown(content: string): { entry: md.Entry|null; body: string } { const {metadata, body} = splitFrontmatter(content); if (metadata === undefined) { @@ -257,6 +262,5 @@ export function toMarkdown(entry: md.Entry, body: string): string { delete entryClone.resource.labels[tag]; } - const frontmatter = yaml.stringify(metadata).trim(); - return `---\n${frontmatter}\n---\n${body}`; + return joinFrontmatter(metadata, body); } diff --git a/toolbox/mdcode/src/libts/layouts/semantic-model.ts b/toolbox/mdcode/src/libts/layouts/semantic-model.ts index 8bb81020e..0b929ef4a 100644 --- a/toolbox/mdcode/src/libts/layouts/semantic-model.ts +++ b/toolbox/mdcode/src/libts/layouts/semantic-model.ts @@ -19,7 +19,7 @@ import {modelDocumentVersion} from '../semantic/loader'; import {nameProblem} from '../semantic/names'; import {YAML_OPTIONS} from '../semantic/yaml_options'; -import {splitFrontmatter} from './documents'; +import {joinFrontmatter, splitFrontmatter} from './documents'; // Sidecar suffixes that are NOT model documents. const SIDECAR_SUFFIXES = ['.aspects.yaml', '.overview.yaml']; @@ -552,6 +552,17 @@ export function parseOverviewSidecar( return {content: split.body.trim() || undefined, contentType, links}; } +// The text of the overview sidecar that holds `overview`. +export function formatOverviewSidecar( + {content = '', ...rest}: OverviewSidecar): string { + // The body ends with a newline, which push trims. + const body = content && !content.endsWith('\n') ? `${content}\n` : content; + // Content whose first line is `---` would otherwise read back as frontmatter. + return Object.keys(rest).length || body.split(/\r?\n/, 1)[0] === '---' ? + joinFrontmatter(rest, body) : + body; +} + // The `name` a profile file declares at its top level, or undefined when it // declares none. A file that does not parse throws, naming the file. function parseProfileName(text: string, file: string): string|undefined { diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 943d5dd63..8c2db3962 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -546,7 +546,7 @@ function emittedAspectsError(model: EmittedModel): string|undefined { } // Most aspects Dataplex returns for one entry in one read. -const MAX_ASPECTS_PER_READ = 100; +export const MAX_ASPECTS_PER_READ = 100; // The aspect types a field can carry at its path, each with what fills it. const FIELD_ASPECT_CONTENT: Record = { @@ -556,7 +556,8 @@ const FIELD_ASPECT_CONTENT: Record = { }; // Why kcmd could not read back every instance of one aspect type `entry` -// carries, or undefined when it could. +// carries, or undefined when it could. The limit holds per type because pull +// reads an entry again once per aspect type when one read returns 100. function instanceCountError(entry: Entry): string|undefined { for (const [type, content] of Object.entries(FIELD_ASPECT_CONTENT)) { const count = Object.keys(entry.aspects ?? {}) diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 871b59ef3..650b3ea5e 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -12,17 +12,15 @@ // #278 merges, move `generateCatalogResources` (and the `KcResources` type // + emit helpers) out of `knowledge_catalog.ts` into this file, delete // `knowledge_catalog.ts`, and repoint its importers -// (`deploy_knowledge_catalog.ts` and the emitter tests). `idOf` -- shared by -// both directions and re-exported from here only for `pull_kc` today -- becomes -// a plain local. Then this is the sole KC<->IR codec and the SCAFFOLD note -// above comes out. +// (`deploy_knowledge_catalog.ts` and the emitter tests). Then this is the sole +// KC<->IR codec and the SCAFFOLD note above comes out. // // The reader is the inverse of `generateCatalogResources`: it reconstructs the // IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). -// `semantic-entity` / `semantic-metric` entries are grouped under their -// `semantic-model` anchor via `parentEntry`; entries of other types are -// ignored. Resources are matched by type-name SUFFIX, so a reader need not know -// which system-type project/location the emitter used. +// `semantic-entity`, `semantic-metric` and `semantic-relationship` entries are +// grouped under their `semantic-model` anchor via `parentEntry`; entries of +// other types are ignored. Resources are matched by type-name SUFFIX, so a +// reader need not know which system-type project/location the emitter used. // // Fidelity is bounded by what the emitter persisted, so this read is the // inverse of the WRITE, not of the authored document. It recovers names, @@ -52,19 +50,32 @@ // field -- the reader leaves it un-typed rather than guess. Relationship NAMES // come back lowercased, since the emitter encodes the name only in the link id // (via `linkSlug`), not in the join aspect. +// +// A model pushed with the second-generation types is rebuilt from them when +// the pull reads them (see readModelV2). Every value push stores there comes +// back, each relationship is read from its own entry, and no entry link is +// read. import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; +import type {OverviewSidecar} from '../layouts/semantic-model'; -import {Action, AiContext, Constraint, CustomExtension, DataType, DeploymentSpec, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; +import {targetProblem, unnamedDeployments} from './deployment_target'; +import {Action, AiContext, Constraint, CustomExtension, DATA_TYPES, DataType, DeploymentSpec, Entity, Field, FIELD_BINDING_KEYS, Metric, ProfileSpec, Relationship, SemanticModel} from './ir'; import {isActionEntry, readAction} from './kc_actions'; import {isConstraintEntry, readConstraint} from './kc_constraints'; -import {targetProblem, unnamedDeployments} from './deployment_target'; -import {noSqlReason, oldSourceForm, sourceToDataSource} from './loader'; +import {entryIdOf} from './kc_ids'; +import {DEFAULT_DIALECT, noSqlReason, oldSourceForm, readFieldExpression, sourceToDataSource} from './loader'; +import {compact, nonEmpty} from './osi_converter'; import {columnReferences, entitiesReached} from './sql_expr_utils'; export interface ReadResult { models: SemanticModel[]; warnings: string[]; + // True when a model was rebuilt from the second-generation types. + secondGeneration?: boolean; + // With the second-generation types, the overview sidecar of each entry + // whose overview holds content, contentType or links, by entry id. + overviews?: Map; } // A guard naming a constraint this pull did not recover -- because its entry was @@ -100,9 +111,14 @@ function warnDanglingGuards( * `pull_kc`); each link between two of a model's entity entries is * reconstructed as a 1:1 / 1:N relationship. Pass `[]` (the default) to * reconstruct entities and metrics only. + * + * With `opts.v2Aspects` set, a model whose `semantic-model` aspect has + * `specVersion` is rebuilt from the second-generation types (see + * readModelV2), which reads no entry link. */ export function modelsFromCatalogResources( - entries: Entry[], entryLinks: EntryLink[] = []): ReadResult { + entries: Entry[], entryLinks: EntryLink[] = [], + opts: {v2Aspects?: boolean} = {}): ReadResult { const warnings: string[] = []; const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); @@ -110,6 +126,8 @@ export function modelsFromCatalogResources( entries.filter(e => semanticType(e) === 'semantic-entity'); const metricEntries = entries.filter(e => semanticType(e) === 'semantic-metric'); + const relationshipEntries = + entries.filter(e => semanticType(e) === 'semantic-relationship'); // Actions have no built-in system type; `kc_actions.ts` owns the custom one // and recognizes an entry carrying it. Constraints are the same, through // `kc_constraints.ts`. @@ -130,14 +148,31 @@ export function modelsFromCatalogResources( e => e.parentEntry === anchorName || (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + let secondGeneration = false; + const overviews = new Map(); const models = anchors.map(anchor => { const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + const children = (pool: Entry[]) => childrenOf(anchor.name, pool); + if (opts.v2Aspects && + aspectData(anchor, 'semantic-model').specVersion !== undefined) { + secondGeneration = true; + const [entities, metrics, relationships] = + [entityEntries, metricEntries, relationshipEntries].map(children); + overviewSidecars([anchor, ...entities, ...metrics, ...relationships]) + .forEach((sidecar, id) => overviews.set(id, sidecar)); + const model = + readModelV2(anchor, name, entities, metrics, relationships); + readActionsAndConstraints( + model, children(actionEntries), children(constraintEntries), + warnings); + return model; + } - const entityEntriesForModel = childrenOf(anchor.name, entityEntries); + const entityEntriesForModel = children(entityEntries); const entities = entityEntriesForModel.map(e => readEntity(e, warnings)); const entityNames = entities.map(e => e.name); - const metrics = childrenOf(anchor.name, metricEntries) - .map(e => readMetric(e, entityNames, warnings)); + const metrics = + children(metricEntries).map(e => readMetric(e, entityNames, warnings)); // Index this model's entities by their entry id, so a schema-join link can // resolve its endpoints from the link's entryReferences. The id segment is @@ -147,7 +182,7 @@ export function modelsFromCatalogResources( // live-fetched links resolvable. const entityByEntryId = new Map(); entityEntriesForModel.forEach( - (e, i) => entityByEntryId.set(idOf(e.name), entities[i])); + (e, i) => entityByEntryId.set(entryIdOf(e.name), entities[i])); // Relationships come from the schema-join entry links whose two endpoints // are both this model's entity entries (M:N edges were never published -- @@ -158,15 +193,8 @@ export function modelsFromCatalogResources( const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; if (description !== undefined) model.description = description; - const actions = childrenOf(anchor.name, actionEntries) - .map(e => readAction(e, warnings)) - .filter((a): a is Action => a !== undefined); - if (actions.length) model.actions = actions; - const constraints = childrenOf(anchor.name, constraintEntries) - .map(e => readConstraint(e, warnings)) - .filter((c): c is Constraint => c !== undefined); - if (constraints.length) model.constraints = constraints; - warnDanglingGuards(actions, constraints, name, warnings); + readActionsAndConstraints( + model, children(actionEntries), children(constraintEntries), warnings); // Deployment targets ride back in both the GOOGLE custom_extensions block // and as synthesized `deployments` entries (`default` when one, // `deployment_1`, `deployment_2`, ... when several). @@ -192,7 +220,298 @@ export function modelsFromCatalogResources( } } - return {models, warnings: [...new Set(warnings)]}; + return { + models, + warnings: [...new Set(warnings)], + ...(secondGeneration ? {secondGeneration, overviews} : {}), + }; +} + + +// Reads a model's action and constraint entries onto `model`. +function readActionsAndConstraints( + model: SemanticModel, actionEntries: Entry[], constraintEntries: Entry[], + warnings: string[]): void { + const actions = actionEntries.map(e => readAction(e, warnings)) + .filter((a): a is Action => a !== undefined); + if (actions.length) model.actions = actions; + const constraints = constraintEntries.map(e => readConstraint(e, warnings)) + .filter((c): c is Constraint => c !== undefined); + if (constraints.length) model.constraints = constraints; + warnDanglingGuards(actions, constraints, model.name, warnings); +} + + +// Rebuilds a model from the second-generation types: its model entry and the +// entity, metric and relationship entries under it, each kind sorted by name. +function readModelV2( + anchor: Entry, name: string, entityEntries: Entry[], metricEntries: Entry[], + relationshipEntries: Entry[]): SemanticModel { + const data = aspectData(anchor, 'semantic-model'); + // The named profiles in stored order, which the readers below fill in. + const profiles = new Map( + stringList(asArray(data.profiles).map(p => p?.name)) + .map(p => [p, {name: p, entities: [], relationships: []}])); + const entities = + sortedByName(entityEntries).map(e => readEntityV2(e, profiles)); + const relationships = sortedByName(relationshipEntries) + .map(e => readRelationshipV2(e, profiles)); + const sortedMetrics = sortedByName(metricEntries); + const metrics = sortedMetrics.map(readMetricV2); + + // Each profile excludes '*' when it excludes every metric, and otherwise the + // metrics it excludes, in name order. + const excludedIn = sortedMetrics.map( + e => stringList(aspectData(e, 'semantic-metric').excludedInProfiles)); + for (const profile of profiles.values()) { + const excluded = + metrics.filter((_, i) => excludedIn[i].includes(profile.name)) + .map(m => m.name); + if (excluded.length) { + profile.metricsExclude = + excluded.length === metrics.length ? '*' : excluded; + } + } + + return compact({ + name, + version: data.specVersion, + entities, + relationships, + metrics, + ...objectPropertiesV2( + anchor, '', data.synonyms, anchor.entrySource?.description), + profiles: nonEmpty([...profiles.values()]), + deployments: nonEmpty(asArray(data.deployments).map( + d => compact({name: d?.name, target: d?.target, profile: d?.profile}))), + }); +} + + +// Rebuilds an entity with every field its schema lists, and each field it +// only rebinds, and adds each named profile's binding of it to `profiles`. +function readEntityV2( + entry: Entry, profiles: Map): Entity { + const name = idOf(entry.name); + const data = aspectData(entry, 'semantic-entity'); + const records = + asArray(aspectData(entry, 'sql-expressions').fieldExpressions); + for (const p of asArray(data.profiles)) { + const bound = records.filter(r => r?.qualifier === p?.name); + profiles.get(p?.name)?.entities.push(compact({ + name, + source: p.source?.resource, + primaryKey: nonEmpty(stringList(p.primaryKey?.fields)), + uniqueKeys: nonEmpty(uniqueKeysOf(p)), + fields: nonEmpty([...new Set(bound.map(r => r.name))].map( + f => ({ + name: f, + ...expressionOf( + bound.filter(r => r.name === f), + `profile '${p.name}': entity '${name}': field '${f}'`), + }))), + fieldsExclude: nonEmpty(stringList(p.excludedFields)), + })); + } + const inline = records.filter(r => !r?.qualifier); + const source = data.source?.resource; + return compact({ + name, + dataSource: typeof source === 'string' ? sourceToDataSource(source) : '', + authoredSource: source, + keys: stringList(data.primaryKey?.fields), + uniqueKeys: nonEmpty(uniqueKeysOf(data)), + extends: nonEmpty(stringList(data.extends)), + abstract: data.abstract === true || undefined, + fields: withRebindings( + asArray(aspectData(entry, 'schema').fields) + .filter(fd => typeof fd?.name === 'string') + .map(fd => readFieldV2(entry, name, fd, inline)), + inline, name), + ...objectPropertiesV2( + entry, '', data.synonyms, entry.entrySource?.description), + }); +} + + +// `fields` plus a field for each name `records` hold that `fields` lack: an +// inherited field the entity only rebinds. Each keeps the place its records +// give it. +function withRebindings( + fields: Field[], records: any[], entity: string): Field[] { + const position = (name: string) => records.findIndex(r => r?.name === name); + const out = [...fields]; + for (const name of new Set(records.map(r => r?.name))) { + if (fields.some(f => f.name === name)) continue; + const at = out.findIndex(f => position(f.name) > position(name)); + out.splice(at < 0 ? out.length : at, 0, { + name, + ...expressionOf( + records.filter(r => r?.name === name), `field '${entity}.${name}'`), + }); + } + return out; +} + + +// Rebuilds a field from its schema record `fd`, the entity's sql-expressions +// `records` that name no profile, and its aspects at its path. +function readFieldV2( + entry: Entry, entity: string, fd: any, records: any[]): Field { + const isTime = fd.annotations?.is_time; + return compact({ + name: fd.name, + ...expressionOf( + records.filter(r => r?.name === fd.name), + `field '${entity}.${fd.name}'`), + type: dataTypeOf(fd.dataType), + dimension: fd.semantic === 'DIMENSION' || isTime !== undefined ? + compact( + {isTime: isTime === undefined ? undefined : isTime === 'true'}) : + undefined, + label: fd.annotations?.label || undefined, + ...objectPropertiesV2( + entry, `Schema.${fd.name}`, fd.synonyms, fd.description), + }); +} + + +// Rebuilds a metric. Its entity is the one push stored, declared or derived, +// and pull writes it back as declared. +function readMetricV2(entry: Entry): Metric { + const name = idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + const entity = typeof data.entity === 'string' ? data.entity : undefined; + return compact({ + name, + ...expressionOf( + asArray(aspectData(entry, 'sql-expressions').expressions), + `metric '${name}'`), + type: dataTypeOf(data.dataType), + entity, + authoredEntity: entity, + ...objectPropertiesV2( + entry, '', data.synonyms, entry.entrySource?.description), + }); +} + + +// Rebuilds a relationship, and adds each named profile's join columns to +// `profiles`. A name holds no '/', so the last segment of the entry id, after +// `/relationships/`, is the name as written. +function readRelationshipV2( + entry: Entry, profiles: Map): Relationship { + const name = idOf(entry.name); + const data = aspectData(entry, 'semantic-relationship'); + for (const p of asArray(data.profiles)) { + profiles.get(p?.name)?.relationships.push({ + name, + fromColumns: stringList(p.sourceFields), + toColumns: stringList(p.targetFields), + }); + } + return compact({ + name, + source: + {entity: data.source?.name, columns: stringList(data.source?.fields)}, + destination: + {entity: data.target?.name, columns: stringList(data.target?.fields)}, + ...objectPropertiesV2( + entry, '', data.synonyms, entry.entrySource?.description), + }); +} + + +// An object's `description`, ai_context and extension blocks, read from its +// aspects on `entry` at `path`, which is `Schema.` for a field and +// empty otherwise. `synonyms` come from the object's own aspect. A part with +// nothing in it is undefined, and custom members are sorted by name. +function objectPropertiesV2( + entry: Entry, path: string, synonyms: unknown, + description: string| + undefined): Pick { + const guidelines = aspectDataOf(entry.aspects, 'guidelines', path); + // Knowledge Catalog's own guidelines are not part of the model. + const authored = guidelines.userManaged !== false ? guidelines : {}; + const members = Object.keys(authored.data ?? {}).sort(); + const ai: AiContext = compact({ + instructions: authored.instructions || undefined, + synonyms: nonEmpty(stringList(synonyms)), + examples: nonEmpty(stringList( + asArray(aspectDataOf(entry.aspects, 'overview', path).examples) + .map(x => x?.content))), + additionalProperties: members.length ? + Object.fromEntries(members.map(m => [m, authored.data[m]])) : + undefined, + }); + return { + description, + aiContext: Object.keys(ai).length ? ai : undefined, + customExtensions: nonEmpty( + asArray(aspectDataOf(entry.aspects, 'semantic-extensions', path) + .customExtensions) + .map(x => ({vendorName: x?.vendorName, data: x?.data}))), + }; +} + + +// The expression `records` hold, read as the loader reads the form they +// keep: a record with no dialect is the string form, and records with one are +// a dialect list, in stored order. Empty when there is no record. +function expressionOf(records: any[], where: string): + Partial> { + if (!records.length) return {}; + const short = records.find(r => !r.dialect); + const doc = short ? short.sql : { + dialects: records.map(r => ({dialect: r.dialect, expression: r.sql})), + }; + return readFieldExpression(doc, where, DEFAULT_DIALECT, () => undefined); +} + + +// The overview sidecar each of `entries` holds in its overview, by entry id: +// its content, contentType and links. An entry with none of the three has +// none. +function overviewSidecars(entries: Entry[]): Map { + const out = new Map(); + for (const entry of entries) { + const {content, contentType, links} = aspectData(entry, 'overview'); + const sidecar = compact({ + content: content || undefined, + contentType, + links: nonEmpty(asArray(links)), + }); + if (Object.keys(sidecar).length) out.set(entryIdOf(entry.name), sidecar); + } + return out; +} + + +// The unique keys of a semantic-entity aspect or one of its profile records. +function uniqueKeysOf(data: any): string[][] { + return asArray(data?.uniqueConstraints) + .map(u => stringList(u?.fields)) + .filter(k => k.length); +} + + +// The IR datatype a second-generation `dataType` names. Opaque, stored for an +// object with no datatype, reads as none. +function dataTypeOf(dataType: unknown): DataType|undefined { + return dataType !== 'Opaque' && + (DATA_TYPES as readonly unknown[]).includes(dataType) ? + dataType as DataType : + undefined; +} + + +// `entries` sorted by the last segment of their entry ids, which is the name +// of the object each holds. +function sortedByName(entries: Entry[]): Entry[] { + return [...entries].sort((a, b) => { + const [x, y] = [idOf(a.name), idOf(b.name)]; + return x < y ? -1 : x > y ? 1 : 0; + }); } @@ -239,9 +558,7 @@ function readEntity(entry: Entry, warnings: string[]): Entity { }; // Additional unique column sets, from the schema aspect's uniqueConstraints // (each constraint's `fields` is one set); dropped when empty. - const uniqueKeys = asArray(schema.uniqueConstraints) - .map(uc => stringList(uc?.fields)) - .filter(fields => fields.length); + const uniqueKeys = uniqueKeysOf(schema); if (uniqueKeys.length) entity.uniqueKeys = uniqueKeys; const description = entry.entrySource?.description; if (description !== undefined) entity.description = description; @@ -424,9 +741,10 @@ function dataSourceFromResource(resource: string|undefined): string { // system-type project/location need not be known. Returns undefined for entries // that are not part of a semantic model. function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| - 'semantic-metric'|undefined { - for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as - const) { + 'semantic-metric'|'semantic-relationship'|undefined { + for (const t + of ['semantic-model', 'semantic-entity', 'semantic-metric', + 'semantic-relationship'] as const) { if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; } return undefined; @@ -443,12 +761,16 @@ function aspectData(entry: Entry, type: string): Record { // The `data` payload of an aspect of the given bare type from an aspect map, // matched by the aspect key's `.` suffix or the aspectType's // `/aspectTypes/` suffix (robust to whichever system-type -// project/location the emitter used). Returns an empty object when the aspect -// is absent. -function aspectDataOf(aspects: Record|undefined, type: string): - Record { +// project/location the emitter used). Only the aspect at `path` matches, which +// is the entry itself when `path` is empty and a field for `Schema.`. +// Returns an empty object when the aspect is absent. +function aspectDataOf( + aspects: Record|undefined, type: string, + path = ''): Record { for (const [key, aspect] of Object.entries(aspects ?? {})) { - if (key.endsWith(`.${type}`) || + const [ref, keyPath = ''] = key.split('@'); + if ((aspect.path || keyPath) !== path) continue; + if (ref.endsWith(`.${type}`) || aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { return aspect.data ?? {}; } @@ -518,8 +840,8 @@ function readRelationships( // Resolve both endpoints from the link's references. A reference that is // not one of this model's entities (another model, or a non-entity) means // the link does not belong here -- skip it quietly, as with M:N edges. - const endA = entityByEntryId.get(idOf(refs[0].name)); - const endB = entityByEntryId.get(idOf(refs[1].name)); + const endA = entityByEntryId.get(entryIdOf(refs[0].name)); + const endB = entityByEntryId.get(entryIdOf(refs[1].name)); if (!endA || !endB) continue; const key = linkDedupKey(link); @@ -620,7 +942,7 @@ export function linkNamePrefix(modelName: string): string { // The id segment of a full entry resource name (after the last '/'). -export function idOf(name: string): string { +function idOf(name: string): string { return name.split('/').pop() ?? name; } diff --git a/toolbox/mdcode/src/libts/semantic/osi_converter.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts index d807b0fd0..3ce86a954 100644 --- a/toolbox/mdcode/src/libts/semantic/osi_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -191,7 +191,8 @@ function profileDocuments(model: SemanticModel, warnings: string[]): * `logical` marks the model as a purely logical one (no physical binding). * Under GOOGLE_VERSION it suppresses the missing-source and missing-expression * warnings, which flag a lossy pull of a bound model. OSSIE_VERSION requires - * both, so they are still given. An OWL import sets it; `pull` does not. + * both, so they are still given. An OWL import sets it, and so does a + * second-generation pull, which loses no binding. * * `compactFlow` renders leaf collections (scalar sequences, all-scalar maps) in * flow style so the output matches the compact convention the semantic-model @@ -1005,7 +1006,7 @@ function blockTargets(ext: CustomExtension, modelName: string): string[] { // Drops undefined-valued keys so the emitted YAML only shows fields the model // actually set (matching the emitters' `compact` convention). -function compact>(obj: T): T { +export function compact>(obj: T): T { const out: Record = {}; for (const [k, v] of Object.entries(obj)) { if (v !== undefined) out[k] = v; diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index d7f45b295..22a741753 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -20,11 +20,12 @@ import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; -import {SemanticModel} from './ir'; +import {MAX_ASPECTS_PER_READ} from './deploy_knowledge_catalog'; import {actionAspectTypes} from './kc_actions'; import {constraintAspectTypes} from './kc_constraints'; +import {linkDedupKey, modelsFromCatalogResources, ReadResult} from './kc_converter'; import {ACTION_TYPE_ID, CONSTRAINT_TYPE_ID, FILE_COPY_TYPE_ID, FileCopy, PUSH_ID_LABEL} from './kc_custom_types'; -import {idOf, linkDedupKey, modelsFromCatalogResources} from './kc_converter'; +import {entryIdOf, Namer} from './kc_ids'; import {loadSqlEngine} from './sql_parser'; export interface KcPullOptions { @@ -37,29 +38,22 @@ export interface KcPullOptions { // Read the second-generation built-in aspect fields and entry-id layout // (selected via KC_V2_ASPECTS=1). Off by default (absent means false). v2Aspects?: boolean; + // The project of the schema-join entry link type, which only a pull with + // v2Aspects off reads. Defaults to `dataplex-types`. + systemTypeProject?: string; } -export interface KcPullResult { - models: SemanticModel[]; +export interface KcPullResult extends ReadResult { // The file copies a push stored for the model, sorted by file name. Set // only with v2Aspects off. Empty when the catalog holds no copy of the // model file, `.yaml`; the caller then writes the model rebuilt from // the aspects. rawFiles?: FileCopy[]; - warnings: string[]; } // Upper bound on in-flight aspect-hydration fetches during a pull. const HYDRATE_CONCURRENCY = 8; -// The built-in schema-join entry link type. Relationships publish as links of -// this type; it is a system type that is referenced (never created), so it -// always lives in `dataplex-types/global` regardless of where the model's own -// entries live (see knowledge_catalog.ts). Pull filters :lookupEntryLinks to -// exactly it, so a group's other links are never fetched or considered. -const SCHEMA_JOIN_LINK_TYPE = - 'projects/dataplex-types/locations/global/entryLinkTypes/schema-join'; - // Reads the semantic model back from a Knowledge Catalog entry group. Emits no // console output; soft warnings (from the reader) are returned for the caller // to print. Hard failures -- more than one model in the group, a fetch error @@ -80,7 +74,7 @@ export async function pullKnowledgeCatalog( opts.project, opts.location, opts.entryGroup)) { // With v2Aspects on, a pull returns no file copies, so it fetches none. if (opts.v2Aspects && isFileCopyEntry(entry)) continue; - const aspectTypes = semanticAspectTypes(entry.entryType); + const aspectTypes = semanticAspectTypes(entry.entryType, opts.v2Aspects); if (aspectTypes) targets.push({entry, aspectTypes}); // else: not part of a semantic model; ignore it. } @@ -92,7 +86,7 @@ export async function pullKnowledgeCatalog( const anchors = targets.filter( t => t.entry.entryType?.endsWith('/entryTypes/semantic-model')); if (anchors.length > 1) { - const ids = anchors.map(a => idOf(a.entry.name)).sort(); + const ids = anchors.map(a => entryIdOf(a.entry.name)).sort(); throw new Error( `entry group ${destination} holds ${anchors.length} semantic models (${ ids.join( @@ -103,30 +97,24 @@ export async function pullKnowledgeCatalog( // Hydrate every semantic entry. A failed fetch means part of the model would // be silently missing, so abort rather than reconstruct an incomplete model. const hydrated = await mapConcurrent( - targets, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { - const res = await cat.lookupEntry( - opts.project, opts.location, entry.name, aspectTypes); - if (res.status !== 200 || !res.result) { - throw new Error(`failed to fetch entry '${entry.name}' (status ${ - res.status}); pull aborted`); - } - return res.result; - }); + targets, HYDRATE_CONCURRENCY, + async ({entry, aspectTypes}) => opts.v2Aspects ? + fetchEveryAspect(cat, opts, entry, aspectTypes) : + fetchEntry(cat, opts, entry, aspectTypes)); - // Second fetch pass: relationships are schema-join entry links, which the - // entry list/lookup does not return. The catalog exposes links only per - // referenced entry (:lookupEntryLinks), so fan out over the entity entries - // and dedup (linkDedupKey) -- schema-join is undirected, so each link comes - // back once from each of its two endpoints. An entity with no relationships - // returns an empty list (expected, not warned); a non-200 is a real fetch - // error and aborts the pull. - const entityEntries = hydrated.filter( + // Second fetch pass: with the first-generation types, the schema-join links + // of the relationships, which listing entries does not return. Links are + // listed per entity entry and deduplicated, since each comes back from both + // endpoints, and a failed lookup aborts the pull. With v2Aspects on, pull + // reads no link. + const entityEntries = opts.v2Aspects ? [] : hydrated.filter( e => e.entryType?.endsWith('/entryTypes/semantic-entity')); const linkLists = await mapConcurrent(entityEntries, HYDRATE_CONCURRENCY, async entry => { const res = await cat.lookupEntryLinks(opts.project, opts.location, { entry: entry.name, - entryLinkTypes: [SCHEMA_JOIN_LINK_TYPE], + entryLinkTypes: + [new Namer(opts).typeName('entryLink', 'schema-join')], }); if (res.status !== 200 || !res.result) { throw new Error( @@ -147,10 +135,11 @@ export async function pullKnowledgeCatalog( } } - const read = modelsFromCatalogResources(hydrated, entryLinks); + const read = modelsFromCatalogResources( + hydrated, entryLinks, {v2Aspects: opts.v2Aspects}); if (opts.v2Aspects) { warnings.push(...read.warnings); - return {models: read.models, warnings: [...new Set(warnings)]}; + return {...read, warnings: [...new Set(warnings)]}; } // A copy belongs to the group's one model, so with no model there are none. if (!read.models.length) { @@ -166,10 +155,10 @@ export async function pullKnowledgeCatalog( const model = read.models[0].name; // The model's own copies sit under its anchor's `.files.` prefix. A copy // another model left behind is not one of them. - const own = `${idOf(anchors[0].entry.name)}.files.`; + const own = `${entryIdOf(anchors[0].entry.name)}.files.`; const before = warnings.length; const copies = readFileCopies( - hydrated.filter(e => idOf(e.name).startsWith(own)), warnings); + hydrated.filter(e => entryIdOf(e.name).startsWith(own)), warnings); const skipped = warnings.slice(before); // Every copy one push stores carries that push's id and how many copies it // stored. Copies whose ids differ, or of which only some carry one, or a @@ -225,6 +214,44 @@ export async function pullKnowledgeCatalog( } +// Fetches `entry` with the aspects of `aspectTypes`. Throws on a failed fetch. +async function fetchEntry( + cat: CatalogClient, opts: KcPullOptions, entry: Entry, + aspectTypes: string[]): Promise { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + throw new Error(`failed to fetch entry '${entry.name}' (status ${ + res.status}); pull aborted`); + } + return res.result; +} + +// Fetches `entry` with every aspect of `aspectTypes`, reading once per type, +// one read at a time, when one read returns MAX_ASPECTS_PER_READ. Throws naming +// the entry when one type alone returns that many. +async function fetchEveryAspect( + cat: CatalogClient, opts: KcPullOptions, entry: Entry, + aspectTypes: string[]): Promise { + const count = (e: Entry) => Object.keys(e.aspects ?? {}).length; + const read = await fetchEntry(cat, opts, entry, aspectTypes); + if (count(read) < MAX_ASPECTS_PER_READ) return read; + const byType: Entry[] = []; + for (const t of aspectTypes) { + byType.push(await fetchEntry(cat, opts, entry, [t])); + } + const full = byType.findIndex(r => count(r) >= MAX_ASPECTS_PER_READ); + if (full >= 0) { + throw new Error( + `entry '${entryIdOf(entry.name)}' holds ${MAX_ASPECTS_PER_READ} or ` + + `more ${aspectTypes[full].split('/').pop()} aspects, and Knowledge ` + + `Catalog returns at most ${MAX_ASPECTS_PER_READ} in one read, so ` + + `pull cannot read them all; pull aborted`); + } + return {...read, aspects: Object.assign({}, ...byType.map(r => r.aspects))}; +} + + // True when an entry is a file copy, matched by its entry type's id suffix so // the project the custom type lives in need not be known. function isFileCopyEntry(entry: Entry): boolean { @@ -245,7 +272,7 @@ function readFileCopies(entries: Entry[], warnings: string[]): FileCopy[] { if (typeof data?.fileName !== 'string' || data.fileName === '' || typeof data.content !== 'string') { warnings.push( - `file copy '${idOf(entry.name)}' holds no file name or no ` + + `file copy '${entryIdOf(entry.name)}' holds no file name or no ` + `content; it is skipped`); continue; } @@ -263,19 +290,33 @@ function readFileCopies(entries: Entry[], warnings: string[]): FileCopy[] { } -// The aspect type resource names to hydrate for a semantic entry, derived from -// its entryType (the aspect types are the parallel resources in the same -// project/location). Every entry also carries the built-in `guidelines` aspect -// (ai_context instructions); an entity additionally carries the built-in -// `schema` aspect that holds its fields, keys, and unique constraints. Returns -// undefined for entries that are not part of a semantic model. -function semanticAspectTypes(entryType: string): string[]|undefined { +// The aspect types each second-generation entry type is read with. +const OBJECT_ASPECTS = ['guidelines', 'semantic-extensions', 'overview']; +const V2_ASPECT_TYPES = new Map([ + ['semantic-model', ['semantic-model', ...OBJECT_ASPECTS]], + [ + 'semantic-entity', + ['semantic-entity', 'schema', 'sql-expressions', ...OBJECT_ASPECTS] + ], + [ + 'semantic-metric', ['semantic-metric', 'sql-expressions', ...OBJECT_ASPECTS] + ], + ['semantic-relationship', ['semantic-relationship', ...OBJECT_ASPECTS]], +]); + +// The aspect types to read for a semantic entry, in its entry type's project +// and location: V2_ASPECT_TYPES with `v2Aspects` on, the first-generation set +// otherwise. Undefined for an entry outside a semantic model. +function semanticAspectTypes(entryType: string, v2Aspects?: boolean): string[]| + undefined { const marker = '/entryTypes/'; const idx = entryType?.indexOf(marker) ?? -1; if (idx < 0) return undefined; const typeBase = entryType.slice(0, idx); const t = entryType.slice(idx + marker.length); const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + const v2Types = v2Aspects ? V2_ASPECT_TYPES.get(t) : undefined; + if (v2Types) return v2Types.map(aspectType); switch (t) { case 'semantic-model': return [aspectType('semantic-model'), aspectType('guidelines')]; diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 7b9238b31..bb49301ee 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -11,7 +11,7 @@ import {BigQueryClient} from '../libts/gcp/bigquery'; import * as context from '../libts/gcp/context'; import * as dataplex from '../libts/gcp/dataplex'; import {SpannerClient} from '../libts/gcp/spanner'; -import {fileNameOf, OverviewSidecar, SemanticModelLayout} from '../libts/layouts/semantic-model'; +import {fileNameOf, formatOverviewSidecar, OverviewSidecar, SemanticModelLayout} from '../libts/layouts/semantic-model'; import {convertOwlToOsi} from '../libts/semantic/converters/owl/convert'; import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; @@ -20,8 +20,8 @@ import {cloudSqlDialects} from '../libts/semantic/cloudsql_dialect'; import {googleBlockTargets, googleDeploymentTargets, resolveModelDeployments, unnamedDeployments} from '../libts/semantic/deployment_target'; import {ProfileSpec, SemanticModel} from '../libts/semantic/ir'; import {FILE_COPY_TYPE_ID, FileCopy, provisionCustomTypes} from '../libts/semantic/kc_custom_types'; -import {deploymentProfileError, isGoogleVendor, LoadedModel, loadSemanticModels, SemanticModelError, setsAKeyTwice, writtenSource} from '../libts/semantic/loader'; -import {serializeModel} from '../libts/semantic/osi_converter'; +import {deploymentProfileError, isGoogleVendor, LoadedModel, loadSemanticModels, OSSIE_VERSION, SemanticModelError, setsAKeyTwice, writtenSource} from '../libts/semantic/loader'; +import {serializeModel, serializeModelFiles} from '../libts/semantic/osi_converter'; import * as pullKc from '../libts/semantic/pull_kc'; import {applyProfileExclusions, AvailabilityReport, CloudSqlLoad, DeclaredProfile, declaredProfiles, DEFAULT_PROFILE, DocumentProfile, expressionSources, inheritanceErrorOf, isLegacyWrapper, isProfileFileForm, loadProfileFile, modelFileAloneMessage, ProfileExclusion, profileErrorMessage, profileForDocument, profileSourceMessage, pruneUnavailable, reservedProfileNameError, sourcesDialect, unknownProfileMessage, validateProfileCompleteness, validateProfileConsistency, withoutExclusions,} from '../libts/semantic/resolve_profiles'; import {createSemanticRuntimes, undeclaredProfileError} from '../libts/semantic/runtime/runtime'; @@ -2319,8 +2319,10 @@ function reportCatalogResult( // local document is a conflict (pull would leave two models), so pull fails // unless --force-remove authorizes deleting the stale local model first. When // the catalog holds file copies of the model and profile files, pull writes -// those instead of the model rebuilt from the aspects. -// Returns a process exit code (0 on success). +// those instead of the model rebuilt from the aspects. A model rebuilt from +// the second-generation types is written the same way, as its model file, +// profile files and overview sidecars. Returns a process exit code (0 on +// success). async function pullSemanticModel( ctx: context.ApiContext, snapshot: kcmd.CatalogSnapshot, options: PullOptions): Promise { @@ -2340,6 +2342,7 @@ async function pullSemanticModel( location: source.location, entryGroup: source.entryGroup, v2Aspects: kcV2Aspects(), + systemTypeProject: process.env.KC_TYPE_PROJECT, }); for (const w of result.warnings) { @@ -2393,7 +2396,29 @@ async function pullSemanticModel( // aspects. Writing both would leave whichever wrote `.yaml` last. // The copies belong to the entry group's one model. if (rawFiles.length) { - return writeFileCopies(layout, result.models[0].name, rawFiles, options); + return writePulledFiles(layout, result.models[0].name, rawFiles, options); + } + if (result.secondGeneration) { + const pulled = result.models[0]; + // The catalog stores no profile's actions, so a profile file keeps the + // actions it binds locally. A vanilla model declares no action. + const model = pulled.version === OSSIE_VERSION ? pulled : { + ...pulled, + profiles: pulled.profiles?.map(p => { + const actions = localProfileActions(layout, pulled.name, p.name); + return actions ? {...p, actions} : p; + }), + }; + const {files, warnings} = serializeModelFiles(model, {logical: true}); + for (const w of warnings) console.warn(`Warning: [${model.name}] ${w}`); + for (const [id, sidecar] of result.overviews ?? []) { + files.push({ + fileName: `${id}.overview.md`, + content: formatOverviewSidecar(sidecar), + }); + } + return writePulledFiles( + layout, model.name, files, options, {sidecars: true}); } let created = 0; @@ -2451,24 +2476,43 @@ async function pullSemanticModel( } -// Writes the file copies a pull returned for `model` into the entry group -// directory, each to the path its name maps to, and leaves a file whose -// content is unchanged as it is. A name that is not one push stores is -// skipped with a warning. Then each local profile file the catalog did not -// return is named in a warning, or deleted under --force-remove. Returns a -// process exit code. -function writeFileCopies( +// The actions the local profile file of `profile` binds, or undefined when it +// binds none or does not load. +function localProfileActions( + layout: SemanticModelLayout, model: string, profile: string) { + const file = layout.localProfileFiles(model).find( + f => (f.profileName ?? path.basename(f.path, '.yaml')) === profile); + if (!file) return undefined; + try { + return loadProfileFile(fs.readFileSync(file.path, 'utf8'), profile).actions; + } catch (err) { + if (!(err instanceof SemanticModelError)) throw err; + return undefined; + } +} + + +// Writes the files a pull returned or rebuilt for `model`, and skips a name +// push does not store. Then it names each local profile file, and with +// `sidecars` each overview sidecar, that the pull did not write, or deletes it +// under --force-remove. +function writePulledFiles( layout: SemanticModelLayout, model: string, files: FileCopy[], - options: PullOptions): number { + options: PullOptions, {sidecars = false} = {}): number { const written = new Set(); const counts = {created: 0, updated: 0, unchanged: 0}; + const forms = [ + `'${model}.yaml'`, `'${model}.profile..yaml'`, + `'${model}.profiles/.yaml'` + ]; + if (sidecars) forms.push('overview sidecars'); for (const {fileName, content} of files) { - const target = fileCopyTarget(layout, model, fileName); + const target = pulledFileTarget(layout, model, fileName, sidecars); if (!target) { console.warn( `Warning: skipped '${fileName}' from the catalog; pull writes only ` + - `'${model}.yaml', '${model}.profile..yaml' and ` + - `'${model}.profiles/.yaml' inside the entry group directory.`); + `${forms.slice(0, -1).join(', ')} and ${forms.at(-1)} inside the ` + + `entry group directory.`); continue; } written.add(path.resolve(target.path)); @@ -2490,17 +2534,23 @@ function writeFileCopies( } // Pull writes each file back to the path it came from, so a local profile - // file the catalog does not have is stale. - for (const stale of layout.localProfileFiles(model)) { + // file or overview sidecar the catalog does not have is stale. + const local: Array<{path: string; profileName?: string; kind: string}> = [ + ...layout.localProfileFiles(model).map(f => ({...f, kind: 'profile file'})), + ...(sidecars ? layout.overviewSidecarPaths() : []) + .map(f => ({path: f.path, kind: 'overview sidecar'})), + ]; + for (const stale of local) { if (written.has(path.resolve(stale.path))) continue; if (!options.forceRemove) { console.warn( - `Warning: local profile file ${stale.path} is not in the catalog; ` + - `delete it, or pull with --force-remove to delete it.`); + `Warning: local ${stale.kind} ${stale.path} is not in the ` + + `catalog; delete it, or pull with --force-remove to delete it.`); } else if (options.dryRun) { console.log(` would remove ${stale.path}`); } else { - // The older `.profiles/` directory has no layout method. + // The older `.profiles/` directory and a sidecar have no layout + // method. if (stale.profileName !== undefined) { layout.removeProfileDocument(model, stale.profileName); } else { @@ -2521,20 +2571,20 @@ function writeFileCopies( } -// Where pull writes a file copy of `model`, and how, or undefined when its -// name is not one push stores. The name comes from the catalog, so only -// `.yaml`, `.profile..yaml` and -// `.profiles/.yaml` are written, each relative, ending in -// `.yaml`, with no `..` segment and no NUL byte. The model file and a sibling -// profile file are written through the layout, which keeps both in the entry -// group directory; the older `.profiles/` directory has no layout -// method. -function fileCopyTarget( - layout: SemanticModelLayout, model: string, fileName: string): - {path: string; write: (text: string) => void}|undefined { +// Where pull writes the catalog's file `fileName` of `model`, and how. +// Undefined unless the name is relative, holds no `..` segment or NUL byte, and +// names `.yaml`, `.profile..yaml`, +// `.profiles/.yaml` or, with `sidecars`, an overview sidecar +// that push reads: `.overview.md` or +// `/{entities,metrics,relationships}/.overview.md`. +function pulledFileTarget( + layout: SemanticModelLayout, model: string, fileName: string, + sidecars: boolean): {path: string; write: (text: string) => void}| + undefined { + const sidecar = sidecars && fileName.endsWith('.overview.md'); if (path.posix.isAbsolute(fileName) || path.win32.isAbsolute(fileName) || fileName.includes('\0') || fileName.split(/[\\/]/).includes('..') || - !fileName.endsWith('.yaml')) { + !(fileName.endsWith('.yaml') || sidecar)) { return undefined; } if (fileName === `${model}.yaml`) { @@ -2544,10 +2594,22 @@ function fileCopyTarget( }; } // A model name holding a path separator maps to a flat model file, and its - // profile files have no such mapping. - if (/[\\/]/.test(model) || !fileName.startsWith(`${model}.`)) { - return undefined; - } + // profile files and sidecars have no such mapping. + if (/[\\/]/.test(model)) return undefined; + const localPath = path.join(path.dirname(layout.modelPath(model)), fileName); + const writeFile = (text: string) => { + fs.mkdirSync(path.dirname(localPath), {recursive: true}); + fs.writeFileSync(localPath, text); + }; + if (sidecar) { + return fileName === `${model}.overview.md` || + fileName.startsWith(`${model}/`) && + /^(entities|metrics|relationships)\/[^/\\]+\.overview\.md$/ + .test(fileName.slice(model.length + 1)) ? + {path: localPath, write: writeFile} : + undefined; + } + if (!fileName.startsWith(`${model}.`)) return undefined; const rest = fileName.slice(model.length + 1); const profileName = /^profile\.(.+)\.yaml$/.exec(rest)?.[1]; if (profileName !== undefined) { @@ -2564,14 +2626,7 @@ function fileCopyTarget( }; } if (!/^profiles\/[^/\\]+$/.test(rest)) return undefined; - const localPath = path.join(path.dirname(layout.modelPath(model)), fileName); - return { - path: localPath, - write: text => { - fs.mkdirSync(path.dirname(localPath), {recursive: true}); - fs.writeFileSync(localPath, text); - }, - }; + return {path: localPath, write: writeFile}; } diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index 647b5ae81..f0b675a5a 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -11,15 +11,23 @@ // kc_converter.test.ts; the focus here is the fetch SEQUENCE: aspect hydration, // the relationship-link fetch (per-entry, deduped across endpoints), the // single-model-per-group invariant, aborting on any fetch failure, ignoring -// foreign entries, and the stored file copies a pull returns. +// foreign entries, and the stored file copies a pull returns. The +// second-generation tests round-trip a whole model through push and pull. -import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import {afterEach, beforeAll, describe, expect, mock, spyOn, test} from 'bun:test'; import {ApiResult} from '../../../src/libts/gcp/api'; import {CatalogClient, Entry, EntryLink} from '../../../src/libts/gcp/dataplex'; -import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {generatePropertyGraph} from '../../../src/libts/semantic/bigquery'; +import {Field, SemanticModel} from '../../../src/libts/semantic/ir'; +import {entryIdOf} from '../../../src/libts/semantic/kc_ids'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {loadModels} from '../../../src/libts/semantic/loader'; +import {serializeModelFiles} from '../../../src/libts/semantic/osi_converter'; import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; +import {applyProfile, loadProfileFile} from '../../../src/libts/semantic/resolve_profiles'; +import {generateSpannerPropertyGraph} from '../../../src/libts/semantic/spanner'; +import {loadSqlEngine} from '../../../src/libts/semantic/sql_parser'; const OPTS = { project: 'dest', @@ -84,7 +92,9 @@ function err(status: number, message: string): ApiResult { } // Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, -// or 404s an unknown name). `lookupFail` forces a failure for one entry name. +// or 404s an unknown name). A lookup returns only the aspects of the types it +// asks for, and at most 100 of them, as Dataplex does. `lookupFail` forces a +// failure for one entry name. // `links` are served by lookupEntryLinks per referenced entry (so an undirected // link comes back once from each endpoint, exercising the puller's dedup); // `linkFail` forces a link-lookup failure for one entry name. @@ -98,10 +108,15 @@ function stubClient( for (const e of listed) yield e; } as any); const lookup = spyOn(CatalogClient.prototype, 'lookupEntry') - .mockImplementation(async (_p, _l, name) => { + .mockImplementation(async (_p, _l, name, types = []) => { if (name === lookupFail) return err(500, 'boom'); const e = byName.get(name); - return e ? ok(e) : err(404, 'not found'); + if (!e) return err(404, 'not found'); + const aspects = + Object.entries(e.aspects ?? {}) + .filter(([, a]) => types.includes(a.aspectType!)) + .slice(0, 100); + return ok({...e, aspects: Object.fromEntries(aspects)}); }); const lookupLinks = spyOn(CatalogClient.prototype, 'lookupEntryLinks') @@ -725,3 +740,381 @@ describe('pullKnowledgeCatalog: file copies', () => { ]); }); }); + + +describe('pullKnowledgeCatalog: second-generation models', () => { + const V2 = {...OPTS, v2Aspects: true}; + const TABLE = (t: string) => + `bigquery.googleapis.com/projects/p/datasets/d/tables/${t}`; + const GRAPH = (g: string) => + `bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/${g}`; + // A model with everything a flag-on push stores. Its entities, metrics and + // relationships are in name order, as a pull returns them, since + // relationship order reaches the links and the DDL. + const SHOP = `version: "0.2.0.dev0/google" +semantic_model: + - name: shop + description: The shop + ai_context: + instructions: Use for sales questions. + synonyms: [store] + examples: [Top customers?] + custom: {owner: crm, limit: .inf} + custom_extensions: [{vendor_name: DBT, data: '{"x": 1}'}] + deployments: + - {name: main, target: ${GRAPH('main')}} + - {name: analytics, target: ${GRAPH('analytics')}, profile: analytical} + entities: + - name: customers + extends: [party] + source: ${TABLE('customers')} + primary_key: [party_id] + unique_keys: [[email]] + description: People who buy + ai_context: + synonyms: [buyers] + examples: [Big spenders?] + custom: {tier: gold, region: west} + custom_extensions: [{vendor_name: DBT, data: '{"m": "t"}'}] + fields: + - {name: party_id, expression: cust_id} + - name: email + expression: LOWER(email) + dimension: {} + label: Email + ai_context: + instructions: Mask it. + synonyms: [mail] + examples: [Gmail users] + custom: {pii: true} + custom_extensions: [{vendor_name: DBT, data: '{"t": []}'}] + - name: signup_date + datatype: Date + expression: signup + dimension: {is_time: false} + - {name: birthday, datatype: Date, expression: bday} + - {name: employees, source: ${TABLE('employees')}, primary_key: [id], + fields: [{name: id, expression: id}]} + - name: orders + source: ${TABLE('orders')} + primary_key: [id] + fields: + - {name: id, expression: id} + - name: amount + datatype: Decimal + expression: + dialects: + - {dialect: BIGQUERY, expression: SAFE_CAST(amt AS NUMERIC)} + - {dialect: SNOWFLAKE, expression: TRY_CAST(amt AS NUMBER)} + - {dialect: ANSI_SQL, expression: CAST(amt AS DECIMAL)} + - {name: cust_id, expression: cust_id} + - name: party + abstract: true + fields: + - {name: party_id, datatype: Integer, ai_context: Never show ids.} + - {name: name, datatype: String} + - {name: vips, extends: [customers], source: ${TABLE('vips')}, + primary_key: [party_id]} + relationships: + - name: customer_orders + from: orders + to: customers + from_columns: [cust_id] + to_columns: [party_id] + description: Each order has one customer + ai_context: + instructions: Join here. + synonyms: [placed_by] + examples: [Who ordered?] + custom: {weight: 2} + custom_extensions: [{vendor_name: DBT, data: '{"r": 1}'}] + - {name: order_recipient, from: orders, to: customers, + from_columns: [cust_id], to_columns: [party_id]} + - {name: orders_by_staff, from: orders, to: employees} + metrics: + - {name: avg_amount, expression: AVG(orders.amount)} + - {name: order_count, entity: orders, expression: COUNT(*)} + - name: revenue + datatype: Decimal + expression: + dialects: + - {dialect: BIGQUERY, expression: SUM(orders.amount)} + - {dialect: ANSI_SQL, expression: SUM(orders.amount)} + ai_context: {synonyms: [sales], examples: [Revenue in May?]} + custom_extensions: [{vendor_name: DBT, data: '{"k": 2}'}] +`; + const PROFILES = { + analytical: `name: analytical +entities: + - name: customers + source: ${TABLE('customers_a')} + primary_key: [cust_key] + unique_keys: [[mail]] + fields: + - {name: email, expression: mail} + - name: name + expression: {dialects: [{dialect: BIGQUERY, expression: full_name}]} + - {name: orders, source: ${TABLE('orders_a')}, fields_exclude: [cust_id]} +relationships: + - {name: customer_orders, from_columns: [customer_key], to_columns: [cust_key]} +metrics_exclude: [revenue, order_count, avg_amount] +`, + ops: `name: ops +entities: + - name: orders + source: ${TABLE('orders_ops')} + primary_key: [order_id] + fields: [{name: amount, expression: amt_ops}] +metrics_exclude: [revenue, avg_amount] +`, + qa: 'name: qa\nmetrics_exclude: "*"\n', + }; + // A vanilla model with a different third-party block on every object that + // takes one, and every construct a vanilla model keeps in GOOGLE blocks. + const DBT = (n: number) => `{vendor_name: DBT, data: '{"n": ${n}}'}`; + const GOOGLE = (data: object) => + `{vendor_name: GOOGLE, data: '${JSON.stringify(data)}'}`; + const ANSI = (sql: string) => + `{dialects: [{dialect: ANSI_SQL, expression: ${sql}}]}`; + const VANILLA = `version: "0.2.0.dev0" +semantic_model: + - name: store + custom_extensions: + - ${DBT(1)} + - ${GOOGLE({ + profiles: [{ + name: 'eu', + entities: [{ + name: 'customers', + source: TABLE('customers_eu'), + primary_key: ['cust_key'], + fields: [{name: 'id', expression: 'cust_key'}], + }], + relationships: [{ + name: 'customer_party', from_columns: ['cust_key'], to_columns: ['id'], + }], + metrics_exclude: '*', + }], + deployments: [ + {name: 'main', target: GRAPH('main')}, + {name: 'eu', target: GRAPH('eu'), profile: 'eu'}, + ], + })} + datasets: + - name: customers + source: ${TABLE('customers')} + primary_key: [id] + custom_extensions: [${DBT(2)}, ${GOOGLE({extends: ['party']})}] + fields: [{name: id, expression: ${ANSI('cust_id')}}] + - name: party + source: ${TABLE('party')} + primary_key: [id] + custom_extensions: [${DBT(4)}] + fields: + - {name: id, expression: ${ANSI('id')}, custom_extensions: [${DBT(5)}]} + relationships: + - name: customer_party + from: customers + to: party + from_columns: [id] + to_columns: [id] + custom_extensions: + - ${DBT(6)} + - ${GOOGLE({description: 'Each customer is a party'})} + metrics: + - name: customer_count + expression: ${ANSI('COUNT(*)')} + custom_extensions: [${DBT(7)}, ${GOOGLE({entity: 'customers'})}] +`; + // A model file and its profile files, as a flag-on push publishes them. + function loaded( + text = SHOP, profiles: Record = PROFILES): SemanticModel { + const model = loadModels(text).models[0]; + const named = Object.entries(profiles); + if (named.length) { + model.profiles = named.map(([name, t]) => loadProfileFile(t, name)); + } + return model; + } + // What a flag-on push of `model` generates: its entries and links, and the + // BigQuery and Spanner DDL of the model under each of its bindings. + function pushed(model: SemanticModel, overviews?: Map) { + const {entries, entryLinks} = + generateCatalogResources(model, {...V2, overviews}); + const bindings = + [model, ...(model.profiles ?? []).map(p => applyProfile(model, p))]; + const graphs = [generatePropertyGraph, generateSpannerPropertyGraph]; + const ddl = bindings.flatMap(m => graphs.map(g => g(m).ddl)); + return {entries, entryLinks, ddl}; + } + const KEY = (type: string) => `dataplex-types.global.${type}`; + const cat = new CatalogClient({} as any); + beforeAll(loadSqlEngine); + + // Pulls what a push of `text` and `profiles` stores, from entries listed in + // reverse so pull's order cannot come from the catalog's, writes the pulled + // model as files and pushes them again, which must give what the first push + // gave. Returns the first push's entries and the pull. + async function roundTrip( + text: string, profiles: Record, + overviews?: Map) { + const first = pushed(loaded(text, profiles), overviews); + stubClient([...first.entries].reverse(), first.entries); + const pulled = await pullKnowledgeCatalog(cat, V2); + // `.profile..yaml` holds the profile . + const [file, ...profileFiles] = serializeModelFiles(pulled.models[0]).files; + const again = loaded( + file.content, + Object.fromEntries( + profileFiles.map(f => [f.fileName.split('.')[2], f.content]))); + expect(pushed(again, pulled.overviews)).toEqual(first); + return {entries: first.entries, pulled}; + } + + test( + 'a Google model pulls back as pushed, with its sidecars, and pushed ' + + 'again gives what its push gave', + async () => { + const overviews = new Map([ + ['shop', {content: '# Shop', contentType: 'MARKDOWN'}], + ['shop/entities/orders', {links: [{url: 'https://wiki/orders'}]}], + ]); + const {entries, pulled} = await roundTrip(SHOP, PROFILES, overviews); + + // Push folds no GOOGLE block and leaves out a member it cannot store. + // Each profile's metrics_exclude comes back as '*' or a sorted list, + // and each metric's entity as declared. + const expected = loaded(); + expected.customExtensions = + expected.customExtensions!.filter(x => x.vendorName !== 'GOOGLE'); + delete expected.aiContext!.additionalProperties!.limit; + expected.profiles![0].metricsExclude = '*'; + expected.profiles![1].metricsExclude = ['avg_amount', 'revenue']; + for (const m of expected.metrics) m.authoredEntity = m.entity; + expect(pulled).toEqual({ + models: [expected], + warnings: [], + secondGeneration: true, + overviews, + }); + expect(Object.keys( + pulled.models[0].entities[0].aiContext!.additionalProperties!)) + .toEqual(['region', 'tier']); + + // With no metric entry, no profile excludes a metric. Guidelines that + // Knowledge Catalog wrote are left out. is_time alone marks a + // dimension. + mock.restore(); + const noMetrics = entries.filter(e => !e.name.includes('/metrics/')); + noMetrics[0].aspects![KEY('guidelines')].data!.userManaged = false; + const {fields} = noMetrics[1].aspects![KEY('schema')].data!; + delete fields.find((f: any) => f.name === 'signup_date').semantic; + stubClient(noMetrics, noMetrics); + const {models: [withoutMetrics]} = await pullKnowledgeCatalog(cat, V2); + expect(withoutMetrics.profiles!.map(p => p.metricsExclude)).toEqual([ + undefined, undefined, undefined + ]); + expect(withoutMetrics.aiContext) + .toEqual({synonyms: ['store'], examples: ['Top customers?']}); + expect(withoutMetrics.entities[0].fields[2].dimension).toEqual({ + isTime: false + }); + + // A model entry with no specVersion is first-generation. + delete noMetrics[0].aspects![KEY('semantic-model')].data!.specVersion; + expect((await pullKnowledgeCatalog(cat, V2)).secondGeneration) + .toBeUndefined(); + }); + + test('a vanilla model pulled and pushed again gives what its push gave', + () => roundTrip(VANILLA, {})); + + test( + 'each entry is read with the aspects of its type, and with the flag ' + + 'off links with the link type of the type project', + async () => { + const opts = {...OPTS, systemTypeProject: 'types'}; + const {entries} = + generateCatalogResources(loaded(), {...opts, v2Aspects: true}); + const {lookup, lookupLinks} = stubClient(entries, entries); + const type = (kind: string, t: string) => + `projects/types/locations/global/${kind}Types/${t}`; + // The aspect types read for the model, an entity, a metric and a + // relationship, each named by its id when it is in the type project. + const reads = async (v2Aspects: boolean) => { + lookup.mockClear(); + await pullKnowledgeCatalog(cat, {...opts, v2Aspects}); + const read = new Map( + lookup.mock.calls.map((c: any[]) => [entryIdOf(c[2]), c[3]])); + return [ + 'shop', 'shop/entities/orders', 'shop/metrics/revenue', + 'shop/relationships/customer_orders' + ].map(id => read.get(id)?.map(t => t.replace(type('aspect', ''), ''))); + }; + const OBJECT = ['guidelines', 'semantic-extensions', 'overview']; + expect(await reads(true)).toEqual([ + ['semantic-model', ...OBJECT], + ['semantic-entity', 'schema', 'sql-expressions', ...OBJECT], + ['semantic-metric', 'sql-expressions', ...OBJECT], + ['semantic-relationship', ...OBJECT], + ]); + expect(lookupLinks).not.toHaveBeenCalled(); + expect(await reads(false)).toEqual([ + ['semantic-model', 'guidelines'], + ['semantic-entity', 'schema', 'guidelines'], + ['semantic-metric', 'guidelines'], + undefined, + ]); + expect(lookupLinks.mock.lastCall?.[2].entryLinkTypes).toEqual([ + type('entryLink', 'schema-join') + ]); + }); + + test( + 'an entity with more than 100 aspects is read whole, and one type ' + + 'with 100 fails the pull', + async () => { + const wide = (n: number, field: Partial): SemanticModel => ({ + name: 'wide', + version: '0.2.0.dev0/google', + entities: [{ + name: 'events', + dataSource: '', + keys: [], + fields: Array.from({length: n}, (_, i) => ({name: `f${i}`, ...field})), + }], + relationships: [], + metrics: [], + }); + // 120 aspects at its fields' paths, 40 of each type. + const model = wide(40, { + aiContext: {instructions: 'Say it.', examples: ['An example']}, + customExtensions: [{vendorName: 'DBT', data: '{}'}], + }); + let {entries} = generateCatalogResources(model, V2); + const {lookup} = stubClient(entries, entries); + // The per-type reads run one at a time. + const serve = lookup.getMockImplementation(); + let inFlight = 0, peak = 0; + lookup.mockImplementation(async (...args: any[]) => { + const one = args[3].length === 1; + if (one) peak = Math.max(peak, ++inFlight); + const res = await serve(...args); + if (one) inFlight--; + return res; + }); + expect((await pullKnowledgeCatalog(cat, V2)).models).toEqual([model]); + expect(peak).toBe(1); + + mock.restore(); + ({entries} = generateCatalogResources( + wide(100, {aiContext: {instructions: 'Say it.'}}), V2)); + stubClient(entries, entries); + await expect(pullKnowledgeCatalog(cat, V2)) + .rejects.toThrow(new Error( + 'entry \'wide/entities/events\' holds 100 or more ' + + 'guidelines aspects, and Knowledge Catalog returns at most ' + + '100 in one read, so pull cannot read them all; pull ' + + 'aborted')); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts index 6513b4d45..10d77804d 100644 --- a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts @@ -8,7 +8,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import {parseOverviewSidecar, SemanticModelLayout} from '../../../src/libts/layouts/semantic-model'; +import {formatOverviewSidecar, parseOverviewSidecar, SemanticModelLayout} from '../../../src/libts/layouts/semantic-model'; let root: string; let catalogPath: string; @@ -484,13 +484,19 @@ describe('SemanticModelLayout profile name rules', () => { describe('SemanticModelLayout overview sidecars', () => { test('parseOverviewSidecar reads the frontmatter and trimmed body, and ' + - 'rejects a frontmatter it cannot hold, naming the file', + 'rejects a frontmatter it cannot hold, naming the file; ' + + 'formatOverviewSidecar writes them back', () => { const file = 'm/metrics/x.overview.md'; const read = (text: string) => parseOverviewSidecar(text, file); - expect(read('---\ncontentType: HTML\nlinks:\n' + - ' - url: https://x.test\n title: Guide\n' + - ' - url: https://y.test\n---\n\n

Hi

\n')) + const text = '---\ncontentType: HTML\nlinks:\n' + + ' - url: https://x.test\n title: Guide\n' + + ' - url: https://y.test\n---\n

Hi

\n'; + expect(formatOverviewSidecar(read(text))).toBe(text); + // Content that opens with a `---` line gets an empty frontmatter. + const ruled = {content: '---\ntitle: Orders\n---\nOne row.'}; + expect(read(formatOverviewSidecar(ruled))).toEqual(ruled); + expect(read(text.replace('\n

', '\n\n

'))) .toEqual({ content: '

Hi

', contentType: 'HTML', diff --git a/toolbox/mdcode/tests/tool/commands.test.ts b/toolbox/mdcode/tests/tool/commands.test.ts index 558834518..53d582a40 100644 --- a/toolbox/mdcode/tests/tool/commands.test.ts +++ b/toolbox/mdcode/tests/tool/commands.test.ts @@ -47,7 +47,7 @@ entities: let dir = ''; let cwd = ''; -let savedEnv: string|undefined; +let savedEnv: NodeJS.ProcessEnv = {}; function writeWorkspace(): void { fs.writeFileSync( @@ -62,7 +62,7 @@ beforeEach(() => { cwd = process.cwd(); dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kcmd-commands-')); process.chdir(dir); - savedEnv = process.env.KC_V2_ASPECTS; + savedEnv = {...process.env}; delete process.env.KC_V2_ASPECTS; spyOn(ApiContext, 'default').mockReturnValue(CTX); spyOn(console, 'log').mockImplementation(() => {}); @@ -80,10 +80,9 @@ beforeEach(() => { }); afterEach(() => { - if (savedEnv === undefined) { - delete process.env.KC_V2_ASPECTS; - } else { - process.env.KC_V2_ASPECTS = savedEnv; + for (const k of ['KC_V2_ASPECTS', 'KC_TYPE_PROJECT']) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; } process.chdir(cwd); if (dir) fs.rmSync(dir, {recursive: true, force: true}); @@ -202,9 +201,10 @@ describe('v2Aspects forwarding', () => { expect(pullSpy.mock.calls[0][1].v2Aspects).toBe(false); }); - test('pull forwards v2Aspects: true when KC_V2_ASPECTS is "1"', async () => { + test('pull forwards v2Aspects: true and KC_TYPE_PROJECT', async () => { writeWorkspace(); process.env.KC_V2_ASPECTS = '1'; + process.env.KC_TYPE_PROJECT = 'types'; const pullSpy = spyOn(pullKc, 'pullKnowledgeCatalog') .mockResolvedValue({models: [], warnings: []}); @@ -213,6 +213,7 @@ describe('v2Aspects forwarding', () => { expect(code).toBe(0); expect(pullSpy).toHaveBeenCalledTimes(1); expect(pullSpy.mock.calls[0][1].v2Aspects).toBe(true); + expect(pullSpy.mock.calls[0][1].systemTypeProject).toBe('types'); }); }); @@ -1555,6 +1556,8 @@ describe('pull with stored file copies', () => { 'commerce.profiles/prod.yml', 'commerce.profile.dev.yml', 'commerce.yml', + // Only a second-generation pull writes overview sidecars. + 'commerce/entities/Customer.overview.md', ]; pulled([...names.map(fileName => ({fileName, content: 'x'})), MODEL_COPY]); @@ -1562,6 +1565,10 @@ describe('pull with stored file copies', () => { for (const name of names) { expect(messages(warnSpy)).toContain(`skipped '${name}' from the catalog`); } + expect(messages(warnSpy)).toContain( + `pull writes only 'commerce.yaml', 'commerce.profile..yaml' ` + + `and 'commerce.profiles/.yaml' inside the entry group ` + + `directory.`); expect(fs.existsSync(local('commerce.profiles'))).toBe(false); expect(fs.existsSync(local('commerce.yml'))).toBe(false); }); @@ -1707,18 +1714,86 @@ describe('pull with stored file copies', () => { expect(fs.existsSync(path.join(outside, 'prod.yaml'))).toBe(true); }); - test('with KC_V2_ASPECTS on, pull leaves local profile files unnamed', + test('with KC_V2_ASPECTS on, a second-generation pull writes the model, ' + + 'profile and sidecar files and names or deletes stale ones', async () => { process.env.KC_V2_ASPECTS = '1'; withStaleProfile(); + const staleSidecar = 'commerce/metrics/old.overview.md'; + fs.mkdirSync(path.dirname(local(staleSidecar)), {recursive: true}); + fs.writeFileSync(local(staleSidecar), 'Old.\n'); const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); - spyOn(pullKc, 'pullKnowledgeCatalog') - .mockResolvedValue({models: [COMMERCE], warnings: []}); + const model: SemanticModel = {...COMMERCE, profiles: [{ + name: 'prod', entities: [], relationships: [], metricsExclude: '*'}], + entities: [{ + name: 'Customer', dataSource: '', keys: [], fields: [{name: 'id'}], + }]}; + const pulled = spyOn(pullKc, 'pullKnowledgeCatalog') + .mockResolvedValue({models: [model], warnings: []}); expect(await pull({forceRemove: true})).toBe(0); expect(messages(warnSpy)).not.toContain('profile file'); expect(fs.existsSync(local('commerce.profile.dev.yaml'))).toBe(true); - expect(fs.existsSync(local('commerce.profile.prod.yaml'))).toBe(true); + expect(fs.readFileSync(local('commerce.profile.prod.yaml'), 'utf8')) + .toBe('name: prod\n'); + + const overviews = new Map([ + ['commerce', {content: '# Commerce\n', contentType: 'HTML', + links: [{url: 'https://x.test'}]}], + ['commerce/entities/Customer', {content: 'A buyer.\n'}], + ['commerce/metrics/Revenue', {content: 'Money in.\n'}], + ['commerce/relationships/placed', {content: 'Orders.\n'}], + // Push reads no sidecar at these paths. + ['commerce/notes/x', {content: 'x\n'}], + ['commerce/entities/a/b', {content: 'x\n'}], + ]); + pulled.mockResolvedValue( + {models: [model], warnings: [], secondGeneration: true, overviews}); + // The catalog holds no profile's actions, so the local ones stay. + const actions = [{name: 'refund', executor: null}]; + fs.writeFileSync(local('commerce.profile.prod.yaml'), + 'name: prod\nactions: [{name: refund, executor: null}]\n'); + const {files} = osi.serializeModelFiles( + {...model, profiles: [{...model.profiles![0], actions}]}); + warnSpy.mockClear(); + expect(await pull()).toBe(0); + // Sources only in profile files draw no missing-source warning. + expect(messages(warnSpy)).not.toMatch(/no source|no expression/); + expect(files).toHaveLength(2); + for (const [name, text] of [ + ...files.map(f => [f.fileName, f.content]), + ['commerce.overview.md', + '---\ncontentType: HTML\nlinks:\n - url: https://x.test\n' + + '---\n# Commerce\n'], + ['commerce/entities/Customer.overview.md', 'A buyer.\n'], + ['commerce/metrics/Revenue.overview.md', 'Money in.\n'], + ['commerce/relationships/placed.overview.md', 'Orders.\n'], + ]) { + expect(fs.readFileSync(local(name), 'utf8')).toBe(text); + } + for (const name of ['commerce/notes/x.overview.md', + 'commerce/entities/a/b.overview.md']) { + expect(fs.existsSync(local(name))).toBe(false); + expect(messages(warnSpy)) + .toContain(`skipped '${name}' from the catalog`); + } + for (const [kind, name] of [['profile file', 'commerce.profile.dev.yaml'], + ['overview sidecar', staleSidecar]]) { + expect(messages(warnSpy)) + .toContain(`local ${kind} ${shown(name)} is not in the catalog`); + expect(fs.existsSync(local(name))).toBe(true); + } + + expect(await pull({forceRemove: true})).toBe(0); + expect(fs.existsSync(local('commerce.profile.dev.yaml'))).toBe(false); + expect(fs.existsSync(local(staleSidecar))).toBe(false); + + // A vanilla model declares no action, so its profiles get none. + pulled.mockResolvedValue({models: [{...model, version: '0.2.0.dev0'}], + warnings: [], secondGeneration: true}); + expect(await pull()).toBe(0); + expect(fs.readFileSync(local('commerce.yaml'), 'utf8')) + .not.toContain('refund'); }); });