diff --git a/packages/project/src/merge/map-uuids.ts b/packages/project/src/merge/map-uuids.ts index 1727e2f91..781a2449d 100644 --- a/packages/project/src/merge/map-uuids.ts +++ b/packages/project/src/merge/map-uuids.ts @@ -8,8 +8,9 @@ * If step names stay the same, this is trivial */ -import { Workflow } from '@openfn/lexicon'; +import { Job } from '@openfn/lexicon'; import { Project } from '../Project'; +import Workflow from '../Workflow'; // changing parent and changing id is probably a new node // changing id but preserving parent should map @@ -39,82 +40,213 @@ export interface MappingResults { } export default (source: Workflow, target: Workflow): MappingResults => { - const nodeMapping: Record = {}; const edgeMapping: Record = {}; - // const mapping: Record = {}; - // sets all nodes as not existing in target - const targetIds: Record = {}; - for (const tstep of target.steps) { - nodeMapping[tstep.id] = null; - targetIds[tstep.id] = tstep.openfn?.uuid || tstep.id; + // Map by id + let { mapping, idMap, unmatchedSource: sourceSteps, unmatchedTarget: targetSteps } = mapById(source.steps, target.steps, {}, new Map()); - // dealing with edges - const next = - typeof tstep.next === 'string' - ? { [tstep.next]: true } - : tstep.next || {}; - for (const [toNode, toValue] of Object.entries(next)) { - const edgeId = tstep.id + '-' + toNode; - edgeMapping[edgeId] = null; - targetIds[edgeId] = - typeof toValue === 'object' ? toValue.openfn?.uuid || edgeId : edgeId; - } + // Map by parent + ({ mapping, idMap, unmatchedSource: sourceSteps, unmatchedTarget: targetSteps } = mapByParent(sourceSteps, targetSteps, mapping, idMap, source.steps)); + + // Map by children + ({ mapping, idMap, unmatchedSource: sourceSteps, unmatchedTarget: targetSteps } = mapByChildren(sourceSteps, targetSteps, mapping, idMap, source.steps)); + + // Map by expression + ({ mapping, idMap, unmatchedSource: sourceSteps, unmatchedTarget: targetSteps } = mapByExpression(sourceSteps, targetSteps, mapping, idMap)); + + for (const s of sourceSteps) { + mapping[s.id] = true; } - for (const sstep of source.steps) { - const ex = nodeMapping[sstep.id]; - // matched nodes are mapped - if (ex === null) nodeMapping[sstep.id] = targetIds[sstep.id]; - else if (ex === undefined) nodeMapping[sstep.id] = true; // true to create id for new nodes + // EDGE MAPPING + // this needs to be a bit smart!!! + // get edges + const targetEdges = getEdges(target.steps); + const sourceEdges = getEdges(source.steps); + for (const [parent, children] of Object.entries(targetEdges)) { + for (const child of children) { + const edgeKey = `${parent}-${child}`; + edgeMapping[edgeKey] = null; + } + } - // dealing with edges - const next = - typeof sstep.next === 'string' - ? { [sstep.next]: true } - : sstep.next || {}; - for (const toNode of Object.keys(next)) { - const edgeId = sstep.id + '-' + toNode; - const ex = edgeMapping[edgeId]; - if (ex === null) edgeMapping[edgeId] = targetIds[edgeId]; - else if (ex === undefined) edgeMapping[edgeId] = true; + for (const [parent, children] of Object.entries(sourceEdges)) { + for (const child of children) { + const tparent = idMap.has(parent) ? idMap.get(parent) : parent; + const tchild = idMap.has(child) ? idMap.get(child) : child; + const edgeKey = `${parent}-${child}`; + const targetEdgeKey = `${tparent}-${tchild}`; + if (edgeMapping[targetEdgeKey] === null) + edgeMapping[edgeKey] = getEdgeUuid(tparent, tchild, target.steps); + else edgeMapping[edgeKey] = true; } } + return { - nodes: nodeMapping, + nodes: mapping, edges: edgeMapping, }; }; -// if newchildren has some additional nodes, that's fine. it's the same children! -// when a node is missing, nope, not the same children -function sameChildren(mainChildren: string[], newChildren: string[]) { - if (newChildren.length < mainChildren) return false; - for (const child of mainChildren) { - if (!newChildren.includes(child)) return false; +function getEdgeUuid( + parentId: string, + childId: string, + steps: Workflow['steps'] +) { + const parentNode = steps.find((step) => step.id === parentId); + if (!parentNode) return; + if (typeof parentNode.next !== 'object') return; + const edge = parentNode.next[childId]; + return edge?.openfn?.uuid; +} +function getStepUuid(step: Workflow['steps'][number]) { + return step?.openfn?.uuid || step.id; +} +function getEdges(steps: Workflow['steps']) { + const edges: Record = {}; + for (const step of steps) { + const next = + typeof step.next === 'string' ? { [step.next]: true } : step.next || {}; + + for (const toNode of Object.keys(next)) { + if (!Array.isArray(edges[step.id])) edges[step.id] = [toNode]; + else edges[step.id].push(toNode); + } } - return true; + return edges; +} + +function getParent(id: string, steps: Workflow['steps']) { + const edges = getEdges(steps); + const found = Object.entries(edges).find(([parent, children]) => + children.includes(id) + ); + if (!found) return; + return found[0]; // getting the parent id +} + +interface Pool { + source: Workflow['steps']; + target: Workflow['steps']; } -function getNodesWithParent(parent: string, parentsMap: Map) { - const matches: string[] = []; - for (const [child, child_parent] of parentsMap.entries()) { - if (parent === child_parent) matches.push(child as string); +interface MapStepResult { + mapping: Record; + idMap: Map; + pool: Pool; +} + +function mapById(sourceSteps, targetSteps, mapping, idMap) { + const unmatchedSource = []; + const unmatchedTarget = [...targetSteps]; + for (const s of sourceSteps) { + const t = targetSteps.find(t => t.id === s.id); + if (t) { + mapping[s.id] = t.openfn?.uuid; + idMap.set(s.id, t.id); + unmatchedTarget.splice(unmatchedTarget.indexOf(t), 1); + } else { + unmatchedSource.push(s); + } } - return matches; + return { mapping, idMap, unmatchedSource, unmatchedTarget }; } -function getNodeWithChildren( - newChildren: string[], - childrenMap: Map -) { - // which node has all of its children in here - for (const [node, children] of childrenMap.entries()) { +function mapByParent(sourceSteps, targetSteps, mapping, idMap, sourceAll) { + const unmatchedSource = []; + const unmatchedTarget = [...targetSteps]; + for (const s of sourceSteps) { + const parent = getParent(s.id, sourceAll); + if (parent) { + const candidates = findByParent(parent, targetSteps); + if (candidates.length === 1) { + const t = candidates[0]; + mapping[s.id] = getStepUuid(t); + idMap.set(s.id, t.id); + unmatchedTarget.splice(unmatchedTarget.indexOf(t), 1); + continue; + } + } + unmatchedSource.push(s); + } + return { mapping, idMap, unmatchedSource, unmatchedTarget }; +} + +function mapByChildren(sourceSteps, targetSteps, mapping, idMap, sourceAll) { + const unmatchedSource = []; + const unmatchedTarget = [...targetSteps]; + for (const s of sourceSteps) { + const children = getEdges(sourceAll)[s.id]; + if (children) { + const candidates = findByChildren(children, targetSteps); + if (candidates.length === 1) { + const t = candidates[0]; + mapping[s.id] = getStepUuid(t); + idMap.set(s.id, t.id); + unmatchedTarget.splice(unmatchedTarget.indexOf(t), 1); + continue; + } + } + unmatchedSource.push(s); + } + return { mapping, idMap, unmatchedSource, unmatchedTarget }; +} + +function mapByExpression(sourceSteps, targetSteps, mapping, idMap) { + const unmatchedSource = []; + const unmatchedTarget = [...targetSteps]; + for (const s of sourceSteps) { + const candidates = findByExpression((s as Job).expression, targetSteps); + if (candidates.length === 1) { + const t = candidates[0]; + mapping[s.id] = getStepUuid(t); + idMap.set(s.id, t.id); + unmatchedTarget.splice(unmatchedTarget.indexOf(t), 1); + continue; + } + unmatchedSource.push(s); + } + return { mapping, idMap, unmatchedSource, unmatchedTarget }; +} + +// very flawed, due to the commented code snipped every step comes with. +function findByExpression(exp: string, steps: Workflow['steps']) { + // find a node having the same expression + return steps.filter( + (step: Job) => + step.expression && !!step.expression.trim() && step.expression === exp + ); +} + +function findByParent(parentId: string, steps: Workflow['steps']) { + // returns all nodes having the parentId as parent + const edges = getEdges(steps); + const matched = edges[parentId]; + if (!matched || matched.length === 0) return []; + return steps.filter((step) => matched.includes(step.id)); +} + +function findByChildren(childIds: string[], steps: Workflow['steps']) { + // return best candidates for the child ids + const edges = getEdges(steps); + const countIndex: Record = {}; + for (const [parent, children] of Object.entries(edges)) { + let count = 0; for (const child of children) { - if (!newChildren.includes(child)) return; + if (childIds.includes(child)) count++; } - return node as string; + if (count > 0) countIndex[parent] = count; } + const parents = Object.entries(countIndex) + .sort(([p1, c1], [p2, c2]) => c2 - c1) + .map(([parent, count]) => parent); + + const stepsIndex = steps.reduce((a, b) => { + a[b.id] = b; + return a; + }, {} as Record); + + return parents.map((p) => stepsIndex[p]); } // Get an array of all UUIDS diff --git a/packages/project/test/merge/map-uuids.test.ts b/packages/project/test/merge/map-uuids.test.ts index b9ebcca4c..d32b379f0 100644 --- a/packages/project/test/merge/map-uuids.test.ts +++ b/packages/project/test/merge/map-uuids.test.ts @@ -36,16 +36,16 @@ test('node name changes but no positional change', (t) => { t.deepEqual(result.nodes, { ['trigger']: target.getUUID('trigger'), + ['a']: target.getUUID('c'), ['b']: target.getUUID('b'), ['c']: null, - ['a']: true, }); // no retained edges t.deepEqual(result.edges, { + ['trigger-a']: target.getUUID('trigger-c'), + ['a-b']: target.getUUID('c-b'), ['trigger-c']: null, ['c-b']: null, - ['trigger-a']: true, - ['a-b']: true, }); }); @@ -71,3 +71,159 @@ test('one connecting node missing', (t) => { ['z-b']: null, }); }); + +test('step removed from the middle', (t) => { + const source = generateWorkflow(['a-b', 'b-c']); + const target = generateWorkflow(['a-b', 'b-x', 'x-c']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: target.getUUID('c'), + ['x']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['b-c']: true, + ['b-x']: null, + ['x-c']: null, + }); +}); + +test('step added to the middle', (t) => { + const source = generateWorkflow(['a-b', 'b-x', 'x-c']); + const target = generateWorkflow(['a-b', 'b-c']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['x']: true, + ['c']: target.getUUID('c'), + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['b-x']: true, + ['x-c']: true, + ['b-c']: null, + }); +}); + +test('children re-ordered. same parent', (t) => { + const source = generateWorkflow(['a-b', 'a-c']); + const target = generateWorkflow(['a-c', 'a-b']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: target.getUUID('c'), + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['a-c']: target.getUUID('a-c'), + }); +}); + +test('step with changed id but same expression', (t) => { + const source = generateWorkflow(['a-b']); + const target = generateWorkflow(['a-x']); + // set expression to b and x + const expression = 'fn(s => s)'; + source.set('b', { expression }); + target.set('x', { expression }); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('x'), + ['x']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-x'), + ['a-x']: null, + }); +}); + +test('edge removed between existing nodes', (t) => { + const source = generateWorkflow(['a-b']); + const target = generateWorkflow(['a-b', 'a-c']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['a-c']: null, + }); +}); + +test('new edge added', (t) => { + const source = generateWorkflow(['a-b', 'a-c']); + const target = generateWorkflow(['a-b']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: true, + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['a-c']: true, + }); +}); + +test('multiple steps and edges added and removed', (t) => { + const source = generateWorkflow(['a-b', 'b-c', 'c-d']); + const target = generateWorkflow(['a-x', 'x-b', 'b-y', 'y-c', 'c-d', 'd-z']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: target.getUUID('c'), + ['d']: target.getUUID('d'), + ['x']: null, + ['y']: null, + ['z']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: true, + ['b-c']: true, + ['c-d']: target.getUUID('c-d'), + ['a-x']: null, + ['x-b']: null, + ['b-y']: null, + ['y-c']: null, + ['d-z']: null, + }); +}); + +test('steps removed from a workflow', (t) => { + const source = generateWorkflow(['a-b', 'a-c', 'b-d', 'c-d']); + const target = generateWorkflow(['a-x', 'x-b', 'a-c', 'b-d', 'c-d', 'd-e']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: target.getUUID('c'), + ['d']: target.getUUID('d'), + ['e']: null, + ['x']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: true, + ['a-c']: target.getUUID('a-c'), + ['b-d']: target.getUUID('b-d'), + ['c-d']: target.getUUID('c-d'), + ['a-x']: null, + ['x-b']: null, + ['d-e']: null, + }); +});