From ad9df8961ce5c2726a4f5389e57158d89fe61afd Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 3 Sep 2025 09:24:34 +0000 Subject: [PATCH 1/5] feat: implement merge algorithm --- packages/project/src/merge/map-uuids.ts | 199 +++++++++++++----- packages/project/test/merge/map-uuids.test.ts | 8 +- 2 files changed, 150 insertions(+), 57 deletions(-) diff --git a/packages/project/src/merge/map-uuids.ts b/packages/project/src/merge/map-uuids.ts index 1727e2f91..f2c07bfab 100644 --- a/packages/project/src/merge/map-uuids.ts +++ b/packages/project/src/merge/map-uuids.ts @@ -8,7 +8,7 @@ * If step names stay the same, this is trivial */ -import { Workflow } from '@openfn/lexicon'; +import { Job, Workflow } from '@openfn/lexicon'; import { Project } from '../Project'; // changing parent and changing id is probably a new node @@ -39,82 +39,175 @@ export interface MappingResults { } export default (source: Workflow, target: Workflow): MappingResults => { + // create nodes, edges const nodeMapping: Record = {}; const edgeMapping: Record = {}; - // const mapping: Record = {}; + const idMap = new Map(); + // NODE MAPPING + for (const target_step of target.steps) nodeMapping[target_step.id] = null; + for (const source_step of source.steps) { + if (!source_step.id) continue; // yh. we'll always have it. + let result: Workflow['steps'] = target.steps; + + // finding by id + result = findById(source_step.id, result.length ? result : target.steps); + if (result.length === 1) { + nodeMapping[source_step.id] = getStepUuid(result[0]); + idMap.set(source_step.id, result[0].id); + continue; + } - // sets all nodes as not existing in target - const targetIds: Record = {}; - for (const tstep of target.steps) { - nodeMapping[tstep.id] = null; - targetIds[tstep.id] = tstep.openfn?.uuid || tstep.id; + // finding by parent + const parent = getParent(source_step.id, source.steps); + if (parent) { + result = findByParent(parent, result.length ? result : target.steps); + if (result.length === 1) { + nodeMapping[source_step.id] = getStepUuid(result[0]); + idMap.set(source_step.id, result[0].id); + continue; + } + } - // dealing with edges - const next = - typeof tstep.next === 'string' - ? { [tstep.next]: true } - : tstep.next || {}; - for (const [toNode, toValue] of Object.entries(next)) { - const edgeId = tstep.id + '-' + toNode; - edgeMapping[edgeId] = null; - targetIds[edgeId] = - typeof toValue === 'object' ? toValue.openfn?.uuid || edgeId : edgeId; + // finding by children + let tmp_final: Workflow['steps'][number]; + const children = getEdges(source.steps)[source_step.id]; + result = findByChildren(children, result.length ? result : target.steps); + if (result.length === 1) { + nodeMapping[source_step.id] = getStepUuid(result[0]); + idMap.set(source_step.id, result[0].id); + continue; + } else if (result.length) { + tmp_final = result[0]; + } + + // finding by expression | very flawed + result = findByExpression( + (source_step as Job).expression, + result.length ? result : target.steps + ); + if (result.length === 1) { + nodeMapping[source_step.id] = getStepUuid(result[0]); + idMap.set(source_step.id, result[0].id); + continue; } + + // if find by expression did nothing but children did then use that. + if (tmp_final) { + nodeMapping[source_step.id] = getStepUuid(tmp_final); + idMap.set(source_step.id, result[0].id); + } else if (nodeMapping[source_step.id] === undefined) + nodeMapping[source_step.id] = true; } - for (const sstep of source.steps) { - const ex = nodeMapping[sstep.id]; - // matched nodes are mapped - if (ex === null) nodeMapping[sstep.id] = targetIds[sstep.id]; - else if (ex === undefined) nodeMapping[sstep.id] = true; // true to create id for new nodes + // EDGE MAPPING + // this needs to be a bit smart!!! + // get edges + const targetEdges = getEdges(target.steps); + const sourceEdges = getEdges(source.steps); + for (const [parent, children] of Object.entries(targetEdges)) { + for (const child of children) { + const edgeKey = `${parent}-${child}`; + edgeMapping[edgeKey] = null; + } + } - // dealing with edges - const next = - typeof sstep.next === 'string' - ? { [sstep.next]: true } - : sstep.next || {}; - for (const toNode of Object.keys(next)) { - const edgeId = sstep.id + '-' + toNode; - const ex = edgeMapping[edgeId]; - if (ex === null) edgeMapping[edgeId] = targetIds[edgeId]; - else if (ex === undefined) edgeMapping[edgeId] = true; + for (const [parent, children] of Object.entries(sourceEdges)) { + for (const child of children) { + const tparent = idMap.has(parent) ? idMap.get(parent) : parent; + const tchild = idMap.has(child) ? idMap.get(child) : child; + const edgeKey = `${parent}-${child}`; + const targetEdgeKey = `${tparent}-${tchild}`; + if (edgeMapping[targetEdgeKey] === null) + edgeMapping[edgeKey] = getEdgeUuid(tparent, tchild, target.steps); + else edgeMapping[edgeKey] = true; } } + return { nodes: nodeMapping, edges: edgeMapping, }; }; -// if newchildren has some additional nodes, that's fine. it's the same children! -// when a node is missing, nope, not the same children -function sameChildren(mainChildren: string[], newChildren: string[]) { - if (newChildren.length < mainChildren) return false; - for (const child of mainChildren) { - if (!newChildren.includes(child)) return false; - } - return true; +function getEdgeUuid( + parentId: string, + childId: string, + steps: Workflow['steps'] +) { + const parentNode = steps.find((step) => step.id === parentId); + if (!parentNode) return; + if (typeof parentNode.next !== 'object') return; + const edge = parentNode.next[childId]; + return edge?.openfn?.uuid; } +function getStepUuid(step: Workflow['steps'][number]) { + return step?.openfn?.uuid || step.id; +} +function getEdges(steps: Workflow['steps']) { + const edges: Record = {}; + for (const step of steps) { + const next = + typeof step.next === 'string' ? { [step.next]: true } : step.next || {}; -function getNodesWithParent(parent: string, parentsMap: Map) { - const matches: string[] = []; - for (const [child, child_parent] of parentsMap.entries()) { - if (parent === child_parent) matches.push(child as string); + for (const toNode of Object.keys(next)) { + if (!Array.isArray(edges[step.id])) edges[step.id] = [toNode]; + else edges[step.id].push(toNode); + } } - return matches; + return edges; } -function getNodeWithChildren( - newChildren: string[], - childrenMap: Map -) { - // which node has all of its children in here - for (const [node, children] of childrenMap.entries()) { +function getParent(id: string, steps: Workflow['steps']) { + const edges = getEdges(steps); + const found = Object.entries(edges).find(([parent, children]) => + children.includes(id) + ); + if (!found) return; + return found[0]; // getting the parent id +} + +function findById(id: string, steps: Workflow['steps']) { + return steps.filter((step: Job) => step.id === id); +} + +// very flawed, due to the commented code snipped every step comes with. +function findByExpression(exp: string, steps: Workflow['steps']) { + // find a node having the same expression + return steps.filter( + (step: Job) => + step.expression && !!step.expression.trim() && step.expression === exp + ); +} + +function findByParent(parentId: string, steps: Workflow['steps']) { + // returns all nodes having the parentId as parent + const edges = getEdges(steps); + const matched = edges[parentId]; + if (!matched || matched.length === 0) return []; + return steps.filter((step) => matched.includes(step.id)); +} + +function findByChildren(childIds: string[], steps: Workflow['steps']) { + // return best candidates for the child ids + const edges = getEdges(steps); + const countIndex: Record = {}; + for (const [parent, children] of Object.entries(edges)) { + let count = 0; for (const child of children) { - if (!newChildren.includes(child)) return; + if (childIds.includes(child)) count++; } - return node as string; + if (count > 0) countIndex[parent] = count; } + const parents = Object.entries(countIndex) + .sort(([p1, c1], [p2, c2]) => c2 - c1) + .map(([parent, count]) => parent); + + const stepsIndex = steps.reduce((a, b) => { + a[b.id] = b; + return a; + }, {} as Record); + + return parents.map((p) => stepsIndex[p]); } // Get an array of all UUIDS diff --git a/packages/project/test/merge/map-uuids.test.ts b/packages/project/test/merge/map-uuids.test.ts index b9ebcca4c..967db7895 100644 --- a/packages/project/test/merge/map-uuids.test.ts +++ b/packages/project/test/merge/map-uuids.test.ts @@ -36,20 +36,20 @@ test('node name changes but no positional change', (t) => { t.deepEqual(result.nodes, { ['trigger']: target.getUUID('trigger'), + ['a']: target.getUUID('c'), ['b']: target.getUUID('b'), ['c']: null, - ['a']: true, }); // no retained edges t.deepEqual(result.edges, { + ['trigger-a']: target.getUUID('trigger-c'), + ['a-b']: target.getUUID('c-b'), ['trigger-c']: null, ['c-b']: null, - ['trigger-a']: true, - ['a-b']: true, }); }); -test('one connecting node missing', (t) => { +test.only('one connecting node missing', (t) => { const source = generateWorkflow(['a-b', 'b-c', 'b-d']); const target = generateWorkflow(['a-z', 'z-b', 'b-c', 'b-d']); From eb3f21174cd5b7f2ef4d04dedcc921aeccd4fc05 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 3 Sep 2025 10:02:45 +0000 Subject: [PATCH 2/5] feat: only iterate over remaining nodes --- packages/project/src/merge/map-uuids.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/project/src/merge/map-uuids.ts b/packages/project/src/merge/map-uuids.ts index f2c07bfab..d6dd22979 100644 --- a/packages/project/src/merge/map-uuids.ts +++ b/packages/project/src/merge/map-uuids.ts @@ -47,10 +47,13 @@ export default (source: Workflow, target: Workflow): MappingResults => { for (const target_step of target.steps) nodeMapping[target_step.id] = null; for (const source_step of source.steps) { if (!source_step.id) continue; // yh. we'll always have it. - let result: Workflow['steps'] = target.steps; + const sampleSteps: Workflow['steps'] = target.steps.filter( + (step) => !idMap.has(step.id) + ); + let result: Workflow['steps'] = []; // finding by id - result = findById(source_step.id, result.length ? result : target.steps); + result = findById(source_step.id, result.length ? result : sampleSteps); if (result.length === 1) { nodeMapping[source_step.id] = getStepUuid(result[0]); idMap.set(source_step.id, result[0].id); @@ -60,7 +63,7 @@ export default (source: Workflow, target: Workflow): MappingResults => { // finding by parent const parent = getParent(source_step.id, source.steps); if (parent) { - result = findByParent(parent, result.length ? result : target.steps); + result = findByParent(parent, result.length ? result : sampleSteps); if (result.length === 1) { nodeMapping[source_step.id] = getStepUuid(result[0]); idMap.set(source_step.id, result[0].id); @@ -71,7 +74,7 @@ export default (source: Workflow, target: Workflow): MappingResults => { // finding by children let tmp_final: Workflow['steps'][number]; const children = getEdges(source.steps)[source_step.id]; - result = findByChildren(children, result.length ? result : target.steps); + result = findByChildren(children, result.length ? result : sampleSteps); if (result.length === 1) { nodeMapping[source_step.id] = getStepUuid(result[0]); idMap.set(source_step.id, result[0].id); @@ -83,7 +86,7 @@ export default (source: Workflow, target: Workflow): MappingResults => { // finding by expression | very flawed result = findByExpression( (source_step as Job).expression, - result.length ? result : target.steps + result.length ? result : sampleSteps ); if (result.length === 1) { nodeMapping[source_step.id] = getStepUuid(result[0]); From 7dbc611110bbb079708f0a20aaf701b6bc7af287 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 3 Sep 2025 11:51:36 +0000 Subject: [PATCH 3/5] tests: add several tests --- packages/project/test/merge/map-uuids.test.ts | 158 +++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/packages/project/test/merge/map-uuids.test.ts b/packages/project/test/merge/map-uuids.test.ts index 967db7895..d32b379f0 100644 --- a/packages/project/test/merge/map-uuids.test.ts +++ b/packages/project/test/merge/map-uuids.test.ts @@ -49,7 +49,7 @@ test('node name changes but no positional change', (t) => { }); }); -test.only('one connecting node missing', (t) => { +test('one connecting node missing', (t) => { const source = generateWorkflow(['a-b', 'b-c', 'b-d']); const target = generateWorkflow(['a-z', 'z-b', 'b-c', 'b-d']); @@ -71,3 +71,159 @@ test.only('one connecting node missing', (t) => { ['z-b']: null, }); }); + +test('step removed from the middle', (t) => { + const source = generateWorkflow(['a-b', 'b-c']); + const target = generateWorkflow(['a-b', 'b-x', 'x-c']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: target.getUUID('c'), + ['x']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['b-c']: true, + ['b-x']: null, + ['x-c']: null, + }); +}); + +test('step added to the middle', (t) => { + const source = generateWorkflow(['a-b', 'b-x', 'x-c']); + const target = generateWorkflow(['a-b', 'b-c']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['x']: true, + ['c']: target.getUUID('c'), + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['b-x']: true, + ['x-c']: true, + ['b-c']: null, + }); +}); + +test('children re-ordered. same parent', (t) => { + const source = generateWorkflow(['a-b', 'a-c']); + const target = generateWorkflow(['a-c', 'a-b']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: target.getUUID('c'), + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['a-c']: target.getUUID('a-c'), + }); +}); + +test('step with changed id but same expression', (t) => { + const source = generateWorkflow(['a-b']); + const target = generateWorkflow(['a-x']); + // set expression to b and x + const expression = 'fn(s => s)'; + source.set('b', { expression }); + target.set('x', { expression }); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('x'), + ['x']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-x'), + ['a-x']: null, + }); +}); + +test('edge removed between existing nodes', (t) => { + const source = generateWorkflow(['a-b']); + const target = generateWorkflow(['a-b', 'a-c']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['a-c']: null, + }); +}); + +test('new edge added', (t) => { + const source = generateWorkflow(['a-b', 'a-c']); + const target = generateWorkflow(['a-b']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: true, + }); + t.deepEqual(result.edges, { + ['a-b']: target.getUUID('a-b'), + ['a-c']: true, + }); +}); + +test('multiple steps and edges added and removed', (t) => { + const source = generateWorkflow(['a-b', 'b-c', 'c-d']); + const target = generateWorkflow(['a-x', 'x-b', 'b-y', 'y-c', 'c-d', 'd-z']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: target.getUUID('c'), + ['d']: target.getUUID('d'), + ['x']: null, + ['y']: null, + ['z']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: true, + ['b-c']: true, + ['c-d']: target.getUUID('c-d'), + ['a-x']: null, + ['x-b']: null, + ['b-y']: null, + ['y-c']: null, + ['d-z']: null, + }); +}); + +test('steps removed from a workflow', (t) => { + const source = generateWorkflow(['a-b', 'a-c', 'b-d', 'c-d']); + const target = generateWorkflow(['a-x', 'x-b', 'a-c', 'b-d', 'c-d', 'd-e']); + + const result = mapUUIDs(source.workflow, target.workflow); + t.deepEqual(result.nodes, { + ['a']: target.getUUID('a'), + ['b']: target.getUUID('b'), + ['c']: target.getUUID('c'), + ['d']: target.getUUID('d'), + ['e']: null, + ['x']: null, + }); + t.deepEqual(result.edges, { + ['a-b']: true, + ['a-c']: target.getUUID('a-c'), + ['b-d']: target.getUUID('b-d'), + ['c-d']: target.getUUID('c-d'), + ['a-x']: null, + ['x-b']: null, + ['d-e']: null, + }); +}); From a3f62963ea3d1243a252236bc49f4f4a5018fb81 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Mon, 8 Sep 2025 07:25:11 +0000 Subject: [PATCH 4/5] feat: update narrowing --- packages/project/src/merge/map-uuids.ts | 139 +++++++++++++++--------- 1 file changed, 89 insertions(+), 50 deletions(-) diff --git a/packages/project/src/merge/map-uuids.ts b/packages/project/src/merge/map-uuids.ts index d6dd22979..88951dbc8 100644 --- a/packages/project/src/merge/map-uuids.ts +++ b/packages/project/src/merge/map-uuids.ts @@ -8,8 +8,9 @@ * If step names stay the same, this is trivial */ -import { Job, Workflow } from '@openfn/lexicon'; +import { Job } from '@openfn/lexicon'; import { Project } from '../Project'; +import Workflow from '../Workflow'; // changing parent and changing id is probably a new node // changing id but preserving parent should map @@ -39,67 +40,47 @@ export interface MappingResults { } export default (source: Workflow, target: Workflow): MappingResults => { - // create nodes, edges - const nodeMapping: Record = {}; const edgeMapping: Record = {}; - const idMap = new Map(); - // NODE MAPPING - for (const target_step of target.steps) nodeMapping[target_step.id] = null; - for (const source_step of source.steps) { + + // Map by id + let { mapping: nodeMapping, pool, idMap } = mapStepsById(source.steps, target.steps); + + for (const source_step of pool.source) { if (!source_step.id) continue; // yh. we'll always have it. - const sampleSteps: Workflow['steps'] = target.steps.filter( + const sampleSteps: Workflow['steps'] = pool.target.filter( (step) => !idMap.has(step.id) ); - let result: Workflow['steps'] = []; + let candidates = sampleSteps; - // finding by id - result = findById(source_step.id, result.length ? result : sampleSteps); - if (result.length === 1) { - nodeMapping[source_step.id] = getStepUuid(result[0]); - idMap.set(source_step.id, result[0].id); + // Parent + let result = mapStepByParent(source_step, source, candidates); + if (result.length) candidates = result; + if (candidates.length === 1) { + nodeMapping[source_step.id] = getStepUuid(candidates[0]); + idMap.set(source_step.id, candidates[0].id); continue; } - // finding by parent - const parent = getParent(source_step.id, source.steps); - if (parent) { - result = findByParent(parent, result.length ? result : sampleSteps); - if (result.length === 1) { - nodeMapping[source_step.id] = getStepUuid(result[0]); - idMap.set(source_step.id, result[0].id); - continue; - } - } - - // finding by children - let tmp_final: Workflow['steps'][number]; - const children = getEdges(source.steps)[source_step.id]; - result = findByChildren(children, result.length ? result : sampleSteps); - if (result.length === 1) { - nodeMapping[source_step.id] = getStepUuid(result[0]); - idMap.set(source_step.id, result[0].id); + // Children + result = mapStepByChildren(source_step, source, candidates); + if (result.length) candidates = result; + if (candidates.length === 1) { + nodeMapping[source_step.id] = getStepUuid(candidates[0]); + idMap.set(source_step.id, candidates[0].id); continue; - } else if (result.length) { - tmp_final = result[0]; } - // finding by expression | very flawed - result = findByExpression( - (source_step as Job).expression, - result.length ? result : sampleSteps - ); - if (result.length === 1) { - nodeMapping[source_step.id] = getStepUuid(result[0]); - idMap.set(source_step.id, result[0].id); + // Expression + result = mapStepByExpression(source_step, candidates); + if (result.length) candidates = result; + if (candidates.length === 1) { + nodeMapping[source_step.id] = getStepUuid(candidates[0]); + idMap.set(source_step.id, candidates[0].id); continue; } - // if find by expression did nothing but children did then use that. - if (tmp_final) { - nodeMapping[source_step.id] = getStepUuid(tmp_final); - idMap.set(source_step.id, result[0].id); - } else if (nodeMapping[source_step.id] === undefined) - nodeMapping[source_step.id] = true; + // If none matched, mark as new + nodeMapping[source_step.id] = true; } // EDGE MAPPING @@ -169,8 +150,48 @@ function getParent(id: string, steps: Workflow['steps']) { return found[0]; // getting the parent id } -function findById(id: string, steps: Workflow['steps']) { - return steps.filter((step: Job) => step.id === id); +interface Pool { + source: Workflow['steps']; + target: Workflow['steps']; +} + +interface MapStepResult { + mapping: Record; + idMap: Map; + pool: Pool; +} + +function mapStepsById( + source: Workflow['steps'], + target: Workflow['steps'] +): MapStepResult { + const targets: Record = {}; + const mapping: Record = {}; + const idMap = new Map(); + + for (const target_step of target) { + targets[target_step.id] = target_step; + } + + const removedIndexes = []; + for (let i = 0; i < source.length; i++) { + const source_step = source[i]; + if (target[source_step.id]) { + mapping[source_step.id] = target[source_step.id]?.openfn?.uuid; + idMap.set(source_step.id, target[source_step.id]?.id); + removedIndexes.push(i); + target = target.filter((t) => t !== target[source_step]); + } + } + + return { + mapping, + idMap, + pool: { + source: source.filter((_, i) => !removedIndexes.includes(i)), + target, + }, + }; } // very flawed, due to the commented code snipped every step comes with. @@ -213,5 +234,23 @@ function findByChildren(childIds: string[], steps: Workflow['steps']) { return parents.map((p) => stepsIndex[p]); } +function mapStepByParent(source_step: Workflow['steps'][number], source: Workflow['steps'], candidates: Workflow['steps']) { + const parent = getParent(source_step.id, source.steps); + if (parent) { + return findByParent(parent, candidates); + } + return []; +} + +function mapStepByChildren(source_step: Workflow['steps'][number], source: Workflow['steps'], candidates: Workflow['steps']) { + const children = getEdges(source.steps)[source_step.id]; + if(!children) return []; + return findByChildren(children, candidates); +} + +function mapStepByExpression(source_step: Workflow['steps'][number], candidates: Workflow['steps']) { + return findByExpression((source_step as Job).expression, candidates); +} + // Get an array of all UUIDS const extractUUIDS = (p: Project) => {}; From 13d8b2f5eec14394a2b1de2340a129973d617ab6 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Mon, 8 Sep 2025 07:35:54 +0000 Subject: [PATCH 5/5] feat: pipeline based --- packages/project/src/merge/map-uuids.ts | 153 ++++++++++++------------ 1 file changed, 75 insertions(+), 78 deletions(-) diff --git a/packages/project/src/merge/map-uuids.ts b/packages/project/src/merge/map-uuids.ts index 88951dbc8..781a2449d 100644 --- a/packages/project/src/merge/map-uuids.ts +++ b/packages/project/src/merge/map-uuids.ts @@ -43,44 +43,19 @@ export default (source: Workflow, target: Workflow): MappingResults => { const edgeMapping: Record = {}; // Map by id - let { mapping: nodeMapping, pool, idMap } = mapStepsById(source.steps, target.steps); + let { mapping, idMap, unmatchedSource: sourceSteps, unmatchedTarget: targetSteps } = mapById(source.steps, target.steps, {}, new Map()); - for (const source_step of pool.source) { - if (!source_step.id) continue; // yh. we'll always have it. - const sampleSteps: Workflow['steps'] = pool.target.filter( - (step) => !idMap.has(step.id) - ); - let candidates = sampleSteps; + // Map by parent + ({ mapping, idMap, unmatchedSource: sourceSteps, unmatchedTarget: targetSteps } = mapByParent(sourceSteps, targetSteps, mapping, idMap, source.steps)); - // Parent - let result = mapStepByParent(source_step, source, candidates); - if (result.length) candidates = result; - if (candidates.length === 1) { - nodeMapping[source_step.id] = getStepUuid(candidates[0]); - idMap.set(source_step.id, candidates[0].id); - continue; - } + // Map by children + ({ mapping, idMap, unmatchedSource: sourceSteps, unmatchedTarget: targetSteps } = mapByChildren(sourceSteps, targetSteps, mapping, idMap, source.steps)); - // Children - result = mapStepByChildren(source_step, source, candidates); - if (result.length) candidates = result; - if (candidates.length === 1) { - nodeMapping[source_step.id] = getStepUuid(candidates[0]); - idMap.set(source_step.id, candidates[0].id); - continue; - } + // Map by expression + ({ mapping, idMap, unmatchedSource: sourceSteps, unmatchedTarget: targetSteps } = mapByExpression(sourceSteps, targetSteps, mapping, idMap)); - // Expression - result = mapStepByExpression(source_step, candidates); - if (result.length) candidates = result; - if (candidates.length === 1) { - nodeMapping[source_step.id] = getStepUuid(candidates[0]); - idMap.set(source_step.id, candidates[0].id); - continue; - } - - // If none matched, mark as new - nodeMapping[source_step.id] = true; + for (const s of sourceSteps) { + mapping[s.id] = true; } // EDGE MAPPING @@ -108,7 +83,7 @@ export default (source: Workflow, target: Workflow): MappingResults => { } return { - nodes: nodeMapping, + nodes: mapping, edges: edgeMapping, }; }; @@ -161,37 +136,77 @@ interface MapStepResult { pool: Pool; } -function mapStepsById( - source: Workflow['steps'], - target: Workflow['steps'] -): MapStepResult { - const targets: Record = {}; - const mapping: Record = {}; - const idMap = new Map(); +function mapById(sourceSteps, targetSteps, mapping, idMap) { + const unmatchedSource = []; + const unmatchedTarget = [...targetSteps]; + for (const s of sourceSteps) { + const t = targetSteps.find(t => t.id === s.id); + if (t) { + mapping[s.id] = t.openfn?.uuid; + idMap.set(s.id, t.id); + unmatchedTarget.splice(unmatchedTarget.indexOf(t), 1); + } else { + unmatchedSource.push(s); + } + } + return { mapping, idMap, unmatchedSource, unmatchedTarget }; +} - for (const target_step of target) { - targets[target_step.id] = target_step; +function mapByParent(sourceSteps, targetSteps, mapping, idMap, sourceAll) { + const unmatchedSource = []; + const unmatchedTarget = [...targetSteps]; + for (const s of sourceSteps) { + const parent = getParent(s.id, sourceAll); + if (parent) { + const candidates = findByParent(parent, targetSteps); + if (candidates.length === 1) { + const t = candidates[0]; + mapping[s.id] = getStepUuid(t); + idMap.set(s.id, t.id); + unmatchedTarget.splice(unmatchedTarget.indexOf(t), 1); + continue; + } + } + unmatchedSource.push(s); } + return { mapping, idMap, unmatchedSource, unmatchedTarget }; +} - const removedIndexes = []; - for (let i = 0; i < source.length; i++) { - const source_step = source[i]; - if (target[source_step.id]) { - mapping[source_step.id] = target[source_step.id]?.openfn?.uuid; - idMap.set(source_step.id, target[source_step.id]?.id); - removedIndexes.push(i); - target = target.filter((t) => t !== target[source_step]); +function mapByChildren(sourceSteps, targetSteps, mapping, idMap, sourceAll) { + const unmatchedSource = []; + const unmatchedTarget = [...targetSteps]; + for (const s of sourceSteps) { + const children = getEdges(sourceAll)[s.id]; + if (children) { + const candidates = findByChildren(children, targetSteps); + if (candidates.length === 1) { + const t = candidates[0]; + mapping[s.id] = getStepUuid(t); + idMap.set(s.id, t.id); + unmatchedTarget.splice(unmatchedTarget.indexOf(t), 1); + continue; + } } + unmatchedSource.push(s); } + return { mapping, idMap, unmatchedSource, unmatchedTarget }; +} - return { - mapping, - idMap, - pool: { - source: source.filter((_, i) => !removedIndexes.includes(i)), - target, - }, - }; +function mapByExpression(sourceSteps, targetSteps, mapping, idMap) { + const unmatchedSource = []; + const unmatchedTarget = [...targetSteps]; + for (const s of sourceSteps) { + const candidates = findByExpression((s as Job).expression, targetSteps); + if (candidates.length === 1) { + const t = candidates[0]; + mapping[s.id] = getStepUuid(t); + idMap.set(s.id, t.id); + unmatchedTarget.splice(unmatchedTarget.indexOf(t), 1); + continue; + } + unmatchedSource.push(s); + } + return { mapping, idMap, unmatchedSource, unmatchedTarget }; } // very flawed, due to the commented code snipped every step comes with. @@ -234,23 +249,5 @@ function findByChildren(childIds: string[], steps: Workflow['steps']) { return parents.map((p) => stepsIndex[p]); } -function mapStepByParent(source_step: Workflow['steps'][number], source: Workflow['steps'], candidates: Workflow['steps']) { - const parent = getParent(source_step.id, source.steps); - if (parent) { - return findByParent(parent, candidates); - } - return []; -} - -function mapStepByChildren(source_step: Workflow['steps'][number], source: Workflow['steps'], candidates: Workflow['steps']) { - const children = getEdges(source.steps)[source_step.id]; - if(!children) return []; - return findByChildren(children, candidates); -} - -function mapStepByExpression(source_step: Workflow['steps'][number], candidates: Workflow['steps']) { - return findByExpression((source_step as Job).expression, candidates); -} - // Get an array of all UUIDS const extractUUIDS = (p: Project) => {};