From 6552f2bfd6cd8f0c0f318f4c5bd50cdfb52725ac Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 4 Mar 2026 09:17:20 -0600 Subject: [PATCH 01/65] feat(web): implement dedicated substitution-spur class, tests Per the search quotient graph documentation, we should no longer have one node type handle all edit operation types. Our current ("legacy") implementation and unit tests are most tailored for the needs of 'substitution' edit types, so this PR starts there. Build-bot: skip build:web Test-bot: skip --- .../main/correction/search-quotient-spur.ts | 2 +- .../correction/substitution-quotient-spur.ts | 48 +++ .../worker-thread/src/main/test-index.ts | 1 + .../buildAlphabeticClusteredFixture.ts | 40 +-- .../helpers/buildCantLinearFixture.ts | 12 +- .../helpers/buildQuotientDocFixture.tests.ts | 32 ++ .../helpers/buildQuotientDocFixture.ts | 246 ++++++++++++++ .../legacy-quotient-spur.tests.ts | 35 +- .../substitution-quotient-spur.tests.ts | 305 ++++++++++++++++++ 9 files changed, 693 insertions(+), 28 deletions(-) create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts create mode 100644 web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts create mode 100644 web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index 20a6eb501b9..9a034904a5b 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -43,7 +43,7 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { readonly inputs?: Distribution; readonly inputSource?: PathInputProperties; - private parentNode: SearchQuotientNode; + protected readonly parentNode: SearchQuotientNode; readonly spaceId: number; readonly inputCount: number; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts new file mode 100644 index 00000000000..baa6d9f5e8f --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts @@ -0,0 +1,48 @@ +/** + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-02-03 + * + * This file adds a SearchQuotientSpur variant modeling match & substitute edit + * operations in regard to the corresponding keystroke. + */ + +import { LexicalModelTypes } from "@keymanapp/common-types"; +import { KMWString } from "@keymanapp/web-utils"; + +import { SearchNode } from "./distance-modeler.js"; +import { PathInputProperties, SearchQuotientNode } from "./search-quotient-node.js"; +import { SearchQuotientSpur } from "./search-quotient-spur.js"; +import { TokenResultMapping } from "./token-result-mapping.js"; + +import Distribution = LexicalModelTypes.Distribution; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; + +export class SubstitutionQuotientSpur extends SearchQuotientSpur { + public insertLength: number; + public leftDeleteLength: number; + + constructor( + parentNode: SearchQuotientNode, + inputs: Distribution>, + inputSource: PathInputProperties | ProbabilityMass + ) { + // Compute this SearchPath's codepoint length & edge length. + const inputSample = inputs?.[0].sample ?? { insert: '', deleteLeft: 0 }; + const insertLength = KMWString.length(inputSample.insert); + super(parentNode, inputs, inputSource, parentNode.codepointLength + insertLength - inputSample.deleteLeft); + + // Compute this SearchPath's codepoint length & edge length. + this.insertLength = insertLength; + this.leftDeleteLength = inputSample.deleteLeft; + } + + construct(parentNode: SearchQuotientNode, inputs: ProbabilityMass>[], inputSource: PathInputProperties): this { + return new SubstitutionQuotientSpur(parentNode, inputs, inputSource) as this; + } + + protected buildEdgesFromResults(baseResults: ReadonlyArray): SearchNode[] { + return baseResults.flatMap((n) => n.buildSubstitutionEdges(this.inputs, this.spaceId)); + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index da9cef8adb4..2f28b93bda0 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -7,6 +7,7 @@ export { ContextTransition } from './correction/context-transition.js'; export * from './correction/correction-searchable.js'; export * from './correction/correction-result-mapping.js'; export * from './correction/distance-modeler.js'; +export * from './correction/substitution-quotient-spur.js'; export * from './correction/search-quotient-cluster.js'; export * from './correction/search-quotient-spur.js'; export * from './correction/search-quotient-node.js'; diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts index 086193a387a..d45cffc7e4c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts @@ -13,9 +13,9 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs' import { models, - LegacyQuotientSpur, SearchQuotientCluster, - LegacyQuotientRoot + SearchQuotientRoot, + SubstitutionQuotientSpur } from '@keymanapp/lm-worker/test-index'; import Distribution = LexicalModelTypes.Distribution; @@ -31,7 +31,7 @@ const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); * @returns */ export const buildAlphabeticClusterFixtures = () => { - const rootPath = new LegacyQuotientRoot(testModel); + const rootPath = new SearchQuotientRoot(testModel); // consonant-cluster 1, insert 1, delete 0 const distrib_c1_i1d0: Distribution = [ @@ -48,9 +48,9 @@ export const buildAlphabeticClusterFixtures = () => { ]; // keystrokes 1, codepoints 1, total inserts 1, delete 0 - const path_k1c1_i1d0 = new LegacyQuotientSpur(rootPath, distrib_c1_i1d0, distrib_c1_i1d0[0]); + const path_k1c1_i1d0 = new SubstitutionQuotientSpur(rootPath, distrib_c1_i1d0, distrib_c1_i1d0[0]); // keystrokes 1, codepoints 2, total inserts 2, delete 0 - const path_k1c2_i2d0 = new LegacyQuotientSpur(rootPath, distrib_c1_i2d0, distrib_c1_i1d0[0]); + const path_k1c2_i2d0 = new SubstitutionQuotientSpur(rootPath, distrib_c1_i2d0, distrib_c1_i1d0[0]); // Second input @@ -62,8 +62,8 @@ export const buildAlphabeticClusterFixtures = () => { { sample: { insert: 'u', deleteLeft: 0, deleteRight: 0, id: 12 }, p: 0.1 }, ]; - const path_k2c2_i2d0 = new LegacyQuotientSpur(path_k1c1_i1d0, distrib_v1_i1d0, distrib_v1_i1d0[0]); - const path_k2c3_i3d0 = new LegacyQuotientSpur(path_k1c2_i2d0, distrib_v1_i1d0, distrib_v1_i1d0[0]); + const path_k2c2_i2d0 = new SubstitutionQuotientSpur(path_k1c1_i1d0, distrib_v1_i1d0, distrib_v1_i1d0[0]); + const path_k2c3_i3d0 = new SubstitutionQuotientSpur(path_k1c2_i2d0, distrib_v1_i1d0, distrib_v1_i1d0[0]); // Third input const distrib_v2_i1d0: Distribution = [ @@ -90,15 +90,15 @@ export const buildAlphabeticClusterFixtures = () => { { sample: { insert: 'รบรบ', deleteLeft: 1, deleteRight: 0, id: 13 }, p: 0.02 }, ]; // 0.2 total - const path_k3c2_i3d1 = new LegacyQuotientSpur(path_k2c2_i2d0, distrib_v2_i1d1, distrib_v2_i1d0[0]); + const path_k3c2_i3d1 = new SubstitutionQuotientSpur(path_k2c2_i2d0, distrib_v2_i1d1, distrib_v2_i1d0[0]); - const path_k3c3_i3d0 = new LegacyQuotientSpur(path_k2c2_i2d0, distrib_v2_i1d0, distrib_v2_i1d0[0]); - const path_k3c3_i4d1a = new LegacyQuotientSpur(path_k2c2_i2d0, distrib_v2_i2d1, distrib_v2_i1d0[0]); - const path_k3c3_i4d1b = new LegacyQuotientSpur(path_k2c3_i3d0, distrib_v2_i1d1, distrib_v2_i1d0[0]); + const path_k3c3_i3d0 = new SubstitutionQuotientSpur(path_k2c2_i2d0, distrib_v2_i1d0, distrib_v2_i1d0[0]); + const path_k3c3_i4d1a = new SubstitutionQuotientSpur(path_k2c2_i2d0, distrib_v2_i2d1, distrib_v2_i1d0[0]); + const path_k3c3_i4d1b = new SubstitutionQuotientSpur(path_k2c3_i3d0, distrib_v2_i1d1, distrib_v2_i1d0[0]); // both are built on path k1c2 (splits at index 1) - const path_k3c4_i4d0 = new LegacyQuotientSpur(path_k2c3_i3d0, distrib_v2_i1d0, distrib_v2_i1d0[0]); - const path_k3c4_i5d1 = new LegacyQuotientSpur(path_k2c3_i3d0, distrib_v2_i2d1, distrib_v2_i1d0[0]); + const path_k3c4_i4d0 = new SubstitutionQuotientSpur(path_k2c3_i3d0, distrib_v2_i1d0, distrib_v2_i1d0[0]); + const path_k3c4_i5d1 = new SubstitutionQuotientSpur(path_k2c3_i3d0, distrib_v2_i2d1, distrib_v2_i1d0[0]); const cluster_k3c3 = new SearchQuotientCluster([path_k3c3_i3d0, path_k3c3_i4d1a, path_k3c3_i4d1b]); // both are built on path k1c2. @@ -116,13 +116,13 @@ export const buildAlphabeticClusterFixtures = () => { { sample: { insert: 'vw', deleteLeft: 0, deleteRight: 0, id: 14 }, p: 0.1 } ]; - const path_k4c4_i2 = new LegacyQuotientSpur(path_k3c2_i3d1, distrib_c2_i2d0, distrib_c2_i2d0[0]); - const path_k4c4_i1 = new LegacyQuotientSpur(cluster_k3c3, distrib_c2_i1d0, distrib_c2_i2d0[0]); + const path_k4c4_i2 = new SubstitutionQuotientSpur(path_k3c2_i3d1, distrib_c2_i2d0, distrib_c2_i2d0[0]); + const path_k4c4_i1 = new SubstitutionQuotientSpur(cluster_k3c3, distrib_c2_i1d0, distrib_c2_i2d0[0]); - const path_k4c5_i2 = new LegacyQuotientSpur(cluster_k3c3, distrib_c2_i2d0, distrib_c2_i2d0[0]); - const path_k4c5_i1 = new LegacyQuotientSpur(cluster_k3c4, distrib_c2_i1d0, distrib_c2_i2d0[0]); + const path_k4c5_i2 = new SubstitutionQuotientSpur(cluster_k3c3, distrib_c2_i2d0, distrib_c2_i2d0[0]); + const path_k4c5_i1 = new SubstitutionQuotientSpur(cluster_k3c4, distrib_c2_i1d0, distrib_c2_i2d0[0]); - const path_k4c6 = new LegacyQuotientSpur(cluster_k3c4, distrib_c2_i2d0, distrib_c2_i2d0[0]); + const path_k4c6 = new SubstitutionQuotientSpur(cluster_k3c4, distrib_c2_i2d0, distrib_c2_i2d0[0]); const cluster_k4c4 = new SearchQuotientCluster([path_k4c4_i2, path_k4c4_i1]); const cluster_k4c5 = new SearchQuotientCluster([path_k4c5_i2, path_k4c5_i1]); @@ -135,8 +135,8 @@ export const buildAlphabeticClusterFixtures = () => { { sample: { insert: 'z', deleteLeft: 0, deleteRight: 0, id: 15 }, p: 0.4 } ]; - const path_k5c6_a = new LegacyQuotientSpur(cluster_k4c4, distrib_c3_i2d0, distrib_c3_i2d0[0]); - const path_k5c6_b = new LegacyQuotientSpur(cluster_k4c5, distrib_c3_i1d0, distrib_c3_i2d0[0]); + const path_k5c6_a = new SubstitutionQuotientSpur(cluster_k4c4, distrib_c3_i2d0, distrib_c3_i2d0[0]); + const path_k5c6_b = new SubstitutionQuotientSpur(cluster_k4c5, distrib_c3_i1d0, distrib_c3_i2d0[0]); const cluster_k5c6 = new SearchQuotientCluster([path_k5c6_a, path_k5c6_b]); diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildCantLinearFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildCantLinearFixture.ts index 3de7986562c..640c6c206e6 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildCantLinearFixture.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildCantLinearFixture.ts @@ -8,7 +8,7 @@ * divergence occurring within the fixture. */ -import { LegacyQuotientRoot, LegacyQuotientSpur, models } from "@keymanapp/lm-worker/test-index"; +import { SearchQuotientRoot, SubstitutionQuotientSpur, models } from "@keymanapp/lm-worker/test-index"; import { jsonFixture } from "@keymanapp/common-test-resources/model-helpers.mjs"; import TrieModel = models.TrieModel; @@ -19,31 +19,31 @@ const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); * Build a linear fixture that models the word 'cant' and words close to that. */ export function buildCantLinearFixture() { - const rootPath = new LegacyQuotientRoot(testModel); + const rootPath = new SearchQuotientRoot(testModel); const distrib1 = [ { sample: {insert: 'c', deleteLeft: 0, id: 11}, p: 0.5 }, { sample: {insert: 'r', deleteLeft: 0, id: 11}, p: 0.4 }, { sample: {insert: 't', deleteLeft: 0, id: 11}, p: 0.1 } ]; - const path1 = new LegacyQuotientSpur(rootPath, distrib1, distrib1[0]); + const path1 = new SubstitutionQuotientSpur(rootPath, distrib1, distrib1[0]); const distrib2 = [ { sample: {insert: 'a', deleteLeft: 0, id: 12}, p: 0.7 }, { sample: {insert: 'e', deleteLeft: 0, id: 12}, p: 0.3 } ]; - const path2 = new LegacyQuotientSpur(path1, distrib2, distrib2[0]); + const path2 = new SubstitutionQuotientSpur(path1, distrib2, distrib2[0]); const distrib3 = [ { sample: {insert: 'n', deleteLeft: 0, id: 13}, p: 0.8 }, { sample: {insert: 'r', deleteLeft: 0, id: 13}, p: 0.2 } ]; - const path3 = new LegacyQuotientSpur(path2, distrib3, distrib3[0]); + const path3 = new SubstitutionQuotientSpur(path2, distrib3, distrib3[0]); const distrib4 = [ { sample: {insert: 't', deleteLeft: 0, id: 14}, p: 1 } ]; - const path4 = new LegacyQuotientSpur(path3, distrib4, distrib4[0]); + const path4 = new SubstitutionQuotientSpur(path3, distrib4, distrib4[0]); return { paths: [null, path1, path2, path3, path4], diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts new file mode 100644 index 00000000000..6bf9f0dea7b --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts @@ -0,0 +1,32 @@ +import { assert } from "chai"; + +import { buildQuotientDocFixture } from "./buildQuotientDocFixture.js"; + +describe('buildQuotientDocFixture() fixture', () => { + it('constructs paths properly', () => { + const {searchRoot, nodes} = buildQuotientDocFixture(); + + [searchRoot /*, nodes.sc1, nodes.sc2*/].forEach((n) => { + assert.equal(n.inputCount, 0); + }); + [/*nodes.k1c0,*/ nodes.k1c1, nodes.k1c2 /*, nodes.k1c3*/].forEach((n) => { + assert.equal(n.inputCount, 1); + }); + [/*nodes.k2c0, nodes.k2c1,*/ nodes.k2c2, nodes.k2c3].forEach((n) => { + assert.equal(n.inputCount, 2); + }); + + [searchRoot/*, nodes.k1c0, nodes.k2c0*/].forEach((n) => { + assert.equal(n.codepointLength, 0); + }); + [/*nodes.sc1, */nodes.k1c1/*, nodes.k2c1*/].forEach((n) => { + assert.equal(n.codepointLength, 1); + }); + [/*nodes.sc2,*/ nodes.k1c2, nodes.k2c2].forEach((n) => { + assert.equal(n.codepointLength, 2); + }); + [/*nodes.k1c3,*/ nodes.k2c3].forEach((n) => { + assert.equal(n.codepointLength, 3); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts new file mode 100644 index 00000000000..66ee12fe631 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts @@ -0,0 +1,246 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-09 + * + * This file defines a unit-text fixture designed for testing + * the internal mechanisms of a search quotient graph built from + * quotient-spurs specialized for each of the three main edit-distance + * operation types. + */ + +import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { + generateSubsetId, + models, + SearchQuotientCluster, + SearchQuotientRoot, + SubstitutionQuotientSpur +} from '@keymanapp/lm-worker/test-index'; + +import Distribution = LexicalModelTypes.Distribution; +import Transform = LexicalModelTypes.Transform; +import TrieModel = models.TrieModel; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + + +/** + * Builds a text fixture matching the [final quotient-graph example]( + * ../../../../../../../engine/predictive-text/worker-thread/docs/correction-search-graph.md) + * documenting the internal SearchQuotientNode design. + * @returns + */ +export function buildQuotientDocFixture() { + const searchRoot = new SearchQuotientRoot(testModel); + let idSeed = 0; + + const key1Id = idSeed++; + const abDistrib: Distribution = [ + { sample: { insert: 'a', deleteLeft: 0, id: key1Id }, p: .45 }, + { sample: { insert: 'b', deleteLeft: 0, id: key1Id }, p: .35 } + ]; + + const cdDistrib: Distribution = [ + { sample: { insert: 'cd', deleteLeft: 0, id: key1Id }, p: .2 } + ]; + + // const sc1 = new InsertionQuotientSpur(searchRoot); + // const sc2 = new InsertionQuotientSpur(sc1); + + // // K1C0 + // const k1c0 = new DeletionQuotientSpur(searchRoot, abDistrib.concat(cdDistrib), { + // segment: { + // transitionId: key1Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: abDistrib[0].p + // }); + + // // K1C1 + // const k1c1_del = new DeletionQuotientSpur(sc1, abDistrib.concat(cdDistrib), { + // segment: { + // transitionId: key1Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: abDistrib[0].p + // }); + const k1c1_ab = new SubstitutionQuotientSpur(searchRoot, abDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + // const k1c1_ins = new InsertionQuotientSpur(k1c0); + const k1c1 = new SearchQuotientCluster([/*k1c1_del,*/ k1c1_ab, /*k1c1_ins*/]); + + // const k1c2_del = new DeletionQuotientSpur(sc2, abDistrib.concat(cdDistrib), { + // segment: { + // transitionId: key1Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: abDistrib[0].p + // }); + // const k1c2_ab = new SubstitutionQuotientSpur(sc1, abDistrib, { + // segment: { + // transitionId: key1Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: abDistrib[0].p + // }); + const k1c2_cd = new SubstitutionQuotientSpur(searchRoot, cdDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + // const k1c2_ins = new InsertionQuotientSpur(k1c1); + const k1c2 = new SearchQuotientCluster([/*k1c2_del, k1c2_ab, */ k1c2_cd, /*k1c2_ins*/]); + + // const k1c3_ab = new SubstitutionQuotientSpur(sc2, abDistrib, { + // segment: { + // transitionId: key1Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: abDistrib[0].p + // }); + // const k1c3_cd = new SubstitutionQuotientSpur(sc1, cdDistrib, { + // segment: { + // transitionId: key1Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: abDistrib[0].p + // }); + // const k1c3_ins = new InsertionQuotientSpur(k1c2); + // const k1c3 = new SearchQuotientCluster([k1c3_ab, k1c3_cd, k1c3_ins]); + + // Onto keystroke 2. + + const key2Id = idSeed++; + const efDistrib: Distribution = [ + { sample: { insert: 'e', deleteLeft: 0, id: key2Id }, p: .4 }, + { sample: { insert: 'f', deleteLeft: 0, id: key2Id }, p: .3 } + ]; + + const ghDistrib: Distribution = [ + { sample: { insert: 'gh', deleteLeft: 0, id: key2Id }, p: .3 } + ]; + + // const k2c0 = new DeletionQuotientSpur(k1c0, efDistrib.concat(ghDistrib), { + // segment: { + // transitionId: key2Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: efDistrib[0].p + // }); + + // const k2c1_del = new DeletionQuotientSpur(k1c1, efDistrib.concat(ghDistrib), { + // segment: { + // transitionId: key2Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: efDistrib[0].p + // }); + // const k2c1_ef = new SubstitutionQuotientSpur(k1c0, efDistrib, { + // segment: { + // transitionId: key2Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: efDistrib[0].p + // }); + // const k2c1_ins = new InsertionQuotientSpur(k2c0); + // const k2c1 = new SearchQuotientCluster([k2c1_del, k2c1_ef, k2c1_ins]); + + // const k2c2_del = new DeletionQuotientSpur(k1c2, efDistrib.concat(ghDistrib), { + // segment: { + // transitionId: key2Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: efDistrib[0].p + // }); + const k2c2_ef = new SubstitutionQuotientSpur(k1c1, efDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + // const k2c2_gh = new SubstitutionQuotientSpur(k1c0, ghDistrib, { + // segment: { + // transitionId: key2Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: efDistrib[0].p + // }); + // const k2c2_ins = new InsertionQuotientSpur(k2c1); + const k2c2 = new SearchQuotientCluster([/*k2c2_del,*/ k2c2_ef /*, k2c2_gh, k2c2_ins*/]); + + // const k2c3_del = new DeletionQuotientSpur(k1c3, efDistrib.concat(ghDistrib), { + // segment: { + // transitionId: key2Id, + // start: 0 + // }, + // // Deletions always get their own unique subset ID. + // subsetId: generateSubsetId(), + // bestProbFromSet: efDistrib[0].p + // }); + const k2c3_ef = new SubstitutionQuotientSpur(k1c2, efDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c3_gh = new SubstitutionQuotientSpur(k1c1, ghDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + // const k2c3_ins = new InsertionQuotientSpur(k2c2); + const k2c3 = new SearchQuotientCluster([/*k2c3_del, */ k2c3_ef, k2c3_gh, /*k2c3_ins*/]); + + return { + searchRoot, + spurs: {/*sc1, sc2,*/ k1c1_ab, /*k1c2_ab,*/ k1c2_cd, /*k1c2_ins, k1c3_ab, k1c3_cd, k1c3_ins,*/ k2c2_ef, k2c3_ef, k2c3_gh /*, k2c3_ins*/}, + nodes: {/* sc1, sc2, k1c0, */ k1c1, k1c2, /* k1c3, k2c0, k2c1, */ k2c2, k2c3} + }; +} diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts index 0989ab9d64e..21bdd5026e6 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts @@ -17,13 +17,46 @@ import { models } from '@keymanapp/lm-worker/test-index'; -import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; import TrieModel = models.TrieModel; const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); +// Similar to the fixture version, but with the LegacyQuotientRoot & LegacyQuotientSpur classes. +export function buildCantLinearFixture() { + const rootPath = new LegacyQuotientRoot(testModel); + + const distrib1 = [ + { sample: {insert: 'c', deleteLeft: 0, id: 11}, p: 0.5 }, + { sample: {insert: 'r', deleteLeft: 0, id: 11}, p: 0.4 }, + { sample: {insert: 't', deleteLeft: 0, id: 11}, p: 0.1 } + ]; + const path1 = new LegacyQuotientSpur(rootPath, distrib1, distrib1[0]); + + const distrib2 = [ + { sample: {insert: 'a', deleteLeft: 0, id: 12}, p: 0.7 }, + { sample: {insert: 'e', deleteLeft: 0, id: 12}, p: 0.3 } + ]; + const path2 = new LegacyQuotientSpur(path1, distrib2, distrib2[0]); + + const distrib3 = [ + { sample: {insert: 'n', deleteLeft: 0, id: 13}, p: 0.8 }, + { sample: {insert: 'r', deleteLeft: 0, id: 13}, p: 0.2 } + ]; + const path3 = new LegacyQuotientSpur(path2, distrib3, distrib3[0]); + + const distrib4 = [ + { sample: {insert: 't', deleteLeft: 0, id: 14}, p: 1 } + ]; + const path4 = new LegacyQuotientSpur(path3, distrib4, distrib4[0]); + + return { + paths: [null, path1, path2, path3, path4], + distributions: [distrib1, distrib2, distrib3, distrib4] + }; +} + describe('LegacyQuotientSpur', () => { describe('constructor', () => { it('initializes from a lexical model', () => { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts new file mode 100644 index 00000000000..a8401aa3b75 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts @@ -0,0 +1,305 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-04 + * + * This file defines tests for the SubstitutionQuotientSpur class of the + * predictive-text correction-search engine's search graph. + */ + +import { assert } from 'chai'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { + generateSubsetId, + models, + SearchQuotientRoot, + SubstitutionQuotientSpur +} from '@keymanapp/lm-worker/test-index'; + +import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; +import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; + +import TrieModel = models.TrieModel; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + +describe('SubstitutionQuotientSpur', () => { + describe('constructor', () => { + it('may be extended from root path', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + + const extendedPath = new SubstitutionQuotientSpur(rootPath, leadEdgeDistribution, leadEdgeDistribution[0]); + + assert.equal(extendedPath.inputCount, 1); + assert.equal(extendedPath.codepointLength, 1); + assert.isNumber(extendedPath.spaceId); + assert.notEqual(extendedPath.spaceId, rootPath.spaceId); + assert.deepEqual(extendedPath.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(extendedPath.parents, [rootPath]); + assert.deepEqual(extendedPath.inputs, leadEdgeDistribution); + assert.deepEqual(extendedPath.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + // Assert the root is unchanged. + assert.equal(rootPath.inputCount, 0); + // Should (still) have codepointLength == 0 once it's defined. + assert.deepEqual(rootPath.bestExample, {text: '', p: 1}); + assert.deepEqual(rootPath.parents, []); + }); + + // TODO: for each of the possible ancestor Spur types, + the root! + // Properties should have proper relationship with each test's ancestor. + it('may be built from arbitrary prior SearchQuotientSpur', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + const inputClone = leadEdgeDistribution.map(e => ({...e})); + + const length1Path = new SubstitutionQuotientSpur( + rootPath, + leadEdgeDistribution, + leadEdgeDistribution[0] + ); + + const tailEdgeDistribution = [ + {sample: {insert: 'r', deleteLeft: 0, id: 17 }, p: 0.6}, + {sample: {insert: 'e', deleteLeft: 0, id: 17 }, p: 0.25}, + {sample: {insert: 'h', deleteLeft: 0, id: 17 }, p: 0.15} + ]; + + const length2Path = new SubstitutionQuotientSpur( + length1Path, + tailEdgeDistribution, + tailEdgeDistribution[0] + ); + + // Verify that the prior distribution remains fully unaltered. + assert.deepEqual(leadEdgeDistribution, inputClone); + + assert.equal(length2Path.inputCount, 2); + assert.equal(length2Path.codepointLength, 2); + assert.isNumber(length2Path.spaceId); + assert.notEqual(length2Path.spaceId, length1Path.spaceId); + assert.deepEqual(length2Path.bestExample, {text: 'tr', p: leadEdgeDistribution[0].p * tailEdgeDistribution[0].p}); + assert.deepEqual(length2Path.parents, [length1Path]); + assert.deepEqual(length2Path.inputs, tailEdgeDistribution); + assert.deepEqual(length2Path.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + }, { + transitionId: tailEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + assert.equal(length1Path.inputCount, 1); + assert.equal(length1Path.codepointLength, 1); + assert.isNumber(length1Path.spaceId); + assert.notEqual(length1Path.spaceId, rootPath.spaceId); + assert.deepEqual(length1Path.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(length1Path.parents, [rootPath]); + assert.deepEqual(length1Path.inputs, leadEdgeDistribution); + }); + + it('may extend with a Transform inserting multiple codepoints', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + const inputClone = leadEdgeDistribution.map(e => ({...e})); + + const length1Path = new SubstitutionQuotientSpur( + rootPath, + leadEdgeDistribution, + leadEdgeDistribution[0] + ); + + const tailEdgeDistribution = [ + {sample: {insert: 'ri', deleteLeft: 0, id: 17 }, p: 0.6}, + {sample: {insert: 'er', deleteLeft: 0, id: 17 }, p: 0.25}, + {sample: {insert: 'hi', deleteLeft: 0, id: 17 }, p: 0.15} + ]; + + const length2Path = new SubstitutionQuotientSpur( + length1Path, + tailEdgeDistribution, + tailEdgeDistribution[0] + ); + + // Verify that the prior distribution remains fully unaltered. + assert.deepEqual(leadEdgeDistribution, inputClone); + + assert.equal(length2Path.inputCount, 2); + assert.equal(length2Path.codepointLength, 3); + assert.isNumber(length2Path.spaceId); + assert.notEqual(length2Path.spaceId, length1Path.spaceId); + assert.deepEqual(length2Path.bestExample, {text: 'tri', p: leadEdgeDistribution[0].p * tailEdgeDistribution[0].p}); + assert.deepEqual(length2Path.parents, [length1Path]); + assert.deepEqual(length2Path.inputs, tailEdgeDistribution); + assert.deepEqual(length2Path.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + }, { + transitionId: tailEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + assert.equal(length1Path.inputCount, 1); + assert.equal(length1Path.codepointLength, 1); + assert.isNumber(length1Path.spaceId); + assert.notEqual(length1Path.spaceId, rootPath.spaceId); + assert.deepEqual(length1Path.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(length1Path.parents, [rootPath]); + assert.deepEqual(length1Path.inputs, leadEdgeDistribution); + }); + }); + + describe('.edgeKey', () => { + it('changes when input source subset IDs differ', () => { + const root = new SearchQuotientRoot(testModel); + + const {distributions} = buildCantLinearFixture(); + const inputSrc = { + segment: { + transitionId: distributions[0][0].sample.id, + start: 0 + }, + subsetId: generateSubsetId(), + bestProbFromSet: distributions[0][0].p + }; + + const spur1 = new SubstitutionQuotientSpur(root, distributions[0], { + ...inputSrc, + subsetId: generateSubsetId() + }); + const spur2 = new SubstitutionQuotientSpur(root, distributions[0], { + ...inputSrc, + subsetId: generateSubsetId() + }); + + assert.notEqual(spur1.edgeKey, spur2.edgeKey); + }); + + it('changes when different parts of the same input source are used', () => { + const root = new SearchQuotientRoot(testModel); + + const {distributions} = buildCantLinearFixture(); + const inputSrc = { + segment: { + transitionId: distributions[0][0].sample.id, + start: 0 + }, + subsetId: generateSubsetId(), + bestProbFromSet: distributions[0][0].p + }; + + const spur1 = new SubstitutionQuotientSpur(root, distributions[0], inputSrc); + const spur2 = new SubstitutionQuotientSpur(root, distributions[0], { + ...inputSrc, + segment: { + ...inputSrc.segment, + end: 1 + } + }); + const spur3 = new SubstitutionQuotientSpur(root, distributions[0], { + ...inputSrc, + segment: { + ...inputSrc.segment, + start: inputSrc.segment.start + 1 + } + }); + + assert.notEqual(spur1.edgeKey, spur2.edgeKey); + assert.notEqual(spur2.edgeKey, spur3.edgeKey); + assert.notEqual(spur3.edgeKey, spur1.edgeKey); + }); + }); + + describe('handleNextNode()', () => { + it('outputs results that directly match inputs', () => { + const canPath = buildCantLinearFixture().paths[3]; + + const matchTargets = [ + 'can', + 'car', + 'cen', // 'cent' and 'center' are supported in this test model. + ]; + const analysis = analyzeQuotientNodeResults(canPath, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('outputs results that substitute inputs', () => { + const canPath = buildCantLinearFixture().paths[3]; + + const matchTargets = [ + // Replacement of first char + 'man', + 'far', + // Replacement of second char + 'con', // 'consider' and variants thereof are also supported. + 'cor', // 'corner' + // Replacement of third char + 'cal', // 'call' + ]; + const analysis = analyzeQuotientNodeResults(canPath, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that insert characters as needed', () => { + const canPath = buildCantLinearFixture().paths[3]; + + const matchTargets = [ + 'char', // 'c' + (insert) 'h' + 'ar' + 'than', // 't' + (insert) 'h' + 'an' + ]; + const analysis = analyzeQuotientNodeResults(canPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that delete incoming keystrokes as needed', () => { + const canPath = buildCantLinearFixture().paths[3]; + + const matchTargets = [ + // Delete only first + 'an', // (delete) 'c'/'r'/'t' + 'an', for 'and' and 'any' + 'en', // (delete) 'c'/'r'/'t' + 'en', for 'end', + // Even delete second + 'n', // model possesses words starting with just 'n' + 't' // ... and 't'. + ]; + const analysis = analyzeQuotientNodeResults(canPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + }); +}); \ No newline at end of file From 856466ec09862e430d01e7cb43a1e2ee06629841 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 20 Apr 2026 14:01:45 -0500 Subject: [PATCH 02/65] change(web): update TokenizationCorrector, QuotientNodeFinalizer unit tests to use new spur type --- .../main/correction/search-quotient-spur.ts | 7 ++++-- .../correction/substitution-quotient-spur.ts | 4 +-- .../quotient-node-finalizer.tests.ts | 25 +++++++++++++------ .../tokenization-corrector.tests.ts | 24 +++++++++--------- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index 9a034904a5b..82fa2acf503 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -24,6 +24,7 @@ import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; +import { LegacyQuotientSpur } from './legacy-quotient-spur.js'; export const QUEUE_NODE_COMPARATOR: QueueComparator = function(arg1, arg2) { return arg1.currentCost - arg2.currentCost; @@ -265,7 +266,8 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { // // stopgap: maybe go ahead and check each input for any that are longer? // won't matter shortly, though. - return [[this, new LegacyQuotientRoot(this.model)]]; + const rootConstructor = this instanceof LegacyQuotientSpur ? LegacyQuotientRoot : SearchQuotientRoot; + return [[this, new rootConstructor(this.model)]]; } else { const firstSet: Distribution = this.inputs.map((input) => ({ // keep insert head @@ -300,9 +302,10 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { }) : this.parentNode; // construct two SearchPath instances based on the two sets! + const rootConstructor = this instanceof LegacyQuotientSpur ? LegacyQuotientRoot : SearchQuotientRoot; return [[ parent, - this.construct(new LegacyQuotientRoot(this.model), secondSet, { + this.construct(new rootConstructor(this.model), secondSet, { ...this.inputSource, segment: { ...this.inputSource.segment, diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts index baa6d9f5e8f..1b34afcb507 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts @@ -20,8 +20,8 @@ import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; export class SubstitutionQuotientSpur extends SearchQuotientSpur { - public insertLength: number; - public leftDeleteLength: number; + public readonly insertLength: number; + public readonly leftDeleteLength: number; constructor( parentNode: SearchQuotientNode, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts index 4feb701f389..e614867c9cb 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts @@ -15,14 +15,15 @@ import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { + EDIT_DISTANCE_COST_SCALE, generateSubsetId, - LegacyQuotientSpur, models, PathInputProperties, PathResult, QuotientNodeFinalizer, SearchQuotientNode, SearchQuotientRoot, + SubstitutionQuotientSpur, TokenResultMapping } from '@keymanapp/lm-worker/test-index'; @@ -76,10 +77,9 @@ function buildFixture_therefore() { }; }); - // TODO: Use SubstitutionQuotientSpur instead! let quotientNodes: SearchQuotientNode[] = [new SearchQuotientRoot(plainModel)]; for(let i=0; i < 9; i++) { - quotientNodes.push(new LegacyQuotientSpur(quotientNodes[i], distributions[i], inputSources[i])); + quotientNodes.push(new SubstitutionQuotientSpur(quotientNodes[i], distributions[i], inputSources[i])); } return quotientNodes; @@ -99,7 +99,7 @@ describe('QuotientNodeFinalizer', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p)); + assert.approximately(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p), Number.EPSILON * 1000); assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); } else { @@ -107,14 +107,16 @@ describe('QuotientNodeFinalizer', () => { } searchResult = therefo.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. do { searchResult = therefo.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + assert.equal(searchResult.type, 'none'); }); it('finds only corrections when predictions are forbidden', () => { @@ -130,6 +132,11 @@ describe('QuotientNodeFinalizer', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { assert.isAbove(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p)); + + // There are two codepoints missing that are necessary to complete a + // full word with the represented prefix. Check that the penalty is set + // appropriately, accounting for floating-point precision issues. + assert.isAtLeast(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p) + EDIT_DISTANCE_COST_SCALE * 1.99); assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); } else { @@ -137,14 +144,16 @@ describe('QuotientNodeFinalizer', () => { } searchResult = therefo.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); do { searchResult = therefo.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. + assert.equal(searchResult.type, 'none'); }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts index 59ae32846fa..435dfd66b60 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts @@ -21,12 +21,12 @@ import { correctionValidForAutoSelect, generateSubsetId, getBestMatches, - LegacyQuotientSpur, models, PathInputProperties, PathResult, SearchQuotientNode, SearchQuotientRoot, + SubstitutionQuotientSpur, TokenizationCorrector, TokenResult, TokenizationResultMapping @@ -86,22 +86,21 @@ function buildFixture_therefore() { const therefTokens: ContextToken[] = []; // as in "therefore" const the_efTokens: ContextToken[] = []; // as in "the effect" - // TODO: Use SubstitutionQuotientSpur instead! let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=0; i < 3; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); } the_efTokens.push(new ContextToken(firstTokenNode, false)); - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, [distributions[3][1]], { + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, [distributions[3][1]], { ...inputSources[3], subsetId: generateSubsetId() }); // whitespace token alternate - using the ' ' input instead. const whitespaceToken = new ContextToken( - new LegacyQuotientSpur( + new SubstitutionQuotientSpur( new SearchQuotientRoot(plainModel), [distributions[3][0]], { ...inputSources[3], subsetId: generateSubsetId() } @@ -112,11 +111,11 @@ function buildFixture_therefore() { let secondTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=4; i < distributions.length; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], { + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], { ...inputSources[i], subsetId: generateSubsetId() }); - secondTokenNode = new LegacyQuotientSpur(secondTokenNode, distributions[i], { + secondTokenNode = new SubstitutionQuotientSpur(secondTokenNode, distributions[i], { ...inputSources[i], subsetId: generateSubsetId() }) @@ -171,17 +170,16 @@ function buildFixture_terminalWhitespace() { const fullTokens: ContextToken[] = []; const lastToken: ContextToken[] = []; - // TODO: Use SubstitutionQuotientSpur instead! let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=0; i < 5; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); } fullTokens.push(new ContextToken(firstTokenNode, false)); // whitespace token alternate - using the ' ' input instead. const whitespaceToken = new ContextToken( - new LegacyQuotientSpur( + new SubstitutionQuotientSpur( new SearchQuotientRoot(plainModel), distributions[5], inputSources[5], @@ -316,14 +314,16 @@ describe('TokenizationCorrector', () => { } searchResult = instance.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); do { searchResult = instance.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. + assert.equal(searchResult.type, 'none'); }); it('finds corrections for a group of tokens with two correctable', () => { From 77b22a21e0e13c88d7402caceeefb0422591c36f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 23 Apr 2026 08:51:53 -0500 Subject: [PATCH 03/65] change(web): adds temp constructRoot() method to handle legacy vs non-legacy splits, merges --- .../src/main/correction/legacy-quotient-spur.ts | 6 ++++++ .../src/main/correction/search-quotient-cluster.ts | 5 +++-- .../src/main/correction/search-quotient-spur.ts | 13 +++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts index 47cdcdb2622..754a25dd50c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts @@ -15,6 +15,8 @@ import { PathResult } from './correction-searchable.js'; import { SearchNode } from './distance-modeler.js'; import { SearchQuotientNode, PathInputProperties } from './search-quotient-node.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; +import { SearchQuotientRoot } from './search-quotient-root.js'; +import { LegacyQuotientRoot } from './legacy-quotient-root.js'; import { TokenResultMapping } from './token-result-mapping.js'; import Distribution = LexicalModelTypes.Distribution; @@ -51,6 +53,10 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { return new LegacyQuotientSpur(parentNode, inputs, inputSource) as this; } + constructRoot(): SearchQuotientRoot { + return new LegacyQuotientRoot(this.model); + } + protected buildEdgesFromResults(priorResults: ReadonlyArray) { // With a newly-available input, we can extend new input-dependent paths from // our previously-reached 'extractedResults' nodes. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts index 01da2315ba8..e8cf33af464 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts @@ -12,7 +12,6 @@ import { QueueComparator, PriorityQueue } from '@keymanapp/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; import { PathResult } from './correction-searchable.js'; -import { LegacyQuotientRoot } from './legacy-quotient-root.js'; import { generateSpaceSeed, InputSegment, SearchQuotientNode } from './search-quotient-node.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; import { TokenResultMapping } from './token-result-mapping.js'; @@ -221,7 +220,9 @@ export class SearchQuotientCluster extends SearchQuotientNode { split(charIndex: number): [SearchQuotientNode, SearchQuotientNode][] { // Don't rebuild if this is already a perfect split point! if(this.codepointLength <= charIndex) { - return [[this, new LegacyQuotientRoot(this.model)]]; + // We'll assume that the search path is either using legacy nodes or is not; + // we shouldn't see mixed-use cases. + return [[this, (this.parents[0] as SearchQuotientSpur).constructRoot()]]; } const results = this.parents.flatMap((p) => p.split(charIndex)); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index 82fa2acf503..54c164a38d7 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -17,14 +17,12 @@ import { EDIT_DISTANCE_COST_SCALE, SearchNode } from './distance-modeler.js'; import { generateSpaceSeed, InputSegment, PathInputProperties, SearchQuotientNode } from './search-quotient-node.js'; import { generateSubsetId } from './tokenization-subsets.js'; import { SearchQuotientRoot } from './search-quotient-root.js'; -import { LegacyQuotientRoot } from './legacy-quotient-root.js'; import { TokenResultMapping } from './token-result-mapping.js'; import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; -import { LegacyQuotientSpur } from './legacy-quotient-spur.js'; export const QUEUE_NODE_COMPARATOR: QueueComparator = function(arg1, arg2) { return arg1.currentCost - arg2.currentCost; @@ -176,6 +174,11 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { inputSource: PathInputProperties ): this; + // TODO: Remove once LegacyQuotientRoot + LegacyQuotientSpur are removed! + constructRoot(): SearchQuotientRoot { + return new SearchQuotientRoot(this.model); + } + // spaces are in sequence here. // `this` = head 'space'. public merge(space: SearchQuotientNode): SearchQuotientNode { @@ -266,8 +269,7 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { // // stopgap: maybe go ahead and check each input for any that are longer? // won't matter shortly, though. - const rootConstructor = this instanceof LegacyQuotientSpur ? LegacyQuotientRoot : SearchQuotientRoot; - return [[this, new rootConstructor(this.model)]]; + return [[this, this.constructRoot()]]; } else { const firstSet: Distribution = this.inputs.map((input) => ({ // keep insert head @@ -302,10 +304,9 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { }) : this.parentNode; // construct two SearchPath instances based on the two sets! - const rootConstructor = this instanceof LegacyQuotientSpur ? LegacyQuotientRoot : SearchQuotientRoot; return [[ parent, - this.construct(new rootConstructor(this.model), secondSet, { + this.construct(this.constructRoot(), secondSet, { ...this.inputSource, segment: { ...this.inputSource.segment, From 8ba4d0bd9ab3146543d02227368234de7768601d Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 23 Apr 2026 11:19:00 -0500 Subject: [PATCH 04/65] change(web): safeguard clusters from clustering roots --- .../src/main/correction/search-quotient-cluster.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts index e8cf33af464..8d5e15e784e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts @@ -13,6 +13,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { PathResult } from './correction-searchable.js'; import { generateSpaceSeed, InputSegment, SearchQuotientNode } from './search-quotient-node.js'; +import { SearchQuotientRoot } from './search-quotient-root.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; import { TokenResultMapping } from './token-result-mapping.js'; @@ -73,6 +74,10 @@ export class SearchQuotientCluster extends SearchQuotientNode { throw new Error(`SearchQuotientNode does not share the same source identifiers as others in the cluster`); } + if(path instanceof SearchQuotientRoot) { + throw new Error(`SearchQuotientRoot instances may not be part of clusters`); + } + lowestPossibleSingleCost = Math.min(lowestPossibleSingleCost, path.lowestPossibleSingleCost); } From e641f5f6bee8082335be2941bd8e2d399e87fad6 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 9 Mar 2026 16:41:58 -0500 Subject: [PATCH 05/65] feat(web): add dedicated quotient-spur type for insertion edits Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/distance-modeler.ts | 10 +- .../correction/insertion-quotient-spur.ts | 53 ++++++ .../main/correction/token-result-mapping.ts | 8 +- .../worker-thread/src/main/test-index.ts | 1 + .../helpers/buildQuotientDocFixture.tests.ts | 10 +- .../helpers/buildQuotientDocFixture.ts | 75 ++++---- .../insertion-quotient-spur.tests.ts | 173 ++++++++++++++++++ .../substitution-quotient-spur.tests.ts | 2 - 8 files changed, 283 insertions(+), 49 deletions(-) create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index b2f1eae1478..97bd2036d23 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -38,7 +38,7 @@ enum TimedTaskTypes { CORRECTING = 2 } -enum PathEdge { +export enum PathEdge { ROOT = 'root', INSERTION = 'insertion', DELETION = 'deletion', @@ -162,7 +162,7 @@ export class SearchNode { * Notes the edit operation used for the most recent edge in the node's * represented search path. */ - private readonly lastEdgeType: PathEdge; + readonly lastEdgeType: PathEdge; constructor(rootTraversal: LexiconTraversal, spaceId: number, toKey?: (arg0: string) => string); constructor(node: SearchNode, spaceId?: number, edgeType?: PathEdge); @@ -276,7 +276,11 @@ export class SearchNode { * character not seen in the input, as if the user accidentally skipped typing * it. No new input will be expected, but the search will continue one * character deeper in the backing lexicon. - * @param spaceId + * @param spaceId A unique identifier associated with the SearchQuotientNode + * that calls this method and processes the resulting SearchNodes. + * + * If left empty, the nodes will be associated with the same SearchQuotientNode + * as this instance. * @returns An array of SearchNodes corresponding to lexical entries that are * prefixed with the lexicon entry represented by the current Node's * matchSequence text. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts new file mode 100644 index 00000000000..7dae0745930 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts @@ -0,0 +1,53 @@ +/** + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-02-03 + * + * This file adds a SearchQuotientSpur variant modeling insertion of + * lexical-entry prefix characters - an operation with no corresponding + * keystroke. + */ + +import { SENTINEL_CODE_UNIT } from "@keymanapp/models-templates"; +import { SearchNode } from "./distance-modeler.js"; +import { SearchQuotientNode } from "./search-quotient-node.js"; +import { SearchQuotientSpur } from "./search-quotient-spur.js"; +import { TokenResultMapping } from "./token-result-mapping.js"; + +export class InsertionQuotientSpur extends SearchQuotientSpur { + public readonly insertLength = 1; + public readonly leftDeleteLength = 0; + + constructor( + parentNode: SearchQuotientNode + ) { + super(parentNode, null, null, parentNode.codepointLength + 1); + } + + construct(parentNode: SearchQuotientNode): this { + return new InsertionQuotientSpur(parentNode) as this; + } + + protected buildEdgesFromResults(baseNodes: ReadonlyArray): SearchNode[] { + // Note that .buildInsertionEdges will not extend any nodes reached by empty-input + // or by deletions. + return baseNodes + // If there are already at least 2 edits for a node, do not add new edits. + // Also, do not permit insert edits to follow delete edits. + .filter((n) => n.lastEdgeType != 'deletion' && n.editCount < 2) + .flatMap((n) => n.buildInsertionEdges(this.spaceId)); + } + + get edgeKey(): string { + return `SR[${this.parentNode.sourceRangeKey}]L${this.codepointLength}INS`; + } + + get bestExample() { + const base = this.parentNode.bestExample; + // We use the SENTINEL char as an insertion place-holder, as there's no + // actual keystroke to source better characters from. + base.text += SENTINEL_CODE_UNIT; + + return base; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts index de2f8515c73..b0c642c7d4e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts @@ -130,8 +130,8 @@ export class TokenResultMapping implements CorrectionResultMapping{ return new SearchNode(this.node, spaceId); } - buildInsertionEdges(): SearchNode[] { - return this.node.buildInsertionEdges(); + buildInsertionEdges(spaceId?: number): SearchNode[] { + return this.node.buildInsertionEdges(spaceId); } buildDeletionEdges(dist: Distribution, edgeId: number): SearchNode[] { @@ -141,4 +141,8 @@ export class TokenResultMapping implements CorrectionResultMapping{ buildSubstitutionEdges(dist: Distribution, edgeId: number): SearchNode[] { return this.node.buildSubstitutionEdges(dist, edgeId); } + + get lastEdgeType() { + return this.node.lastEdgeType; + } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index 2f28b93bda0..c3468beb80d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -7,6 +7,7 @@ export { ContextTransition } from './correction/context-transition.js'; export * from './correction/correction-searchable.js'; export * from './correction/correction-result-mapping.js'; export * from './correction/distance-modeler.js'; +export * from './correction/insertion-quotient-spur.js'; export * from './correction/substitution-quotient-spur.js'; export * from './correction/search-quotient-cluster.js'; export * from './correction/search-quotient-spur.js'; diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts index 6bf9f0dea7b..f8c661213bd 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts @@ -6,10 +6,10 @@ describe('buildQuotientDocFixture() fixture', () => { it('constructs paths properly', () => { const {searchRoot, nodes} = buildQuotientDocFixture(); - [searchRoot /*, nodes.sc1, nodes.sc2*/].forEach((n) => { + [searchRoot, nodes.sc1, nodes.sc2].forEach((n) => { assert.equal(n.inputCount, 0); }); - [/*nodes.k1c0,*/ nodes.k1c1, nodes.k1c2 /*, nodes.k1c3*/].forEach((n) => { + [/*nodes.k1c0,*/ nodes.k1c1, nodes.k1c2, nodes.k1c3].forEach((n) => { assert.equal(n.inputCount, 1); }); [/*nodes.k2c0, nodes.k2c1,*/ nodes.k2c2, nodes.k2c3].forEach((n) => { @@ -19,13 +19,13 @@ describe('buildQuotientDocFixture() fixture', () => { [searchRoot/*, nodes.k1c0, nodes.k2c0*/].forEach((n) => { assert.equal(n.codepointLength, 0); }); - [/*nodes.sc1, */nodes.k1c1/*, nodes.k2c1*/].forEach((n) => { + [nodes.sc1, nodes.k1c1/*, nodes.k2c1*/].forEach((n) => { assert.equal(n.codepointLength, 1); }); - [/*nodes.sc2,*/ nodes.k1c2, nodes.k2c2].forEach((n) => { + [nodes.sc2, nodes.k1c2, nodes.k2c2].forEach((n) => { assert.equal(n.codepointLength, 2); }); - [/*nodes.k1c3,*/ nodes.k2c3].forEach((n) => { + [nodes.k1c3, nodes.k2c3].forEach((n) => { assert.equal(n.codepointLength, 3); }); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts index 66ee12fe631..f370a45e6a0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts @@ -15,6 +15,7 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs' import { generateSubsetId, + InsertionQuotientSpur, models, SearchQuotientCluster, SearchQuotientRoot, @@ -48,8 +49,8 @@ export function buildQuotientDocFixture() { { sample: { insert: 'cd', deleteLeft: 0, id: key1Id }, p: .2 } ]; - // const sc1 = new InsertionQuotientSpur(searchRoot); - // const sc2 = new InsertionQuotientSpur(sc1); + const sc1 = new InsertionQuotientSpur(searchRoot); + const sc2 = new InsertionQuotientSpur(sc1); // // K1C0 // const k1c0 = new DeletionQuotientSpur(searchRoot, abDistrib.concat(cdDistrib), { @@ -93,15 +94,15 @@ export function buildQuotientDocFixture() { // subsetId: generateSubsetId(), // bestProbFromSet: abDistrib[0].p // }); - // const k1c2_ab = new SubstitutionQuotientSpur(sc1, abDistrib, { - // segment: { - // transitionId: key1Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: abDistrib[0].p - // }); + const k1c2_ab = new SubstitutionQuotientSpur(sc1, abDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); const k1c2_cd = new SubstitutionQuotientSpur(searchRoot, cdDistrib, { segment: { transitionId: key1Id, @@ -111,29 +112,29 @@ export function buildQuotientDocFixture() { subsetId: generateSubsetId(), bestProbFromSet: abDistrib[0].p }); - // const k1c2_ins = new InsertionQuotientSpur(k1c1); - const k1c2 = new SearchQuotientCluster([/*k1c2_del, k1c2_ab, */ k1c2_cd, /*k1c2_ins*/]); + const k1c2_ins = new InsertionQuotientSpur(k1c1); + const k1c2 = new SearchQuotientCluster([/*k1c2_del, */ k1c2_ab, k1c2_cd, k1c2_ins]); - // const k1c3_ab = new SubstitutionQuotientSpur(sc2, abDistrib, { - // segment: { - // transitionId: key1Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: abDistrib[0].p - // }); - // const k1c3_cd = new SubstitutionQuotientSpur(sc1, cdDistrib, { - // segment: { - // transitionId: key1Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: abDistrib[0].p - // }); - // const k1c3_ins = new InsertionQuotientSpur(k1c2); - // const k1c3 = new SearchQuotientCluster([k1c3_ab, k1c3_cd, k1c3_ins]); + const k1c3_ab = new SubstitutionQuotientSpur(sc2, abDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c3_cd = new SubstitutionQuotientSpur(sc1, cdDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c3_ins = new InsertionQuotientSpur(k1c2); + const k1c3 = new SearchQuotientCluster([k1c3_ab, k1c3_cd, k1c3_ins]); // Onto keystroke 2. @@ -235,12 +236,12 @@ export function buildQuotientDocFixture() { subsetId: generateSubsetId(), bestProbFromSet: efDistrib[0].p }); - // const k2c3_ins = new InsertionQuotientSpur(k2c2); - const k2c3 = new SearchQuotientCluster([/*k2c3_del, */ k2c3_ef, k2c3_gh, /*k2c3_ins*/]); + const k2c3_ins = new InsertionQuotientSpur(k2c2); + const k2c3 = new SearchQuotientCluster([/*k2c3_del, */ k2c3_ef, k2c3_gh, k2c3_ins]); return { searchRoot, - spurs: {/*sc1, sc2,*/ k1c1_ab, /*k1c2_ab,*/ k1c2_cd, /*k1c2_ins, k1c3_ab, k1c3_cd, k1c3_ins,*/ k2c2_ef, k2c3_ef, k2c3_gh /*, k2c3_ins*/}, - nodes: {/* sc1, sc2, k1c0, */ k1c1, k1c2, /* k1c3, k2c0, k2c1, */ k2c2, k2c3} + spurs: {sc1, sc2, k1c1_ab, k1c2_ab, k1c2_cd, k1c2_ins, k1c3_ab, k1c3_cd, k1c3_ins, k2c2_ef, k2c3_ef, k2c3_gh, k2c3_ins}, + nodes: {sc1, sc2, /* k1c0, */ k1c1, k1c2, k1c3, /* k2c0, k2c1, */ k2c2, k2c3} }; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts new file mode 100644 index 00000000000..205433ca778 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts @@ -0,0 +1,173 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-06 + * + * This file defines tests for the InsertionQuotientSpur class of the + * predictive-text correction-search engine's search graph. + */ + +import { assert } from 'chai'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { + InsertionQuotientSpur, + models, + SearchQuotientRoot, + SubstitutionQuotientSpur +} from '@keymanapp/lm-worker/test-index'; + +import TrieModel = models.TrieModel; + +import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; +import { buildQuotientDocFixture } from '../../helpers/buildQuotientDocFixture.js'; + +import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + +describe('InsertionQuotientSpur', () => { + describe('constructor', () => { + it('may be extended from root path', () => { + const rootPath = new SearchQuotientRoot(testModel); + const extendedPath = new InsertionQuotientSpur(rootPath); + + assert.equal(extendedPath.inputCount, 0); + assert.equal(extendedPath.codepointLength, 1); + assert.isNumber(extendedPath.spaceId); + assert.notEqual(extendedPath.spaceId, rootPath.spaceId); + assert.equal(extendedPath.bestExample.text.length, 1); + assert.equal(extendedPath.bestExample.p, 1); + + assert.deepEqual(extendedPath.parents, [rootPath]); + assert.deepEqual(extendedPath.inputs, null); + + // Assert the root is unchanged. + assert.equal(rootPath.inputCount, 0); + + // Should (still) have codepointLength == 0 once it's defined. + assert.deepEqual(rootPath.bestExample, {text: '', p: 1}); + assert.deepEqual(rootPath.parents, []); + }); + + it('may be built from arbitrary prior SearchQuotientSpur', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + const inputClone = leadEdgeDistribution.map(e => ({...e})); + + const length1Path = new SubstitutionQuotientSpur( + rootPath, + leadEdgeDistribution, + leadEdgeDistribution[0] + ); + + const length2Path = new InsertionQuotientSpur(length1Path); + + // Verify that the prior distribution remains fully unaltered. + assert.deepEqual(leadEdgeDistribution, inputClone); + + assert.equal(length2Path.inputCount, 1); + assert.equal(length2Path.codepointLength, 2); + assert.isNumber(length2Path.spaceId); + assert.notEqual(length2Path.spaceId, length1Path.spaceId); + assert.equal(length2Path.bestExample.text.length, 2); + assert.equal(length2Path.bestExample.p, leadEdgeDistribution[0].p); + assert.deepEqual(length2Path.parents, [length1Path]); + assert.deepEqual(length2Path.inputs, null); + assert.deepEqual(length2Path.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + assert.equal(length1Path.inputCount, 1); + assert.equal(length1Path.codepointLength, 1); + assert.isNumber(length1Path.spaceId); + assert.notEqual(length1Path.spaceId, rootPath.spaceId); + assert.deepEqual(length1Path.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(length1Path.parents, [rootPath]); + assert.deepEqual(length1Path.inputs, leadEdgeDistribution); + }); + }); + + describe('.edgeKey', () => { + it('is different for different insert locations', () => { + const {sc1, sc2} = buildQuotientDocFixture().nodes; + assert.notEqual(sc1.edgeKey, sc2.edgeKey); + }); + + it('is different for an insert and the spur with the most recent processed input', () => { + const { k1c2_cd, k1c3_ins } = buildQuotientDocFixture().spurs; + assert.notEqual(k1c2_cd.edgeKey, k1c3_ins.edgeKey); + + const k1c2_cd_ins = new InsertionQuotientSpur(k1c2_cd); + assert.notEqual(k1c2_cd_ins.edgeKey, k1c2_cd.edgeKey); + }); + }); + + describe('handleNextNode()', () => { + it('outputs results that insert characters as needed', () => { + const canPath = buildCantLinearFixture().paths[3]; + const followingInsert = new InsertionQuotientSpur(canPath); + + const matchTargets = [ + 'cann', // 'can' + (insert) 'n' (=> 'cannot') + 'care', // 'car' + (insert) 'e' + 'rang', // 'ran' + (insert) 'g' (=> 'range') + ]; + const analysis = analyzeQuotientNodeResults(followingInsert, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.sameMembers(analysis.missing, []); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('outputs results extending prior inserts if insertion spur is parent', () => { + const canPath = buildCantLinearFixture().paths[3]; + const firstInsert = new InsertionQuotientSpur(canPath); + const followingInsert = new InsertionQuotientSpur(firstInsert) + + const matchTargets = [ + 'canno', // 'can' + (insert) 'n' (=> 'cannot') + 'range', // 'ran' + (insert) 'g' (=> 'range') + ]; + const analysis = analyzeQuotientNodeResults(followingInsert, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.sameMembers(analysis.missing, []); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that delete incoming keystrokes as needed', () => { + const canPath = buildCantLinearFixture().paths[3]; + const followingInsert = new InsertionQuotientSpur(canPath); + + const matchTargets = [ + // Delete only first + 'an', // (delete) 'c'/'r'/'t' + 'an', for 'and' and 'any' + 'en', // (delete) 'c'/'r'/'t' + 'en', for 'end', + // Even delete second + 'n', // model possesses words starting with just 'n' + 't', // ... and 't'. + // Delete only third + 'ca', + 'ce', + ]; + const analysis = analyzeQuotientNodeResults(followingInsert, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it.skip('does not output results when immediately following a deletion spur edit', () => { + + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts index a8401aa3b75..8419c4ab4bc 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts @@ -58,8 +58,6 @@ describe('SubstitutionQuotientSpur', () => { assert.deepEqual(rootPath.parents, []); }); - // TODO: for each of the possible ancestor Spur types, + the root! - // Properties should have proper relationship with each test's ancestor. it('may be built from arbitrary prior SearchQuotientSpur', () => { const rootPath = new SearchQuotientRoot(testModel); From fb644ff80651e994fa9788cfbac6333fad049678 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 23 Apr 2026 11:16:19 -0500 Subject: [PATCH 06/65] fix(web): prevent null deref when merging on an insertion node --- .../src/main/correction/search-quotient-cluster.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts index 8d5e15e784e..9492e55252e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts @@ -181,16 +181,16 @@ export class SearchQuotientCluster extends SearchQuotientNode { // What if we're trying to merge something previously split? // That can only happen at the head of the incoming space, so we check for it early here. - if(space.inputCount == 1 && space instanceof SearchQuotientSpur) { + if(space.inputCount == 1 && space instanceof SearchQuotientSpur && space.parents[0] instanceof SearchQuotientRoot) { // In such a case... the 'leading edge' of the incoming space needs to be checked // against the trailing edge of `this` instance's entries. const thisTailInputSource = this.inputSegments[this.inputSegments.length - 1]; - const thisTailSpaceIds = this.parents.map((path) => (path as SearchQuotientSpur).inputSource.subsetId); + const thisTailSpaceIds = this.parents.map((path) => (path as SearchQuotientSpur).inputSource?.subsetId); const spaceHeadInputSource = space.inputSegments[0]; const isOnSplitInput = thisTailSpaceIds.some((entry) => entry == space.inputSource.subsetId) - && thisTailInputSource.end == spaceHeadInputSource.start; + && thisTailInputSource?.end == spaceHeadInputSource.start; // In this case, we only rebuild the single path; an outer stack frame will reconstitute // the split cluster from the individual paths built here. From 30be0fe81b6492e39d0ac07cba8a4bdcb1356725 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 23 Apr 2026 11:27:17 -0500 Subject: [PATCH 07/65] change(web): remove unit test that expected root-based cluster --- .../correction-search/search-quotient-cluster.tests.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts index 7b12d8a8577..029aa6bed31 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts @@ -118,16 +118,6 @@ const determineTargetSplitSequences = (constituentPaths: SearchQuotientSpur[][], describe('SearchQuotientCluster', () => { describe('constructor()', () => { - it('initializes from LegacySearchRoot', () => { - const path = new LegacyQuotientRoot(testModel); - const cluster = new SearchQuotientCluster([path]); - assert.equal(cluster.inputCount, 0); - assert.equal(cluster.codepointLength, 0); - assert.isNumber(cluster.spaceId); - assert.deepEqual(cluster.bestExample, {text: '', p: 1}); - assert.deepEqual(cluster.parents, [path]); - }); - it('initializes from arbitrary SearchQuotientSpur', () => { const rootPath = new LegacyQuotientRoot(testModel); From 4816729d0b1c3857ab1ca631235314c922b98505 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 10 Mar 2026 16:02:56 -0500 Subject: [PATCH 08/65] feat(web): add dedicated DeletionQuotientSpur type for handling deletion-edit search-graph edges Build-bot: skip build:web Test-bot: skip --- .../main/correction/deletion-quotient-spur.ts | 55 +++++ .../worker-thread/src/main/test-index.ts | 1 + .../helpers/buildQuotientDocFixture.tests.ts | 8 +- .../helpers/buildQuotientDocFixture.ts | 197 ++++++++-------- .../deletion-quotient-spur.tests.ts | 210 ++++++++++++++++++ .../insertion-quotient-spur.tests.ts | 23 +- 6 files changed, 390 insertions(+), 104 deletions(-) create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/deletion-quotient-spur.ts create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/deletion-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/deletion-quotient-spur.ts new file mode 100644 index 00000000000..d592a757e31 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/deletion-quotient-spur.ts @@ -0,0 +1,55 @@ +/** + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-02-03 + * + * This file adds a SearchQuotientSpur variant modeling deletion of the corresponding + * keystroke. + */ + +import { LexicalModelTypes } from "@keymanapp/common-types"; + +import { SearchNode } from "./distance-modeler.js"; +import { PathInputProperties, SearchQuotientNode } from "./search-quotient-node.js"; +import { SearchQuotientSpur } from "./search-quotient-spur.js"; +import { TokenResultMapping } from "./token-result-mapping.js"; + +import Distribution = LexicalModelTypes.Distribution; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; + +export class DeletionQuotientSpur extends SearchQuotientSpur { + public readonly insertLength: number = 0; + public readonly leftDeleteLength: number = 0; + + constructor( + parentNode: SearchQuotientNode, + inputs: Distribution>, + inputSource: PathInputProperties | ProbabilityMass + ) { + super(parentNode, inputs, inputSource, parentNode.codepointLength); + } + + construct(parentNode: SearchQuotientNode, inputs: ProbabilityMass>[], inputSource: PathInputProperties): this { + return new DeletionQuotientSpur(parentNode, inputs, inputSource) as this; + } + + protected buildEdgesFromResults(baseResults: ReadonlyArray): SearchNode[] { + return baseResults + // If there are already at least 2 edits for a node, do not add new edits. + .filter((n) => n.editCount < 2) + .flatMap((n) => n.buildDeletionEdges(this.inputs, this.spaceId)); + } + + get edgeKey(): string { + const baseKey = super.edgeKey; + // Be sure to mark the distinction between this node; appending a suffix can + // stack for multiple sequential deletions. + return `${baseKey}DEL`; + } + + get bestExample() { + // As deletion spurs add no new input, we can just re-use the parent node's version. + return this.parentNode.bestExample; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index c3468beb80d..a50eda0ea23 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -7,6 +7,7 @@ export { ContextTransition } from './correction/context-transition.js'; export * from './correction/correction-searchable.js'; export * from './correction/correction-result-mapping.js'; export * from './correction/distance-modeler.js'; +export * from './correction/deletion-quotient-spur.js'; export * from './correction/insertion-quotient-spur.js'; export * from './correction/substitution-quotient-spur.js'; export * from './correction/search-quotient-cluster.js'; diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts index f8c661213bd..812c26b50c6 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts @@ -9,17 +9,17 @@ describe('buildQuotientDocFixture() fixture', () => { [searchRoot, nodes.sc1, nodes.sc2].forEach((n) => { assert.equal(n.inputCount, 0); }); - [/*nodes.k1c0,*/ nodes.k1c1, nodes.k1c2, nodes.k1c3].forEach((n) => { + [nodes.k1c0, nodes.k1c1, nodes.k1c2, nodes.k1c3].forEach((n) => { assert.equal(n.inputCount, 1); }); - [/*nodes.k2c0, nodes.k2c1,*/ nodes.k2c2, nodes.k2c3].forEach((n) => { + [nodes.k2c0, nodes.k2c1, nodes.k2c2, nodes.k2c3].forEach((n) => { assert.equal(n.inputCount, 2); }); - [searchRoot/*, nodes.k1c0, nodes.k2c0*/].forEach((n) => { + [searchRoot, nodes.k1c0, nodes.k2c0].forEach((n) => { assert.equal(n.codepointLength, 0); }); - [nodes.sc1, nodes.k1c1/*, nodes.k2c1*/].forEach((n) => { + [nodes.sc1, nodes.k1c1, nodes.k2c1].forEach((n) => { assert.equal(n.codepointLength, 1); }); [nodes.sc2, nodes.k1c2, nodes.k2c2].forEach((n) => { diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts index f370a45e6a0..d9623c27dc7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts @@ -14,6 +14,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { + DeletionQuotientSpur, generateSubsetId, InsertionQuotientSpur, models, @@ -52,27 +53,27 @@ export function buildQuotientDocFixture() { const sc1 = new InsertionQuotientSpur(searchRoot); const sc2 = new InsertionQuotientSpur(sc1); - // // K1C0 - // const k1c0 = new DeletionQuotientSpur(searchRoot, abDistrib.concat(cdDistrib), { - // segment: { - // transitionId: key1Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: abDistrib[0].p - // }); - - // // K1C1 - // const k1c1_del = new DeletionQuotientSpur(sc1, abDistrib.concat(cdDistrib), { - // segment: { - // transitionId: key1Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: abDistrib[0].p - // }); + // K1C0 + const k1c0 = new DeletionQuotientSpur(searchRoot, abDistrib.concat(cdDistrib), { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + + // K1C1 + const k1c1_del = new DeletionQuotientSpur(sc1, abDistrib.concat(cdDistrib), { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); const k1c1_ab = new SubstitutionQuotientSpur(searchRoot, abDistrib, { segment: { transitionId: key1Id, @@ -82,18 +83,18 @@ export function buildQuotientDocFixture() { subsetId: generateSubsetId(), bestProbFromSet: abDistrib[0].p }); - // const k1c1_ins = new InsertionQuotientSpur(k1c0); - const k1c1 = new SearchQuotientCluster([/*k1c1_del,*/ k1c1_ab, /*k1c1_ins*/]); - - // const k1c2_del = new DeletionQuotientSpur(sc2, abDistrib.concat(cdDistrib), { - // segment: { - // transitionId: key1Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: abDistrib[0].p - // }); + const k1c1_ins = new InsertionQuotientSpur(k1c0); + const k1c1 = new SearchQuotientCluster([k1c1_del, k1c1_ab, k1c1_ins]); + + const k1c2_del = new DeletionQuotientSpur(sc2, abDistrib.concat(cdDistrib), { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); const k1c2_ab = new SubstitutionQuotientSpur(sc1, abDistrib, { segment: { transitionId: key1Id, @@ -113,7 +114,7 @@ export function buildQuotientDocFixture() { bestProbFromSet: abDistrib[0].p }); const k1c2_ins = new InsertionQuotientSpur(k1c1); - const k1c2 = new SearchQuotientCluster([/*k1c2_del, */ k1c2_ab, k1c2_cd, k1c2_ins]); + const k1c2 = new SearchQuotientCluster([k1c2_del, k1c2_ab, k1c2_cd, k1c2_ins]); const k1c3_ab = new SubstitutionQuotientSpur(sc2, abDistrib, { segment: { @@ -148,46 +149,46 @@ export function buildQuotientDocFixture() { { sample: { insert: 'gh', deleteLeft: 0, id: key2Id }, p: .3 } ]; - // const k2c0 = new DeletionQuotientSpur(k1c0, efDistrib.concat(ghDistrib), { - // segment: { - // transitionId: key2Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: efDistrib[0].p - // }); - - // const k2c1_del = new DeletionQuotientSpur(k1c1, efDistrib.concat(ghDistrib), { - // segment: { - // transitionId: key2Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: efDistrib[0].p - // }); - // const k2c1_ef = new SubstitutionQuotientSpur(k1c0, efDistrib, { - // segment: { - // transitionId: key2Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: efDistrib[0].p - // }); - // const k2c1_ins = new InsertionQuotientSpur(k2c0); - // const k2c1 = new SearchQuotientCluster([k2c1_del, k2c1_ef, k2c1_ins]); - - // const k2c2_del = new DeletionQuotientSpur(k1c2, efDistrib.concat(ghDistrib), { - // segment: { - // transitionId: key2Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: efDistrib[0].p - // }); + const k2c0 = new DeletionQuotientSpur(k1c0, efDistrib.concat(ghDistrib), { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + + const k2c1_del = new DeletionQuotientSpur(k1c1, efDistrib.concat(ghDistrib), { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c1_ef = new SubstitutionQuotientSpur(k1c0, efDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c1_ins = new InsertionQuotientSpur(k2c0); + const k2c1 = new SearchQuotientCluster([k2c1_del, k2c1_ef, k2c1_ins]); + + const k2c2_del = new DeletionQuotientSpur(k1c2, efDistrib.concat(ghDistrib), { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); const k2c2_ef = new SubstitutionQuotientSpur(k1c1, efDistrib, { segment: { transitionId: key2Id, @@ -197,27 +198,27 @@ export function buildQuotientDocFixture() { subsetId: generateSubsetId(), bestProbFromSet: efDistrib[0].p }); - // const k2c2_gh = new SubstitutionQuotientSpur(k1c0, ghDistrib, { - // segment: { - // transitionId: key2Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: efDistrib[0].p - // }); - // const k2c2_ins = new InsertionQuotientSpur(k2c1); - const k2c2 = new SearchQuotientCluster([/*k2c2_del,*/ k2c2_ef /*, k2c2_gh, k2c2_ins*/]); - - // const k2c3_del = new DeletionQuotientSpur(k1c3, efDistrib.concat(ghDistrib), { - // segment: { - // transitionId: key2Id, - // start: 0 - // }, - // // Deletions always get their own unique subset ID. - // subsetId: generateSubsetId(), - // bestProbFromSet: efDistrib[0].p - // }); + const k2c2_gh = new SubstitutionQuotientSpur(k1c0, ghDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c2_ins = new InsertionQuotientSpur(k2c1); + const k2c2 = new SearchQuotientCluster([k2c2_del, k2c2_ef, k2c2_gh, k2c2_ins]); + + const k2c3_del = new DeletionQuotientSpur(k1c3, efDistrib.concat(ghDistrib), { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); const k2c3_ef = new SubstitutionQuotientSpur(k1c2, efDistrib, { segment: { transitionId: key2Id, @@ -237,11 +238,11 @@ export function buildQuotientDocFixture() { bestProbFromSet: efDistrib[0].p }); const k2c3_ins = new InsertionQuotientSpur(k2c2); - const k2c3 = new SearchQuotientCluster([/*k2c3_del, */ k2c3_ef, k2c3_gh, k2c3_ins]); + const k2c3 = new SearchQuotientCluster([k2c3_del, k2c3_ef, k2c3_gh, k2c3_ins]); return { searchRoot, - spurs: {sc1, sc2, k1c1_ab, k1c2_ab, k1c2_cd, k1c2_ins, k1c3_ab, k1c3_cd, k1c3_ins, k2c2_ef, k2c3_ef, k2c3_gh, k2c3_ins}, - nodes: {sc1, sc2, /* k1c0, */ k1c1, k1c2, k1c3, /* k2c0, k2c1, */ k2c2, k2c3} + spurs: {sc1, sc2, k1c1_ab, k1c2_ab, k1c2_cd, k1c2_ins, k1c3_ab, k1c3_cd, k1c3_ins, k2c1_del, k2c2_del, k2c2_ef, k2c3_ef, k2c3_gh, k2c3_ins}, + nodes: {sc1, sc2, k1c0, k1c1, k1c2, k1c3, k2c0, k2c1, k2c2, k2c3} }; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts new file mode 100644 index 00000000000..c30282c8309 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts @@ -0,0 +1,210 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-06 + * + * This file defines tests for the InsertionQuotientSpur class of the + * predictive-text correction-search engine's search graph. + */ + +import { assert } from 'chai'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { + DeletionQuotientSpur, + models, + SearchQuotientRoot, + SubstitutionQuotientSpur +} from '@keymanapp/lm-worker/test-index'; + +import TrieModel = models.TrieModel; + +import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; +import { buildQuotientDocFixture } from '../../helpers/buildQuotientDocFixture.js'; + +import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + +describe('DeletionQuotientSpur', () => { + describe('constructor', () => { + it('may be extended from root path', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + + const extendedPath = new DeletionQuotientSpur(rootPath, leadEdgeDistribution, leadEdgeDistribution[0]); + + assert.equal(extendedPath.inputCount, 1); + assert.equal(extendedPath.codepointLength, 0); + assert.isNumber(extendedPath.spaceId); + assert.notEqual(extendedPath.spaceId, rootPath.spaceId); + assert.deepEqual(extendedPath.bestExample, { text: '', p: 1 } ); + + assert.deepEqual(extendedPath.parents, [rootPath]); + assert.deepEqual(extendedPath.inputs, leadEdgeDistribution); + + // Assert the root is unchanged. + assert.equal(rootPath.inputCount, 0); + + // Should (still) have codepointLength == 0 once it's defined. + assert.deepEqual(rootPath.bestExample, {text: '', p: 1}); + assert.deepEqual(rootPath.parents, []); + }); + + it('may be built from arbitrary prior SearchQuotientSpur', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + const inputClone = leadEdgeDistribution.map(e => ({...e})); + + const length1Path = new SubstitutionQuotientSpur( + rootPath, + leadEdgeDistribution, + leadEdgeDistribution[0] + ); + + const tailEdgeDistribution = [ + {sample: {insert: 'r', deleteLeft: 0, id: 17 }, p: 0.6}, + {sample: {insert: 'e', deleteLeft: 0, id: 17 }, p: 0.25}, + {sample: {insert: 'h', deleteLeft: 0, id: 17 }, p: 0.15} + ]; + + const length2Path = new DeletionQuotientSpur( + length1Path, + tailEdgeDistribution, + tailEdgeDistribution[0] + ); + + // Verify that the prior distribution remains fully unaltered. + assert.deepEqual(leadEdgeDistribution, inputClone); + + assert.equal(length2Path.inputCount, 2); + assert.equal(length2Path.codepointLength, 1); + assert.isNumber(length2Path.spaceId); + assert.notEqual(length2Path.spaceId, length1Path.spaceId); + assert.deepEqual(length2Path.bestExample, length1Path.bestExample); + assert.deepEqual(length2Path.parents, [length1Path]); + assert.deepEqual(length2Path.inputs, tailEdgeDistribution); + assert.deepEqual(length2Path.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + }, { + transitionId: tailEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + assert.equal(length1Path.inputCount, 1); + assert.equal(length1Path.codepointLength, 1); + assert.isNumber(length1Path.spaceId); + assert.notEqual(length1Path.spaceId, rootPath.spaceId); + assert.deepEqual(length1Path.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(length1Path.parents, [rootPath]); + assert.deepEqual(length1Path.inputs, leadEdgeDistribution); + }); + }); + + describe('.edgeKey', () => { + it('is different for different delete locations', () => { + const {k1c0, k2c0} = buildQuotientDocFixture().nodes; + + assert.notEqual(k2c0.edgeKey, k1c0.edgeKey); + }); + + it('is different from the parent node\'s key', () => { + const { k1c1_ab, k2c1_del, k1c2_cd, k2c2_del } = buildQuotientDocFixture().spurs; + + assert.notEqual(k2c1_del.edgeKey, k1c1_ab.edgeKey); + assert.notEqual(k2c2_del.edgeKey, k1c2_cd.edgeKey); + }); + }); + + describe('handleNextNode()', () => { + it('does not output results that directly match inputs', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const matchTargets = [ + 'can', + 'car', + 'cen', // 'cent' and 'center' are supported in this test model. + ]; + const analysis = analyzeQuotientNodeResults(deletionPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that substitute inputs', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const matchTargets = [ + // Replacement of first char + 'man', + 'far', + // Replacement of second char + 'con', // 'consider' and variants thereof are also supported. + 'cor', // 'corner' + // Replacement of third char + 'cal', // 'call' + ]; + const analysis = analyzeQuotientNodeResults(deletionPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that insert characters as needed', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const matchTargets = [ + 'can', // 'can' + (insert) 'n' (=> 'cannot') + 'car', // 'car' + (insert) 'e' + 'ran', // 'ran' + (insert) 'g' (=> 'range') + 'cann', // 'can' + (insert) 'n' (=> 'cannot') + 'care', // 'car' + (insert) 'e' + 'rang', // 'ran' + (insert) 'g' (=> 'range') + ]; + const analysis = analyzeQuotientNodeResults(deletionPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('outputs results that delete incoming keystrokes as needed', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const matchTargets = [ + // Deletes the last char. + 'ca', + 'ce', + 're' + ]; + const analysis = analyzeQuotientNodeResults(deletionPath, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.sameMembers(analysis.missing, []); + assert.isEmpty(analysis.foundWithDuplicates); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts index 205433ca778..074f261d349 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts @@ -11,6 +11,7 @@ import { assert } from 'chai'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { + DeletionQuotientSpur, InsertionQuotientSpur, models, SearchQuotientRoot, @@ -166,8 +167,26 @@ describe('InsertionQuotientSpur', () => { assert.isEmpty(analysis.foundWithDuplicates); }); - it.skip('does not output results when immediately following a deletion spur edit', () => { - + it('does not output results when immediately following a deletion spur edit', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const followingInsert = new InsertionQuotientSpur(deletionPath); + + for( + let searchResult = followingInsert.handleNextNode(); + searchResult.type != 'none'; + searchResult = followingInsert.handleNextNode() + ) { + // While the insertion quotient node will forward results from the deletion version, + // it should never actually build search routes that go delete -> insert directly. + if(searchResult.type == 'complete') { + assert.notEqual(searchResult.mapping.lastEdgeType, 'insertion'); + assert.notEqual(searchResult.mapping.spaceId, followingInsert.spaceId); + } + } + // We should reach the end of the loop without once processing an actual 'insert' case. }); }); }); \ No newline at end of file From feba7cd450e04210fa515c45ef48967798f25270 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 10 Mar 2026 16:23:57 -0500 Subject: [PATCH 09/65] feat(web): extend search-graph test-helper funcs to handle specialized spur types Build-bot: skip build:web Test-bot: skip --- .../helpers/constituentPaths.tests.ts | 50 +++++++++++++++++++ .../helpers/constituentPaths.ts | 27 +++++++++- .../helpers/toSpurTypeSequence.ts | 20 ++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 web/src/test/auto/headless/engine/predictive-text/helpers/toSpurTypeSequence.ts diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts index 8ea47e9029f..30cac6a53a9 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts @@ -9,9 +9,13 @@ import { assert } from 'chai'; +import { DeletionQuotientSpur, InsertionQuotientSpur } from '@keymanapp/lm-worker/test-index'; + import { constituentPaths } from "./constituentPaths.js"; +import { toSpurTypeSequence } from './toSpurTypeSequence.js'; import { buildCantLinearFixture } from './buildCantLinearFixture.js'; import { buildAlphabeticClusterFixtures } from './buildAlphabeticClusteredFixture.js'; +import { buildQuotientDocFixture } from './buildQuotientDocFixture.js'; describe('constituentPaths', () => { it('includes a single entry array when all parents are SearchQuotientSpurs', () => { @@ -42,4 +46,50 @@ describe('constituentPaths', () => { return p; })); }); + + describe('for the final quotient-graph doc example', () => { + it('handles insertion-only quotient-graph paths', () => { + const { sc2 } = buildQuotientDocFixture().nodes; + + const sc2Constituents = constituentPaths(sc2); + assert.equal(sc2Constituents.length, 1); + sc2Constituents.forEach(s => s.forEach(p => assert.isTrue(p instanceof InsertionQuotientSpur))); + }); + + it('handles deletion-only quotient-graph paths', () => { + const { k2c0 } = buildQuotientDocFixture().nodes; + + const k2c0Constituents = constituentPaths(k2c0); + assert.equal(k2c0Constituents.length, 1); + k2c0Constituents.forEach(s => s.forEach(p => assert.isTrue(p instanceof DeletionQuotientSpur))); + }); + + it('does not emit sequences with inserts immediately following deletes', () => { + const { k2c3 } = buildQuotientDocFixture().nodes; + + const k2c3Constituents = constituentPaths(k2c3); + + const shouldNotOccur = k2c3Constituents.find((seq) => { + const typeSeq = toSpurTypeSequence(seq); + return typeSeq.find((type, index) => { + return type == 'delete' && typeSeq[index+1] == 'insert'; + }); + }); + assert.isNotOk(shouldNotOccur); + }); + + it('does emit sequences with deletes immediately following inserts', () => { + const { k2c3 } = buildQuotientDocFixture().nodes; + + const k2c3Constituents = constituentPaths(k2c3); + + const shouldOccur = k2c3Constituents.find((seq) => { + const typeSeq = toSpurTypeSequence(seq); + return typeSeq.find((type, index) => { + return type == 'insert' && typeSeq[index+1] == 'delete'; + }); + }); + assert.isNotOk(shouldOccur); + }); + }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts index 9c01297f0ce..80f047989b1 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts @@ -8,6 +8,8 @@ */ import { + DeletionQuotientSpur, + InsertionQuotientSpur, SearchQuotientCluster, SearchQuotientNode, SearchQuotientRoot, @@ -26,8 +28,29 @@ export function constituentPaths(node: SearchQuotientNode): SearchQuotientSpur[] } else if(node instanceof SearchQuotientCluster) { return node.parents.flatMap((p) => constituentPaths(p)); } else if(node instanceof SearchQuotientSpur) { - const parentPaths = constituentPaths(node.parents[0]); - let pathsToExtend = parentPaths; + const parentPaths = constituentPaths(node.parents[0]); let pathsToExtend = parentPaths; + + if(node instanceof InsertionQuotientSpur) { + pathsToExtend = pathsToExtend.filter(s => { + const tail = s[s.length - 1]; + + // Deletion nodes and modules should always be ordered after those for + // insertion in order to avoid duplicating search paths. (Insertions may + // stick to the right of a root, while deletions always process inputs; they + // may thus precede deletions.) + // + // Also, internally, insertion edges are not built after deletion (or empty) edges. + if(tail instanceof DeletionQuotientSpur) { + return false; + } else if(tail.insertLength == 0 && tail.leftDeleteLength == 0) { + // Insertions should also not appear after empty nodes; there's no net + // difference between inserting before and inserting after. + return false; + } + + return true; + }); + } if(parentPaths.length > 0) { return pathsToExtend.map(p => { diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/toSpurTypeSequence.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/toSpurTypeSequence.ts new file mode 100644 index 00000000000..74c9ea182cb --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/toSpurTypeSequence.ts @@ -0,0 +1,20 @@ +import { + DeletionQuotientSpur, + InsertionQuotientSpur, + SearchQuotientNode, + SubstitutionQuotientSpur +} from "@keymanapp/lm-worker/test-index"; + +export function toSpurTypeSequence(spurs: SearchQuotientNode[]): ('insert' | 'delete' | 'substitute' | 'legacy')[] { + return spurs.map(s => { + if(s instanceof InsertionQuotientSpur) { + return 'insert'; + } else if(s instanceof DeletionQuotientSpur) { + return 'delete'; + } else if(s instanceof SubstitutionQuotientSpur) { + return 'substitute'; + } else { + return 'legacy'; + } + }) +} \ No newline at end of file From 599044b4c8a29f448aa298663e92b1239a52ed05 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 23 Apr 2026 13:35:29 -0500 Subject: [PATCH 10/65] fix(web): link and patch up predictive-text test helper function unit tests I happened to discover that the predictive-text "helper" fixtures' unit tests were not actually being run. This links them in (during /worker-thread builds) to ensure they get run too. Build-bot: skip build:web Test-bot: skip --- web/src/engine/predictive-text/worker-thread/build.sh | 2 ++ .../helpers/buildAlphabeticClusteredFixture.tests.ts | 10 ++++++---- .../helpers/buildAlphabeticClusteredFixture.ts | 4 +--- .../predictive-text/helpers/constituentPaths.tests.ts | 2 +- .../correction-search/search-quotient-cluster.tests.ts | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/build.sh b/web/src/engine/predictive-text/worker-thread/build.sh index 0b950479b63..5d3289ac073 100755 --- a/web/src/engine/predictive-text/worker-thread/build.sh +++ b/web/src/engine/predictive-text/worker-thread/build.sh @@ -24,6 +24,7 @@ SRCMAP_CLEANER="${KEYMAN_ROOT}/web/build/tools/building/sourcemap-root/index.js" ################################ Main script ################################ SUBPROJECT_NAME=engine/predictive-text/worker-thread +SUBPROJECT_HELPERS=engine/predictive-text/helpers builder_describe \ "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \ @@ -103,6 +104,7 @@ function do_test() { WTR_INSPECT=" --manual" fi + test-headless-typescript $SUBPROJECT_HELPERS test-headless-typescript $SUBPROJECT_NAME web-test-runner --config ./src/tests/test-runner/web-test-runner${WTR_CONFIG}.config.mjs ${WTR_INSPECT} diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.tests.ts index b680159c044..91198443786 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.tests.ts @@ -9,10 +9,9 @@ import { assert } from "chai"; -import { SearchQuotientNode } from "@keymanapp/lm-worker/test-index"; +import { SearchQuotientNode, SearchQuotientSpur } from "@keymanapp/lm-worker/test-index"; import { constituentPaths } from "./constituentPaths.js"; -import { quotientPathHasInputs } from "./quotientPathHasInputs.js"; import { buildAlphabeticClusterFixtures } from "./buildAlphabeticClusteredFixture.js"; describe('buildAlphabeticClusteredFixture() fixture', () => { @@ -24,7 +23,10 @@ describe('buildAlphabeticClusteredFixture() fixture', () => { const allDists = Object.values(distributions).map(set => Object.values(set)).flat(); const finalClusterPaths = constituentPaths(clusters.cluster_k5c6) as SearchQuotientNode[][]; - allPaths.forEach((spur) => assert.isOk(finalClusterPaths.find(seq => seq.indexOf(spur) > -1))); - allDists.forEach((dist) => assert.isOk(allPaths.find(path => quotientPathHasInputs(path, [dist])))); + allPaths + // This one path doesn't get clustered; it's the only way to reach excess length in this fixture. + .filter((spur) => spur != paths[4].path_k4c6) + .forEach((spur) => assert.isOk(finalClusterPaths.find(seq => seq.indexOf(spur) > -1), `spur ${spur.spaceId} is disconnected`)); + allDists.forEach((dist) => assert.isOk(allPaths.find(path => (path as SearchQuotientSpur).inputs == dist), `distribution ${JSON.stringify(dist)} has no matching path`)); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts index d45cffc7e4c..e330eb51d8e 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts @@ -163,10 +163,8 @@ export const buildAlphabeticClusterFixtures = () => { distrib_c3_i2d0 } }, + root: rootPath, paths: { - 0: { - rootPath - }, 1: { path_k1c1_i1d0, path_k1c2_i2d0, diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts index 30cac6a53a9..d3df0742d50 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts @@ -89,7 +89,7 @@ describe('constituentPaths', () => { return type == 'insert' && typeSeq[index+1] == 'delete'; }); }); - assert.isNotOk(shouldOccur); + assert.isOk(shouldOccur); }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts index 029aa6bed31..421b3df1fdc 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts @@ -808,7 +808,7 @@ describe('SearchQuotientCluster', () => { const mergeResult = baseCluster.merge(new LegacyQuotientSpur( // Is (and mocks) the head result from `path_k4c4_i1.split(3)`. - fixture.paths[0].rootPath, + fixture.root, // Mocks the tail result from `path_k4c4_i1.split(3)`. fixture.distributions[4].distrib_c2_i1d0, { segment: { From 8fb978612b231543bcfe2941fc308b548ebbae0f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 10 Mar 2026 16:25:56 -0500 Subject: [PATCH 11/65] feat(web): add test for split operations on quotient-graph doc example Build-bot: skip build:web Test-bot: skip --- .../search-quotient-spur.tests.ts | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts index d6d61f1c9eb..04065b0e596 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts @@ -26,8 +26,10 @@ import { } from '@keymanapp/lm-worker/test-index'; import { constituentPaths } from '../../helpers/constituentPaths.js'; +import { toSpurTypeSequence } from '../../helpers/toSpurTypeSequence.js'; import { quotientPathHasInputs } from '../../helpers/quotientPathHasInputs.js'; import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; +import { buildQuotientDocFixture } from '../../helpers/buildQuotientDocFixture.js'; import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform; @@ -1143,6 +1145,132 @@ describe('SearchQuotientSpur', () => { assert.deepEqual((head as LegacyQuotientSpur).inputSource, headTarget.inputSource); assert.deepEqual((tail as LegacyQuotientSpur).inputSource, tailTarget.inputSource); }); + + + describe('correctly handles the quotient-path doc example', () => { + it('splits properly at index 0', () => { + const fixture = buildQuotientDocFixture(); + const k2c3 = fixture.nodes.k2c3; + + const splits = k2c3.split(0); + assert.equal(splits.length, 1); + assert.equal(splits[0][0], fixture.searchRoot); + assert.isTrue(splits[0][1].isSameNode(k2c3)); + }); + + it('splits properly at index 1', () => { + const nodes = buildQuotientDocFixture().nodes; + const k2c3 = nodes.k2c3; + + const splits = k2c3.split(1); + /* + * Unironically, actually 5. + * + * 3 are "clean", with 2 "dirty" - the "dirty" two both split in the + * middle of input for the first keystroke. + */ + assert.equal(splits.length, 5); + assert.sameDeepMembers(splits.map(s => s.map(n => n.inputCount)), [ + // clean + [0, 2], // 1 inserted char, then both inputs + [1, 1], // 1 std keystroke, then the other input + [2, 0], // BOTH keystrokes processed (with one deleted), then insertions afterward + // dirty + [1, 2], // 1/2 keystroke inserted, other 1/2 is in tail + [2, 1] // first keystroke deleted, second keystroke 1/2 inserted with remainder in tail + ]); + + splits.forEach(s => { + assert.equal(s[0].codepointLength, 1); + assert.equal(s[1].codepointLength, 2); + }); + + const cleanSplit0KeyHead = splits.find(s => s[0].inputCount == 0 && s[1].inputCount == 2); + const cleanSplit1KeyHead = splits.find(s => s[0].inputCount == 1 && s[1].inputCount == 1); + const cleanSplit2KeyHead = splits.find(s => s[0].inputCount == 2 && s[1].inputCount == 0); + + assert.equal(constituentPaths(cleanSplit0KeyHead[0]).length, 1); + assert.sameDeepMembers(constituentPaths(cleanSplit0KeyHead[0]).map(toSpurTypeSequence), [ + ['insert'] + ]); + assert.equal(constituentPaths(cleanSplit1KeyHead[0]).length, 2); + assert.sameDeepMembers(constituentPaths(cleanSplit1KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'delete'], + ['substitute'] + ]); + assert.equal(constituentPaths(cleanSplit2KeyHead[0]).length, 3); + assert.sameDeepMembers(constituentPaths(cleanSplit2KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'delete', 'delete'], + ['substitute', 'delete'], + ['delete', 'substitute'] + ]); + }); + + it('splits properly at index 2', () => { + const nodes = buildQuotientDocFixture().nodes; + const k2c3 = nodes.k2c3; + + const splits = k2c3.split(2); + /* + * Unironically, actually 5. + * + * 3 are "clean", with 2 "dirty" - the "dirty" two both split in the + * middle of input for the first keystroke. + */ + assert.equal(splits.length, 5); + assert.sameDeepMembers(splits.map(s => s.map(n => n.inputCount)), [ + // clean + [0, 2], // 1 inserted char, then both inputs + [1, 1], // 1 std keystroke, then the other input + [2, 0], // BOTH keystrokes processed (with one deleted), then insertions afterward + // dirty + [1, 2], //insert, then 1/2 keystroke inserted, other 1/2 is in tail + [2, 1] // first keystroke deleted or short, second keystroke 1/2 inserted with remainder in tail + ]); + + splits.forEach(s => { + assert.equal(s[0].codepointLength, 2); + assert.equal(s[1].codepointLength, 1); + }); + + const cleanSplit0KeyHead = splits.find(s => s[0].inputCount == 0 && s[1].inputCount == 2); + const cleanSplit1KeyHead = splits.find(s => s[0].inputCount == 1 && s[1].inputCount == 1); + const cleanSplit2KeyHead = splits.find(s => s[0].inputCount == 2 && s[1].inputCount == 0); + + assert.equal(constituentPaths(cleanSplit0KeyHead[0]).length, 1); + assert.sameDeepMembers(constituentPaths(cleanSplit0KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'insert'] + ]); + assert.equal(constituentPaths(cleanSplit1KeyHead[0]).length, 4); + assert.sameDeepMembers(constituentPaths(cleanSplit1KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'insert', 'delete'], // is too high an edit-cost, though... + ['substitute', 'insert'], + ['insert', 'substitute'], + ['substitute'] + ]); + assert.equal(constituentPaths(cleanSplit2KeyHead[0]).length, 8); + assert.sameDeepMembers(constituentPaths(cleanSplit2KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'insert', 'delete', 'delete'], // is too high an edit-cost, though... + ['substitute', 'substitute'], + ['substitute', 'delete'], + ['delete', 'substitute'], + ['insert', 'substitute', 'delete'], + ['substitute', 'insert', 'delete'], + ['insert', 'delete', 'substitute'], + ['delete', 'substitute', 'insert'] + ]); + }); + + it('splits properly at index 3', () => { + const fixture = buildQuotientDocFixture(); + const k2c3 = fixture.nodes.k2c3; + + const splits = k2c3.split(3); + assert.equal(splits.length, 1); + assert.equal(splits[0][0], k2c3); + assert.isTrue(splits[0][1].isSameNode(fixture.searchRoot)); + }); + }); }); // Placed after `split()` because many cases mock a reversal of split-test results. From 3f768dd383dc56568ee1eb20a57159081af5b3ec Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 5 May 2026 15:47:58 -0500 Subject: [PATCH 12/65] change(web): drop unit test targeting now-invalid case --- .../correction-search/search-quotient-cluster.tests.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts index 7b12d8a8577..029aa6bed31 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts @@ -118,16 +118,6 @@ const determineTargetSplitSequences = (constituentPaths: SearchQuotientSpur[][], describe('SearchQuotientCluster', () => { describe('constructor()', () => { - it('initializes from LegacySearchRoot', () => { - const path = new LegacyQuotientRoot(testModel); - const cluster = new SearchQuotientCluster([path]); - assert.equal(cluster.inputCount, 0); - assert.equal(cluster.codepointLength, 0); - assert.isNumber(cluster.spaceId); - assert.deepEqual(cluster.bestExample, {text: '', p: 1}); - assert.deepEqual(cluster.parents, [path]); - }); - it('initializes from arbitrary SearchQuotientSpur', () => { const rootPath = new LegacyQuotientRoot(testModel); From 6abdee4ed4f4f5b518784355d306ff6dd60e0294 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 5 May 2026 16:06:00 -0500 Subject: [PATCH 13/65] fix(web): block use of predictable "fallback" result to cases where no corrections could be found Build-bot: skip build:web Test-bot: skip --- .../main/correction/tokenization-corrector.ts | 47 +++++++++----- .../tokenization-corrector.tests.ts | 65 +++++++++++++++++++ 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 1818fa51d9e..e47c6b04fad 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -62,6 +62,7 @@ export class TokenizationCorrector implements CorrectionSearchable; private lastTotalCost: number; private handleHasBeenCalled: boolean = false; + private predictableMatchFound: boolean = false; get currentCost(): number { const correctable = this.selectionQueue.peek(); @@ -288,16 +289,19 @@ export class TokenizationCorrector implements CorrectionSearchable correction-string map with the obtained result. this._generatedTokenResults.set(correctableToUpdate.spaceId, tokenResult.mapping); } @@ -351,11 +359,20 @@ export class TokenizationCorrector implements CorrectionSearchable { assert.equal(searchResult.type, 'none'); }); + it('finds a default correction for a single correctable token without a model match', () => { + const fixture = buildFixture_therefore(); + + const theref = fixture.theref.tail; + const xInput: ProbabilityMass = { + sample: { + insert: 'x', + deleteLeft: 0, + id: 123 + }, + p: 1 + } + const therefx = new SubstitutionQuotientSpur(theref.searchModule, [xInput], xInput); + const yInput: ProbabilityMass = { + sample: { + insert: 'y', + deleteLeft: 0, + id: 124 + }, + p: 1 + } + const therefxy = new SubstitutionQuotientSpur(therefx, [yInput], yInput); + const zInput: ProbabilityMass = { + sample: { + insert: 'z', + deleteLeft: 0, + id: 125 + }, + p: 1 + } + const therefxyz = new ContextToken(new SubstitutionQuotientSpur(therefxy, [zInput], zInput)); + const therefxyzTokenization = new ContextTokenization([therefxyz], null, null); + + const instance = new TokenizationCorrector( + therefxyzTokenization, + 1, + fixture.filter + ); + + let searchResult: PathResult; + do { + searchResult = instance.handleNextNode(); + } while(searchResult.type == 'intermediate'); + + assert.equal(searchResult.type, 'complete'); + if(searchResult.type == 'complete') { + const mapping = searchResult.mapping; + const tokenResults = mapping.matchedResult; + assert.isNotNaN(searchResult.cost); + assert.equal(searchResult.cost, searchResult.mapping.totalCost); + assert.equal(tokenResults.length, 1); + assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['therefxyz']); + + // Now that an entry has been found, verify the corrector's state. + assert.isNotOk(instance.predictableToken); // should become an uncorrectable. + assert.isTrue(instance.generatedTokenResults.has(therefxyz)); + assert.equal(instance.generatedTokenResults.get(therefxyz), tokenResults[0]); + } + + // There should be no further possible suggestions. + searchResult = instance.handleNextNode(); + assert.equal(searchResult.type, 'none'); + }); + it('finds corrections for a group of tokens with two correctable', () => { const fixture = buildFixture_therefore(); From 6ca785cd13b1c0742d9c5048e3c6731da89d550f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 8 May 2026 15:04:36 -0500 Subject: [PATCH 14/65] change(web): address PR suggestion of local var reuse --- .../src/main/correction/tokenization-corrector.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index e47c6b04fad..720db56ac86 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -277,8 +277,9 @@ export class TokenizationCorrector implements CorrectionSearchable { - if(correctableToUpdate != this._predictable) { + if(correctionIsThePredictable) { // Lock the 'correctable' token now that either a valid correction for // it has been found or all possible corrections are exhausted. We only // consider a single correction for most of a tokenization's tokens, @@ -289,8 +290,9 @@ export class TokenizationCorrector implements CorrectionSearchable Date: Thu, 14 May 2026 22:48:44 +0700 Subject: [PATCH 15/65] fix(web): apply logic fix from suggestion Co-authored-by: Eberhard Beilharz --- .../worker-thread/src/main/correction/tokenization-corrector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 720db56ac86..a8b229f67b8 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -279,7 +279,7 @@ export class TokenizationCorrector implements CorrectionSearchable { - if(correctionIsThePredictable) { + if(!correctionIsThePredictable) { // Lock the 'correctable' token now that either a valid correction for // it has been found or all possible corrections are exhausted. We only // consider a single correction for most of a tokenization's tokens, From e9346dea9164bfe65543f2a46b0c733d23ed5764 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 14 May 2026 13:38:41 -0500 Subject: [PATCH 16/65] change(web): use enum val for deletion edge check --- .../src/main/correction/insertion-quotient-spur.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts index 7dae0745930..26aa9e55b38 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts @@ -9,7 +9,7 @@ */ import { SENTINEL_CODE_UNIT } from "@keymanapp/models-templates"; -import { SearchNode } from "./distance-modeler.js"; +import { PathEdge, SearchNode } from "./distance-modeler.js"; import { SearchQuotientNode } from "./search-quotient-node.js"; import { SearchQuotientSpur } from "./search-quotient-spur.js"; import { TokenResultMapping } from "./token-result-mapping.js"; @@ -34,7 +34,7 @@ export class InsertionQuotientSpur extends SearchQuotientSpur { return baseNodes // If there are already at least 2 edits for a node, do not add new edits. // Also, do not permit insert edits to follow delete edits. - .filter((n) => n.lastEdgeType != 'deletion' && n.editCount < 2) + .filter((n) => n.lastEdgeType != PathEdge.DELETION && n.editCount < 2) .flatMap((n) => n.buildInsertionEdges(this.spaceId)); } From 98165291ea47f30c026c64eb2e0649b31d884f41 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 15 May 2026 01:40:44 +0700 Subject: [PATCH 17/65] docs(web): fix header comment Co-authored-by: Eberhard Beilharz --- .../correction-search/deletion-quotient-spur.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts index c30282c8309..e9020b51ec7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts @@ -3,7 +3,7 @@ * * Created by jahorton on 2026-03-06 * - * This file defines tests for the InsertionQuotientSpur class of the + * This file defines tests for the DeletionQuotientSpur class of the * predictive-text correction-search engine's search graph. */ From 30f2f363b3d06a59364fd85c565b1bc75a169f2f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 15 May 2026 01:47:47 +0700 Subject: [PATCH 18/65] change(web): Apply EB suggestions from code review Co-authored-by: Eberhard Beilharz --- .../engine/predictive-text/helpers/constituentPaths.tests.ts | 2 +- .../engine/predictive-text/helpers/constituentPaths.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts index 30cac6a53a9..d3df0742d50 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts @@ -89,7 +89,7 @@ describe('constituentPaths', () => { return type == 'insert' && typeSeq[index+1] == 'delete'; }); }); - assert.isNotOk(shouldOccur); + assert.isOk(shouldOccur); }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts index 80f047989b1..38325078719 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts @@ -28,7 +28,8 @@ export function constituentPaths(node: SearchQuotientNode): SearchQuotientSpur[] } else if(node instanceof SearchQuotientCluster) { return node.parents.flatMap((p) => constituentPaths(p)); } else if(node instanceof SearchQuotientSpur) { - const parentPaths = constituentPaths(node.parents[0]); let pathsToExtend = parentPaths; + const parentPaths = constituentPaths(node.parents[0]); + let pathsToExtend = parentPaths; if(node instanceof InsertionQuotientSpur) { pathsToExtend = pathsToExtend.filter(s => { @@ -36,7 +37,7 @@ export function constituentPaths(node: SearchQuotientNode): SearchQuotientSpur[] // Deletion nodes and modules should always be ordered after those for // insertion in order to avoid duplicating search paths. (Insertions may - // stick to the right of a root, while deletions always process inputs; they + // stick to the right of a root, while deletions always process inputs; insertions // may thus precede deletions.) // // Also, internally, insertion edges are not built after deletion (or empty) edges. From 9696ca8b76db509ac90effbd3c1300bac41fd2cb Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 28 May 2026 17:04:06 +0200 Subject: [PATCH 19/65] =?UTF-8?q?chore:=20establish=20epic/boundary-correc?= =?UTF-8?q?tion=20=F0=9F=94=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relates-to: #12893 Test-bot: skip --- docs/epics.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 docs/epics.md diff --git a/docs/epics.md b/docs/epics.md new file mode 100644 index 00000000000..77166f3cd1e --- /dev/null +++ b/docs/epics.md @@ -0,0 +1,18 @@ +# Epics + +This document lists epics in the Keyman project. + + Name | Lead | Target release | Spec +-----------------------|-----------|----------------|-------------------- + epic/embed-osk-in-kmx | @mcdurdin | 19.0 | [embed-osk-in-kmx] + epic/autocorrect | @jahorton | 19.0 | + epic/boundary-correction | @jahorton | 20.0 | + + +Note: when establishing a new epic branch, add the details of the epic to this +file, as a clean way to start an epic without unreviewed code changes in the +base epic PR. + +--- + +[embed-osk-in-kmx]: https://docs.google.com/document/d/15EjtSH7NAsGrdapfB3E7SlS6CqI-2zPy3PNyfUiAv54/edit?tab=t.0 \ No newline at end of file From 1a1c93d098a1ec5b02c68fd4f0f8e793b1568a5c Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 28 May 2026 12:46:08 -0500 Subject: [PATCH 20/65] docs(web): add test file header --- .../helpers/buildQuotientDocFixture.tests.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts index 6bf9f0dea7b..777bdb77867 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts @@ -1,3 +1,12 @@ +/** + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-10 + * + * This file validates that the fixture corresponding to the quotient-graph-doc examples + * meets certain expectations and properties relied upon in other unit tests. + */ + import { assert } from "chai"; import { buildQuotientDocFixture } from "./buildQuotientDocFixture.js"; From da12b14a3f6a85e09352d269006e5e0ed6882d06 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 28 May 2026 16:36:14 -0500 Subject: [PATCH 21/65] change(web): disables one unit test temporarily (to be restored in followup PR) --- .../correction-search/tokenization-corrector.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts index c9900fc96fe..1117b9b7db9 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts @@ -282,7 +282,7 @@ describe('TokenizationCorrector', () => { }); describe('handleNextNode', () => { - it('finds corrections for a single correctable token', () => { + it.skip('finds corrections for a single correctable token', () => { const fixture = buildFixture_therefore(); const tokenization = fixture.theref; From b432c918c1cae30f7d0cb2163aac96dfebf45141 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 28 May 2026 16:38:02 -0500 Subject: [PATCH 22/65] change(web): Revert "change(web): disables one unit test temporarily (to be restored in followup PR)" This reverts commit da12b14a3f6a85e09352d269006e5e0ed6882d06. --- .../correction-search/tokenization-corrector.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts index 543af7e23ca..a53c3a4a4b7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts @@ -283,7 +283,7 @@ describe('TokenizationCorrector', () => { }); describe('handleNextNode', () => { - it.skip('finds corrections for a single correctable token', () => { + it('finds corrections for a single correctable token', () => { const fixture = buildFixture_therefore(); const tokenization = fixture.theref; From 08eda109d731db743f6a12f4ea8ece02bf1a3d61 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 24 Apr 2026 12:55:17 -0500 Subject: [PATCH 23/65] change(web): track ContextState tokenization in array to prep for whitespace correction Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-state.ts | 71 +++++++++++--- .../main/correction/context-tokenization.ts | 7 +- .../src/main/correction/context-transition.ts | 10 +- .../worker-thread/src/main/predict-helpers.ts | 5 +- .../context/context-state.tests.ts | 96 +++++++++---------- .../context/context-tracker.tests.ts | 4 +- .../context/context-transition.tests.ts | 26 ++--- .../determine-suggestion-alignment.tests.ts | 6 +- ...ine-suggestion-context-transition.tests.ts | 10 +- 9 files changed, 143 insertions(+), 92 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index 673acf54937..895970f2854 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -46,7 +46,13 @@ export class ContextState { /** * Denotes the possible tokenization(s) for the represented Context. */ - tokenization: ContextTokenization; + _tokenizations: ContextTokenization[]; + + /** + * Tracks the tokenization pattern best matching the word boundary + * patterns in the actual context. + */ + _displayTokenization: ContextTokenization; /** * Denotes the keystroke-sourced Transform that was last applied to a @@ -97,7 +103,14 @@ export class ContextState { * is visible to the user. */ get displayTokenization(): ContextTokenization { - return this.tokenization; + return this._displayTokenization; + } + + /** + * Denotes the possible tokenization(s) for the represented Context. + */ + get tokenizations(): ContextTokenization[] { + return this._tokenizations; } /** @@ -127,13 +140,16 @@ export class ContextState { * @param tokenization Precomputed tokenization for the context, leveraging previous * correction-search progress and results */ - constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization); - constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization) { + constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization, tokenizations?: ContextTokenization[]); + constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization, tokenizations?: ContextTokenization[]) { if(!(param1 instanceof ContextState)) { this.context = param1; this.model = model; if(tokenization) { - this.tokenization = tokenization; + this._tokenizations = tokenizations ? tokenizations : [tokenization]; + this._displayTokenization = tokenization; + + this.inputTransforms = new Map(); } else { this.initFromReset(); } @@ -142,7 +158,8 @@ export class ContextState { Object.assign(this, stateToClone); this.inputTransforms = new Map(stateToClone.inputTransforms); - this.tokenization = new ContextTokenization(stateToClone.tokenization); + this._displayTokenization = new ContextTokenization(stateToClone._displayTokenization); + this._tokenizations = stateToClone.tokenizations.map((t) => new ContextTokenization(t)); // A shallow copy of the array is fine, but we'd be best off // not aliasing the array itself. @@ -173,10 +190,35 @@ export class ContextState { if(baseTokens.length == 0) { baseTokens.push(ContextToken.fromRawText(this.model, '')); } - this.tokenization = new ContextTokenization(baseTokens); + this._displayTokenization = new ContextTokenization(baseTokens); + this._tokenizations = [this._displayTokenization]; this.inputTransforms = new Map(); } + /** + * Builds a variant of the ContextState with no input metadata, reiterating the resulting + * state of a recent transition but without any associated input mutation data. + * @param stateToClone + * @param context + * @returns + */ + transitionContextWindow(context: Context) { + // The context may have slid since the last ContextState observation. We check for slide effects + // and apply them here as well. + const slideUpdateTransform = determineContextSlideTransform(this.context, context); + + const model = this.model; + + // Should a context-reset or similar occur, there is no input to replace, nor is there + // pre-tail token transform data to preserve. It's important that part, in particular, + // be cleared so that Suggestions in this state are not adversely affected. + const displayTokenization = this._displayTokenization.applyContextSlide(model, slideUpdateTransform); + const tokenizations = this.tokenizations.map((t) => t.applyContextSlide(model, slideUpdateTransform)); + + const state = new ContextState(context, this.model, displayTokenization, tokenizations); + return state; + } + /** * As written, this method attempts to determine the context state and tokenization(s) * that result from applying an incoming transform distribution to the incoming context @@ -209,8 +251,7 @@ export class ContextState { const slideUpdateTransform = determineContextSlideTransform(this.context, context); - // Goal: allow multiple base tokenizations. - const startTokenizations = [this.tokenization].map((t) => { + const startTokenizations = this.tokenizations.map((t) => { return t.applyContextSlide(lexicalModel, slideUpdateTransform); }); @@ -222,14 +263,15 @@ export class ContextState { // If the tokenizations match, clone the ContextState; we want to preserve a post-application // context separately from pre-application contexts for predictions based on empty roots. const state = new ContextState(this); - state.tokenization = [...startTokenizations.values()][0]; + state._tokenizations = [...startTokenizations.values()]; + state._displayTokenization = this._displayTokenization.applyContextSlide(lexicalModel, slideUpdateTransform); transition.finalize(state, transformDistribution); return transition; } const { subsets, keyMatchingUserContext: trueInputSubsetKey } = precomputeTransitions(startTokenizations, transformDistribution); - const resultTokenization = transitionTokenizations(subsets, transformDistribution).get(trueInputSubsetKey); - + const possibleTokenizations = transitionTokenizations(subsets, transformDistribution); + const resultTokenization = possibleTokenizations.get(trueInputSubsetKey); // ------------ // So, if we have a suggestion transition ID at the end and didn't just apply... @@ -240,10 +282,11 @@ export class ContextState { // 'any'.) const state = new ContextState(applyTransform(trueInput, context), lexicalModel); - state.tokenization = resultTokenization; + state._tokenizations = [resultTokenization]; // TODO: [...possibleTokenizations.values()]; + state._displayTokenization = resultTokenization; state.appliedInput = transformDistribution?.[0].sample; transition.finalize(state, transformDistribution); - transition.revertableTransitionId = state.tokenization.tail.appliedTransitionId; + transition.revertableTransitionId = state._displayTokenization.tail.appliedTransitionId; return transition; } } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index f52a1741388..4942ce060ad 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -209,9 +209,14 @@ export class ContextTokenization { // Assumption: deleteLeft is always empty. (There's nothing to the left; // we apply on that side.) + + // This can occur during a context-reset or similar operation that would + // repeat the same prediction state. We build clean tokenizations here, + // bereft of `preservationTransform` data and other data about the original + // state's input. if(TransformUtils.isEmpty(transform)) { // No edits needed? Why retokenize? - return new ContextTokenization(this); + return new ContextTokenization(this.tokens); } // Step 1: build a window for window-start retokenization in case the context-window slid. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts index a75245908e8..1ffb277fc30 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -179,9 +179,13 @@ export class ContextTransition { // body and after any appended whitespace. resultingTokenization.tail.appliedTransitionId = suggestion.transformId; - const resultingState = new ContextState(applyTransform(transformToApply, baseState.context), lexicalModel); - resultingState.tokenization = resultingTokenization; // [resultingTokenization].concat(preservedVariations); - resultingState.appliedInput = baseState.appliedInput; + const resultingState = new ContextState( + applyTransform(transformToApply, baseState.context), + lexicalModel, + resultingTokenization, + [resultingTokenization] // [resultingTokenization].concat(preservedVariations); + ); + resultingState.appliedInput = transformToApply; resultingState.appliedSuggestionId = suggestion.id; resultingState.suggestions = this.final.suggestions; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 9739adba856..45648ab1a2e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -299,8 +299,7 @@ export function determineContextTransition( if(inputIsEmpty) { // Directly build a simple empty transition that duplicates the last seen state. // This should also clear the preservation transform if it exists! - const tokenization = new ContextTokenization(contextTracker.latest.final.tokenization.tokens); - const priorState = new ContextState(context, transition.final.model, tokenization); + const priorState = contextTracker.latest.final.transitionContextWindow(context); transition = new ContextTransition(priorState, inputTransform.id); transition.finalize(priorState, transformDistribution); } else if( @@ -567,7 +566,7 @@ export async function correctAndEnumerate( // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. // The 'eventual' logic will be significantly more complex, though still manageable. - const tokenizations = [transition.final.tokenization]; + const tokenizations = transition.final.tokenizations; const searchModules = tokenizations.map(t => t.tail.searchModule); // Only run the correction search when corrections are enabled. diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts index 5e579004777..677b541181a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts @@ -36,7 +36,7 @@ describe('ContextState', () => { assert.equal(state.context, context); assert.equal(state.model, plainModel); - assert.isOk(state.tokenization); + assert.isOk(state.displayTokenization); assert.isUndefined(state.isManuallyApplied); assert.isNotOk(state.suggestions); assert.isNotOk(state.appliedSuggestionId); @@ -46,36 +46,36 @@ describe('ContextState', () => { it('creates one empty token for an empty context', () => { let context = { left: '', right: '', startOfBuffer: true, endOfBuffer: true }; let state = new ContextState(context, plainModel); - assert.isOk(state.tokenization); - assert.equal(state.tokenization.tokens.length, 1); - assert.equal(state.tokenization.tail.exampleInput, ''); + assert.isOk(state.displayTokenization); + assert.equal(state.displayTokenization.tokens.length, 1); + assert.equal(state.displayTokenization.tail.exampleInput, ''); }); it('creates tokens for initial text (without ending whitespace)', () => { let context = { left: 'the quick brown fox', right: '', startOfBuffer: true, endOfBuffer: true }; let state = new ContextState(context, plainModel); - assert.isOk(state.tokenization); - assert.equal(state.tokenization.tokens.length, 7); - assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']); + assert.isOk(state.displayTokenization); + assert.equal(state.displayTokenization.tokens.length, 7); + assert.deepEqual(state.displayTokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']); let context2 = { left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; let state2 = new ContextState(context2, plainModel); - assert.deepEqual(state2.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(state2.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); }); it('creates tokens for initial text (with extra empty token for ending whitespace)', () => { let context = { left: 'the quick brown fox ', right: '', startOfBuffer: true, endOfBuffer: true }; let state = new ContextState(context, plainModel); - assert.isOk(state.tokenization); - assert.equal(state.tokenization.tokens.length, 9); - assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']); + assert.isOk(state.displayTokenization); + assert.equal(state.displayTokenization.tokens.length, 9); + assert.deepEqual(state.displayTokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']); let context2 = { left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; let state2 = new ContextState(context2, plainModel); - assert.deepEqual(state2.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(state2.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); }); }); @@ -97,7 +97,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // // Phrased this way to facilitate TS type-inference; assert.isTrue() does // // NOT do this for us! @@ -124,7 +124,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // // Phrased this way to facilitate TS type-inference; assert.isTrue() does // // NOT do this for us! @@ -151,7 +151,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // if(!newContextMatch.final.tokenization.alignment.canAlign) { // assert.fail("context alignment failed"); @@ -176,7 +176,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // if(!newContextMatch.final.tokenization.alignment.canAlign) { // assert.fail("context alignment failed"); @@ -201,7 +201,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // if(!newContextMatch.final.tokenization.alignment.canAlign) { // assert.fail("context alignment failed"); @@ -223,7 +223,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // if(!newContextMatch.final.tokenization.alignment.canAlign) { // assert.fail("context alignment failed"); @@ -246,18 +246,18 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch?.final; // space transform - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); // empty transform - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1); - assert.isTrue(state.tokenization.tail.searchModule instanceof SearchQuotientSpur); - assert.deepEqual((state.tokenization.tail.searchModule as SearchQuotientSpur).lastInput, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1); + assert.isTrue(state.displayTokenization.tail.searchModule instanceof SearchQuotientSpur); + assert.deepEqual((state.displayTokenization.tail.searchModule as SearchQuotientSpur).lastInput, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); }); it("properly matches and aligns when whitespace before final empty token is extended", function() { @@ -273,19 +273,19 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch?.final; // Two whitespaces, one of which is new! - const preTail = state.tokenization.tokens[state.tokenization.tokens.length - 2]; + const preTail = state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2]; assert.equal(preTail.searchModule.inputCount, 2); assert.deepEqual((preTail.searchModule.parents[0] as SearchQuotientSpur).lastInput, [{sample: transform, p: 1}]); - assert.equal(state.tokenization.tail.searchModule.inputCount, 1); - assert.isTrue(state.tokenization.tail.searchModule instanceof SearchQuotientSpur); - assert.deepEqual((state.tokenization.tail.searchModule as SearchQuotientSpur).lastInput, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); + assert.equal(state.displayTokenization.tail.searchModule.inputCount, 1); + assert.isTrue(state.displayTokenization.tail.searchModule instanceof SearchQuotientSpur); + assert.deepEqual((state.displayTokenization.tail.searchModule as SearchQuotientSpur).lastInput, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); }); it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() { @@ -301,7 +301,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isOk(newContextMatch?.final); - assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch?.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); }); it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { @@ -317,13 +317,13 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: '', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: '', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch.final; - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1); }) // Needs improved context-state management (due to 2x tokens) @@ -343,15 +343,15 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch.final; - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); assert.equal( - state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1 + state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1 ); // if(!newContextMatch.final.tokenization.alignment.canAlign) { @@ -374,15 +374,15 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, [{sample: transform, p: 1}]); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve all text preceding the new token when applying a suggestion. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: 'd ', deleteLeft: 0}); + assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: 'd ', deleteLeft: 0}); // The 'wordbreak' transform let state = newContextMatch.final; - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); assert.equal( - state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1 + state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1 ); }); @@ -399,14 +399,14 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, [{sample: transform, p: 1}]); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve all text preceding the new token when applying a suggestion. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: 'tor ', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: 'tor ', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch.final; - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1); }); it('handles case where tail token is split into three rather than two', function() { @@ -432,7 +432,7 @@ describe('ContextState', () => { let problemContextMatch = baseState.analyzeTransition({left: "text'", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}]); assert.isNotNull(problemContextMatch); - assert.deepEqual(problemContextMatch.final.tokenization.exampleInput, ['text', '\'', '"']); + assert.deepEqual(problemContextMatch.final.displayTokenization.exampleInput, ['text', '\'', '"']); }); }); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts index f27ebd72f7d..989e950b24b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts @@ -76,10 +76,10 @@ describe('ContextTracker', function() { assert.equal(postContextMatch.final.appliedSuggestionId, baseSuggestion.id); // Penultimate token corresponds to whitespace, which does not have a 'raw' representation. - assert.equal(postContextMatch.final.tokenization.tokens[postContextMatch.final.tokenization.tokens.length - 2].exampleInput, ' '); + assert.equal(postContextMatch.final.displayTokenization.tokens[postContextMatch.final.displayTokenization.tokens.length - 2].exampleInput, ' '); // Final token is empty (follows a wordbreak) - assert.equal(postContextMatch.final.tokenization.tail.exampleInput, ''); + assert.equal(postContextMatch.final.displayTokenization.tail.exampleInput, ''); }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts index 263a24aa471..d3cfd914469 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts @@ -25,9 +25,9 @@ var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), function assertClonedStateMatch(a: ContextState, b: ContextState) { assert.notEqual(a, b); - assert.notEqual(a.tokenization, b.tokenization); - assert.notSameOrderedMembers(a.tokenization.tokens, b.tokenization.tokens); - assert.sameOrderedMembers(a.tokenization.exampleInput, b.tokenization.exampleInput); + assert.notEqual(a.displayTokenization, b.displayTokenization); + assert.notSameOrderedMembers(a.displayTokenization.tokens, b.displayTokenization.tokens); + assert.sameOrderedMembers(a.displayTokenization.exampleInput, b.displayTokenization.exampleInput); assert.deepEqual(a.suggestions, b.suggestions); } @@ -50,7 +50,7 @@ describe('ContextTransition', () => { const transition = new ContextTransition(baseState, 1); assert.sameOrderedMembers( - transition.base.tokenization.tokens.map((t) => t.exampleInput), + transition.base.displayTokenization.tokens.map((t) => t.exampleInput), ['hello', ' ', 'world', ' ', ''] ); assert.equal(transition.transitionId, 1); @@ -67,7 +67,7 @@ describe('ContextTransition', () => { const transition = new ContextTransition(baseState, 1); assert.sameOrderedMembers( - transition.base.tokenization.tokens.map((t) => t.exampleInput), + transition.base.displayTokenization.tokens.map((t) => t.exampleInput), ['hello', ' ', 'world', ' ', ''] ); @@ -143,17 +143,17 @@ describe('ContextTransition', () => { assert.notEqual(appliedTransition.base, transition); assert.isOk(appliedTransition.appended); assert.notEqual(appliedTransition.appended, transition); - assert.sameOrderedMembers(appliedTransition.base.final.tokenization.exampleInput, [ + assert.sameOrderedMembers(appliedTransition.base.final.displayTokenization.exampleInput, [ 'hello', ' ', 'world' ]); - assert.sameOrderedMembers(appliedTransition.appended.final.tokenization.exampleInput, [ + assert.sameOrderedMembers(appliedTransition.appended.final.displayTokenization.exampleInput, [ 'hello', ' ', 'world', ' ', '' ]); assert.equal(appliedTransition.base.final.appliedSuggestionId, suggestions[0].id); assert.equal(appliedTransition.appended.final.appliedSuggestionId, suggestions[0].id); // 3 long, only last token was edited. - appliedTransition.base.final.tokenization.tokens.forEach((token, index) => { + appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 2) { assert.equal(token.appliedTransitionId, suggestions[0].transformId); } else { @@ -161,7 +161,7 @@ describe('ContextTransition', () => { } }); - appliedTransition.appended.final.tokenization.tokens.forEach((token, index) => { + appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 2) { assert.equal(token.appliedTransitionId, suggestions[0].transformId); } else { @@ -227,17 +227,17 @@ describe('ContextTransition', () => { assert.notEqual(appliedTransition.base, transition); assert.isOk(appliedTransition.appended); assert.notEqual(appliedTransition.appended, transition); - assert.sameOrderedMembers(appliedTransition.base.final.tokenization.exampleInput, [ + assert.sameOrderedMembers(appliedTransition.base.final.displayTokenization.exampleInput, [ 'hello', ' ', 'world', ' ', 'the' ]); - assert.sameOrderedMembers(appliedTransition.appended.final.tokenization.exampleInput, [ + assert.sameOrderedMembers(appliedTransition.appended.final.displayTokenization.exampleInput, [ 'hello', ' ', 'world', ' ', 'the', ' ', '' ]); assert.equal(appliedTransition.base.final.appliedSuggestionId, suggestions[0].id); assert.equal(appliedTransition.appended.final.appliedSuggestionId, suggestions[0].id); // 3 long, only last token was edited. - appliedTransition.base.final.tokenization.tokens.forEach((token, index) => { + appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 4) { assert.equal(token.appliedTransitionId, suggestions[0].transformId); } else { @@ -245,7 +245,7 @@ describe('ContextTransition', () => { } }); - appliedTransition.appended.final.tokenization.tokens.forEach((token, index) => { + appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 4) { assert.equal(token.appliedTransitionId, suggestions[0].transformId); } else { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts index be81e711610..be2d1177571 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts @@ -48,7 +48,7 @@ describe('determineSuggestionAlignment', () => { transition.finalize(transition.base, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); // transition, model - const results = determineSuggestionAlignment(transition, transition.final.tokenization, plainCasedModel); + const results = determineSuggestionAlignment(transition, transition.final.displayTokenization, plainCasedModel); assert.deepEqual(results.predictionContext, context); assert.equal(results.deleteLeft, "techn".length); @@ -65,7 +65,7 @@ describe('determineSuggestionAlignment', () => { const transition = baseState.analyzeTransition(context, [{sample: { insert: '', deleteLeft: 1 }, p: 1}]) // transition, model - const results = determineSuggestionAlignment(transition, transition.final.tokenization, plainCasedModel); + const results = determineSuggestionAlignment(transition, transition.final.displayTokenization, plainCasedModel); assert.deepEqual(results.predictionContext, context); assert.equal(results.deleteLeft, "tech".length + 1 /* for the deleted whitespace */); @@ -82,7 +82,7 @@ describe('determineSuggestionAlignment', () => { const transition = baseState.analyzeTransition(context, [{sample: { insert: 'n', deleteLeft: 1 }, p: 1}]) // transition, model - const results = determineSuggestionAlignment(transition, transition.final.tokenization, plainCasedModel); + const results = determineSuggestionAlignment(transition, transition.final.displayTokenization, plainCasedModel); assert.deepEqual(results.predictionContext, context); assert.equal(results.deleteLeft, "techn".length + 1 /* for the deleted whitespace */); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index 218f18313ad..d232d90b1a7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -103,12 +103,12 @@ describe('determineContextTransition', () => { assert.isOk(transition); assert.equal(transition, tracker.latest); assert.isFalse(warningEmitterSpy.called); - assert.sameOrderedMembers(transition.final.tokenization.exampleInput, ['this', ' ', 'is', ' ', 'for', ' ', 'techn']); - assert.isOk(transition.final.tokenization.transitionEdits); + assert.sameOrderedMembers(transition.final.displayTokenization.exampleInput, ['this', ' ', 'is', ' ', 'for', ' ', 'techn']); + assert.isOk(transition.final.displayTokenization.transitionEdits); assert.equal(transition.final.context.left, targetContext.left); assert.equal(transition.final.context.right ?? "", targetContext.right ?? ""); assert.sameDeepOrderedMembers(transition.inputDistribution, inputDistribution); - assert.isNotOk(transition.final.tokenization.taillessTrueKeystroke); + assert.isNotOk(transition.final.displayTokenization.taillessTrueKeystroke); assert.equal(transition.transitionId, 1); } finally { warningEmitterSpy.restore(); @@ -226,8 +226,8 @@ describe('determineContextTransition', () => { assert.notEqual(extendingTransition, baseTransition); // These values support delayed reversions. - assert.equal(extendingTransition.final.tokenization.tokens[6].appliedTransitionId, pred_testing.transformId); - assert.equal(extendingTransition.final.tokenization.tokens[7].appliedTransitionId, pred_testing.transformId); + assert.equal(extendingTransition.final.displayTokenization.tokens[6].appliedTransitionId, pred_testing.transformId); + assert.equal(extendingTransition.final.displayTokenization.tokens[7].appliedTransitionId, pred_testing.transformId); // We start a new token here, rather than continue (and/or replace) an old one; // this shouldn't be set here yet. From cc1aca63d554b92cb235c3d8a612df1144c261a8 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 24 Apr 2026 13:24:03 -0500 Subject: [PATCH 24/65] refactor(web): replace ContextToken.addInput with constructor taking a SearchQuotientNode Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-token.ts | 9 - .../main/correction/context-tokenization.ts | 9 +- .../context/context-token.tests.ts | 409 ++++++++++-------- .../context/context-tokenization.tests.ts | 9 +- .../context/tokenization-subsets.tests.ts | 143 +++--- 5 files changed, 307 insertions(+), 272 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 942773fa57a..0c492c76255 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -15,7 +15,6 @@ import { LegacyQuotientSpur } from "./legacy-quotient-spur.js"; import { LegacyQuotientRoot } from "./legacy-quotient-root.js"; import { generateSubsetId } from './tokenization-subsets.js'; -import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; @@ -119,14 +118,6 @@ export class ContextToken { return new ContextToken(searchModule, isPartial); } - /** - * Call this to record the original keystroke Transforms for the context range - * corresponding to this token. - */ - addInput(inputSource: PathInputProperties, distribution: Distribution) { - this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource); - } - get inputCount() { return this._searchModule.inputCount; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 4942ce060ad..72992f84d34 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -15,12 +15,13 @@ import { TransformUtils } from '../transformUtils.js'; import { computeDistance, EditOperation, EditTuple } from './classical-calculation.js'; import { determineModelTokenizer } from '../model-helpers.js'; import { ExtendedEditOperation, SegmentableDistanceCalculation } from './segmentable-calculation.js'; +import { LegacyQuotientRoot } from './legacy-quotient-root.js'; +import { LegacyQuotientSpur } from './legacy-quotient-spur.js'; import { PathInputProperties } from './search-quotient-node.js'; import { TransitionEdge } from './tokenization-subsets.js'; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; -import { LegacyQuotientRoot } from './legacy-quotient-root.js'; // May be able to "get away" with 2 & 5 or so, but having extra will likely help // with edit path stability. @@ -642,8 +643,10 @@ export class ContextTokenization { inputSource.segment.end = appliedLength; } - affectedToken = new ContextToken(affectedToken); - affectedToken.addInput(inputSource, distribution); + affectedToken = new ContextToken( + new LegacyQuotientSpur(affectedToken.searchModule, distribution, inputSource), + affectedToken.isPartial + ); const tokenize = determineModelTokenizer(lexicalModel); affectedToken.isWhitespace = tokenize({left: affectedToken.exampleInput, startOfBuffer: false, endOfBuffer: false}).left[0]?.isWhitespace ?? false; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts index b9bebb4d35b..75b699cc428 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts @@ -15,7 +15,7 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs' import { LexicalModelTypes } from '@keymanapp/common-types'; import { KMWString } from 'keyman/common/web-utils'; -import { ContextToken, ExecutionTimer, generateSubsetId, getBestMatches, InputSegment, LegacyQuotientRoot, models, SearchQuotientSpur } from '@keymanapp/lm-worker/test-index'; +import { ContextToken, ExecutionTimer, generateSubsetId, getBestMatches, InputSegment, LegacyQuotientRoot, LegacyQuotientSpur, models, SearchQuotientSpur } from '@keymanapp/lm-worker/test-index'; import { quotientPathHasInputs } from "../../helpers/quotientPathHasInputs.js"; @@ -126,36 +126,45 @@ describe('ContextToken', function() { const srcTransform = { insert: "can't", deleteLeft: 0, deleteRight: 0, id: 1 }; const srcSubsetId = generateSubsetId(); - const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); - const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); - const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); - token1.addInput({ - segment: { - transitionId: srcTransform.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetId - }, [{sample: {insert: 'can', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); - - token2.addInput({ - segment: { - transitionId: srcTransform.id, - start: 3 - }, - bestProbFromSet: 1, - subsetId: srcSubsetId - }, [{sample: {insert: "'", deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); - - token3.addInput({ - segment: { - transitionId: srcTransform.id, - start: 4 - }, - bestProbFromSet: 1, - subsetId: srcSubsetId - }, [{sample: {insert: 't', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: {insert: 'can', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}], { + segment: { + transitionId: srcTransform.id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetId + } + )); + + token2 = new ContextToken(new LegacyQuotientSpur( + token2.searchModule, + [{sample: {insert: "'", deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}], { + segment: { + transitionId: srcTransform.id, + start: 3 + }, + bestProbFromSet: 1, + subsetId: srcSubsetId + } + )); + + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: {insert: 't', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}], { + segment: { + transitionId: srcTransform.id, + start: 4 + }, + bestProbFromSet: 1, + subsetId: srcSubsetId + } + )); const merged = ContextToken.merge([token1, token2, token3]); assert.equal(merged.exampleInput, "can't"); @@ -184,67 +193,85 @@ describe('ContextToken', function() { ]; // apples - const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); // and - const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); // sour - const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); // grapes - const token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); - const tokensToMerge = [token1, token2, token3, token4] + let token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); - token1.addInput({ - segment: { - transitionId: srcTransforms[0].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[0] - }, [{sample: srcTransforms[0], p: 1}]); - token1.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - - token2.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 1 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: "and", deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - - token3.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 4 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - token3.addInput({ - segment: { - transitionId: srcTransforms[2].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[2] - }, [{sample: srcTransforms[2], p: 1}]); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: srcTransforms[0], p: 1}], { + segment: { + transitionId: srcTransforms[0].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[0] + } + )); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); - token4.addInput({ - segment: { - transitionId: srcTransforms[3].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[3] - }, [{sample: srcTransforms[3], p: 1}]); + token2 = new ContextToken(new LegacyQuotientSpur( + token2.searchModule, + [{sample: {insert: "and", deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 1 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 4 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: srcTransforms[2], p: 1}], { + segment: { + transitionId: srcTransforms[2].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[2] + } + )); + + token4 = new ContextToken(new LegacyQuotientSpur( + token4.searchModule, + [{sample: srcTransforms[3], p: 1}], { + segment: { + transitionId: srcTransforms[3].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[3] + } + )); + + const tokensToMerge = [token1, token2, token3, token4]; const merged = ContextToken.merge(tokensToMerge); assert.equal(merged.exampleInput, "applesandsourgrapes"); assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({ @@ -273,68 +300,86 @@ describe('ContextToken', function() { generateSubsetId() ]; - // apples - const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); + // apples + let token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); // and - const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); // sour - const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); // grapes - const token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); - const tokensToMerge = [token1, token2, token3, token4] + let token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); - token1.addInput({ - segment: { - transitionId: srcTransforms[0].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[0] - }, [{sample: srcTransforms[0], p: 1}]); - token1.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - - token2.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 1 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: toMathematicalSMP("and"), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - - token3.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 4 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - token3.addInput({ - segment: { - transitionId: srcTransforms[2].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[2] - }, [{sample: srcTransforms[2], p: 1}]); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: srcTransforms[0], p: 1}], { + segment: { + transitionId: srcTransforms[0].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[0] + } + )); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); - token4.addInput({ - segment: { - transitionId: srcTransforms[3].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[3] - }, [{sample: srcTransforms[3], p: 1}]); + token2 = new ContextToken(new LegacyQuotientSpur( + token2.searchModule, + [{sample: {insert: toMathematicalSMP("and"), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 1 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); + + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 4 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: srcTransforms[2], p: 1}], { + segment: { + transitionId: srcTransforms[2].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[2] + } + )); + + token4 = new ContextToken(new LegacyQuotientSpur( + token4.searchModule, + [{sample: srcTransforms[3], p: 1}], { + segment: { + transitionId: srcTransforms[3].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[3] + } + )); + const tokensToMerge = [token1, token2, token3, token4]; const merged = ContextToken.merge(tokensToMerge); assert.equal(merged.exampleInput, toMathematicalSMP("applesandsourgrapes")); assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({ @@ -370,15 +415,18 @@ describe('ContextToken', function() { ] ] - const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); + let tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({ - segment: { - transitionId: keystrokeDistributions[i][0].sample.id, - start: 0 - }, bestProbFromSet: .75, - subsetId: generateSubsetId() - }, keystrokeDistributions[i]); + tokenToSplit = new ContextToken(new LegacyQuotientSpur( + tokenToSplit.searchModule, + keystrokeDistributions[i], { + segment: { + transitionId: keystrokeDistributions[i][0].sample.id, + start: 0 + }, bestProbFromSet: .75, + subsetId: generateSubsetId() + } + )); }; assert.equal(tokenToSplit.sourceRangeKey, 'T11+T12+T13+T14'); @@ -413,16 +461,19 @@ describe('ContextToken', function() { const splitTextArray = ['big', 'large', 'transform']; const subsetId = generateSubsetId(); - const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); + let tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({ - segment: { - transitionId: keystrokeDistributions[i][0].sample.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId - }, keystrokeDistributions[i]); + tokenToSplit = new ContextToken(new LegacyQuotientSpur( + tokenToSplit.searchModule, + keystrokeDistributions[i], { + segment: { + transitionId: keystrokeDistributions[i][0].sample.id, + start: 0 + }, + bestProbFromSet: 1, + subsetId + } + )); }; assert.equal(tokenToSplit.sourceRangeKey, `T${keystrokeDistributions[0][0].sample.id}`); @@ -484,16 +535,19 @@ describe('ContextToken', function() { generateSubsetId() ]; - const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); + let tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({ - segment: { - transitionId: keystrokeDistributions[i][0].sample.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: subsetIds[i] - }, keystrokeDistributions[i]); + tokenToSplit = new ContextToken(new LegacyQuotientSpur( + tokenToSplit.searchModule, + keystrokeDistributions[i], { + segment: { + transitionId: keystrokeDistributions[i][0].sample.id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: subsetIds[i] + } + )); }; assert.equal(tokenToSplit.exampleInput, 'largelongtransforms'); @@ -611,16 +665,19 @@ describe('ContextToken', function() { generateSubsetId() ]; - const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); + let tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({ - segment: { - transitionId: keystrokeDistributions[i][0].sample.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: subsetIds[i] - }, keystrokeDistributions[i]); + tokenToSplit = new ContextToken(new LegacyQuotientSpur( + tokenToSplit.searchModule, + keystrokeDistributions[i], { + segment: { + transitionId: keystrokeDistributions[i][0].sample.id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: subsetIds[i] + } + )); }; assert.equal(tokenToSplit.exampleInput, toMathematicalSMP('largelongtransforms')); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 3c8351f7d0b..44ad985578f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -28,7 +28,8 @@ import { models, TransitionEdge, SearchQuotientSpur, - traceInsertEdits + traceInsertEdits, + LegacyQuotientSpur } from '@keymanapp/lm-worker/test-index'; import Transform = LexicalModelTypes.Transform; @@ -50,13 +51,15 @@ function toTransformToken(text: string, transformId?: number) { let isWhitespace = text == ' '; let token = ContextToken.fromRawText(plainModel, ''); const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed }; - token.addInput({ + token = new ContextToken(new LegacyQuotientSpur( + token.searchModule, + [ { sample: textAsTransform, p: 1 } ], { segment: { transitionId: textAsTransform.id, start: 0 }, bestProbFromSet: 1, subsetId: generateSubsetId() - }, [ { sample: textAsTransform, p: 1 } ]); + })); token.isWhitespace = isWhitespace; return token; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index 04647123c02..8fc319eb646 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -20,7 +20,7 @@ import { buildEdgeWindow, ContextToken, ContextTokenization, - generateSubsetId, + LegacyQuotientSpur, models, precomputationSubsetKeyer, TokenizationTransitionEdits, @@ -181,16 +181,11 @@ describe('precomputationSubsetKeyer', function() { [...tokenization.tokens, (() => { const token = ContextToken.fromRawText(plainModel, 'da'); // source text: 'date' - token.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ + const dist = [ {sample: {insert: 'te', deleteLeft: 0, id: 13}, p: 1} - ]); - return token; + ]; + const space = new LegacyQuotientSpur(token.searchModule, dist, dist[0]); + return new ContextToken(space); })()], { insert: 's', deleteLeft: 0, deleteRight: 0 }, false @@ -213,16 +208,11 @@ describe('precomputationSubsetKeyer', function() { [...tokenization.tokens, (() => { const token = ContextToken.fromRawText(plainModel, 'da'); // source text: 'date' - token.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ - {sample: {insert: 't', deleteLeft: 0}, p: 1} - ]); - return token; + const dist = [ + {sample: {insert: 't', deleteLeft: 0, id: 13}, p: 1} + ]; + const space = new LegacyQuotientSpur(token.searchModule, dist, dist[0]); + return new ContextToken(space); })()], { insert: 'es', deleteLeft: 0, deleteRight: 0, id: 14 }, false @@ -257,17 +247,15 @@ describe('precomputationSubsetKeyer', function() { ...buildEdgeWindow( [...tokenization.tokens, (() => { const token = ContextToken.fromRawText(plainModel, 'da'); - token.isPartial = true; // source text: 'dat' - token.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [{sample: {insert: 'ts', deleteLeft: 0, id: 13}, p: 1} - ]); - return token; + const dist = [ + {sample: {insert: 'ts', deleteLeft: 0, id: 13}, p: 1} + ]; + const space = new LegacyQuotientSpur(token.searchModule, dist, dist[0]); + let token2 = new ContextToken(space); + token2.isPartial = true; + + return token2; })()], { insert: 'e', deleteLeft: 1, deleteRight: 0, id: 14 }, false @@ -289,18 +277,15 @@ describe('precomputationSubsetKeyer', function() { ...buildEdgeWindow( [...tokenization.tokens, (() => { const token = ContextToken.fromRawText(plainModel, 'da'); - token.isPartial = true; // source text: 'dat' - token.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ + const dist = [ {sample: {insert: 't', deleteLeft: 0, id: 13}, p: 1} - ]); - return token; + ]; + const space = new LegacyQuotientSpur(token.searchModule, dist, dist[0]); + let token2 = new ContextToken(space); + token2.isPartial = true; + + return token2; })()], { insert: 'e', deleteLeft: 0, deleteRight: 0, id: 14 }, false @@ -747,27 +732,25 @@ describe('TokenizationSubsetBuilder', function() { const trueSourceTransform: Transform = { insert: 'รฉ', deleteLeft: 1, id: 13 }; - const fourCharTailToken = new ContextToken(baseTokenization.tail); - fourCharTailToken.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ - { sample: trueSourceTransform, p: .6 } - ]); - - const fiveCharTailToken = new ContextToken(baseTokenization.tail); - fiveCharTailToken.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ + let fourCharTailToken = new ContextToken(baseTokenization.tail); + let fourCharTailDist = [{sample: trueSourceTransform, p: .6}]; + let fourCharTailSpace = new LegacyQuotientSpur( + fourCharTailToken.searchModule, + fourCharTailDist, + fourCharTailDist[0] + ); + fourCharTailToken = new ContextToken(fourCharTailSpace); + + let fiveCharTailToken = new ContextToken(baseTokenization.tail); + let fiveCharTailDist = [ { sample: { insert: 's', deleteLeft: 0, id: 13 }, p: .4 } - ]); + ]; + let fiveCharTailSpace = new LegacyQuotientSpur( + fiveCharTailToken.searchModule, + fiveCharTailDist, + fiveCharTailDist[0] + ); + fiveCharTailToken = new ContextToken(fiveCharTailSpace); const subsetBuilder = new TokenizationSubsetBuilder(); const fourCharTokenization = new ContextTokenization([...baseTokenization.tokens.slice(0, -1), fourCharTailToken]); @@ -796,27 +779,25 @@ describe('TokenizationSubsetBuilder', function() { const trueSourceTransform: Transform = { insert: 'รฉ', deleteLeft: 1, id: 13 }; - const twoCharTailToken = new ContextToken(baseTokenization.tail); - twoCharTailToken.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: .6, - subsetId: generateSubsetId() - }, [ - { sample: trueSourceTransform, p: .6 } - ]); - - const threeCharTailToken = new ContextToken(baseTokenization.tail); - threeCharTailToken.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: .6, - subsetId: generateSubsetId() - }, [ - { sample: { insert: 'a', deleteLeft: 0, id: 13}, p: .4 } - ]); + let twoCharTailToken = new ContextToken(baseTokenization.tail); + let twoCharTailDist = [{sample: trueSourceTransform, p: .6}]; + let twoCharTailSpace = new LegacyQuotientSpur( + twoCharTailToken.searchModule, + twoCharTailDist, + twoCharTailDist[0] + ); + twoCharTailToken = new ContextToken(twoCharTailSpace); + + let threeCharTailToken = new ContextToken(baseTokenization.tail); + let threeCharTailDist = [ + { sample: { insert: 'a', deleteLeft: 0, id: 13 }, p: .4 } + ]; + let threeCharTailSpace = new LegacyQuotientSpur( + threeCharTailToken.searchModule, + threeCharTailDist, + threeCharTailDist[0] + ); + threeCharTailToken = new ContextToken(threeCharTailSpace); const subsetBuilder = new TokenizationSubsetBuilder(); const twoCharTokenization = new ContextTokenization([...baseTokenization.tokens.slice(0, -1), twoCharTailToken]); From 9d35d3d8263c8677f9a9592978420e9758e59af9 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 21 May 2026 16:01:35 -0500 Subject: [PATCH 25/65] change(web): generalize determineSuggestionRange To facilitate using the same suggestion-application-range logic for all model types, not just the first-class ones that implement LexiconTraversals. Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-token.ts | 47 ++++++- .../worker-thread/src/main/predict-helpers.ts | 121 +++++++++++++----- .../determine-suggestion-range.tests.ts | 20 +-- 3 files changed, 142 insertions(+), 46 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 0c492c76255..ce1d8f9830a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -35,11 +35,42 @@ function textToCharTransforms(text: string, transformId?: number): Transform[] { [...text].map(insert => ({insert, deleteLeft: 0})); } + +/** + * Implements an interface similar to ContextToken that is useful for handling + * cases that should not be considered correctable. + */ +export interface ContextTokenLike { + /** + * Generates text corresponding to the net effects of the most likely inputs + * received that can correspond to the represented token. + */ + exampleInput: string; + + /** + * Reports the length in codepoints of corrected text represented by the + * current token. + */ + codepointLength: number; + + /** + * Whether or not the token is likely still being edited by the user (due to + * adjacency of the caret) + */ + isPartial?: boolean; + + /** + * Gets a compact string-based representation of `inputRange` that + * maps compatible token source ranges to each other. + */ + sourceRangeKey?: string; +} + /** * Represents cached data about one token (either a word or a unit of whitespace) * in the context and associated correction-search progress and results. */ -export class ContextToken { +export class ContextToken implements ContextTokenLike { /** * Indicates whether or not the token is considered whitespace. */ @@ -54,6 +85,10 @@ export class ContextToken { } private _searchModule: SearchQuotientNode; + /** + * Whether or not the token is likely still being edited by the user (due to + * adjacency of the caret) + */ isPartial: boolean; /** @@ -118,6 +153,14 @@ export class ContextToken { return new ContextToken(searchModule, isPartial); } + /** + * Reports the length in codepoints of corrected text represented by the + * current token. + */ + get codepointLength() { + return this._searchModule.codepointLength; + } + get inputCount() { return this._searchModule.inputCount; } @@ -155,7 +198,7 @@ export class ContextToken { /** * Generates text corresponding to the net effects of the most likely inputs - * received that can correspond to the current instance. + * received that can correspond to the represented token. */ get exampleInput(): string { return this.searchModule.bestExample.text; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 45648ab1a2e..b9d8919b9a3 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -5,9 +5,9 @@ import { searchForProperty, WordBreakProperty } from '@keymanapp/models-wordbrea import { TransformUtils } from './transformUtils.js'; import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; +import { ContextTokenLike } from './correction/context-token.js'; import { ContextTokenization } from './correction/context-tokenization.js'; import { ContextTracker } from './correction/context-tracker.js'; -import { ContextToken } from './correction/context-token.js'; import { ContextState, determineContextSlideTransform } from './correction/context-state.js'; import { ContextTransition } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; @@ -73,6 +73,43 @@ export const CORRECTION_SEARCH_THRESHOLDS = { REPLACEMENT_SEARCH_THRESHOLD: 4 as const // e^-4 = 0.0183156388. Allows "80%" of an extra edit. } +/** + * Represents the minimum replacement range and effects required for + * suggestions. + * + * These values are based on properties of the transition from their base + * context-tokenization to their target-tokenization (and its represented + * context variant). + */ +export interface SuggestionReplacement { + /** + * Tokens lost from the base context-tokenization in the target + * context-tokenization due to the transition event. + * + * These are implicitly replaced when applying Suggestions. + */ + tokensToRemove: T[], + + /** + * Tokens added (after the removed tokens) to the base context-tokenization to + * produce the target context-tokenization. + * + * As these are "new" tokens generated by the transition, Suggestions should represent + * corrections and predictions rooted upon these tokens. + */ + tokensToPredict: T[], + + /** + * Indicates the total range of left-deletion needed when applying suggestions. + */ + deleteLeft: number, + + /** + * Indicates the id of the underlying context transition. + */ + transitionId?: number +} + /** * Collates information related to suggestions during the suggestion generation * process. @@ -395,53 +432,67 @@ export function determineSuggestionAlignment( * @param variantForSuggestions * @returns */ -export function determineSuggestionRange( - userContextTokenization: ContextTokenization, - variantForSuggestions: ContextTokenization -): { tokensToRemove: ContextToken[], tokensToPredict: ContextToken[] } { - // Assumption: spaceIds monotonically increase as new ones are generated. - // Given this, we backtrace on the token tails until finding a spot where the - // spaceIds match, dropping any that are newer than the last found in the - // other. - // - // We full-replace all tokens affected by an applied suggestion, so if there's - // a mismatch between the final form of a token, that implies that suggestions - // would replace the original form of the token anyway. - const tokenSetA = userContextTokenization.tokens.slice(); - const tokenSetB = variantForSuggestions.tokens.slice(); - - const tokensToRemove: ContextToken[] = []; - const tokensToPredict: ContextToken[] = []; - - const tailIdFor = (tokens: ContextToken[]) => tokens[tokens.length-1]?.spaceId ?? -1; - let tailOfA = tailIdFor(tokenSetA); - let tailOfB = tailIdFor(tokenSetB); - while(tailOfA != tailOfB) { - if(tailOfA < tailOfB) { - tokensToPredict.push(tokenSetB.pop()); - tailOfB = tailIdFor(tokenSetB); - } else { - tokensToRemove.push(tokenSetA.pop()); - tailOfA = tailIdFor(tokenSetA); +export function determineSuggestionRange( + userContextTokenization: T[], + variantForSuggestions: T[], + equalityChecker: (a: T, b: T) => boolean +): SuggestionReplacement { + // Add null/undefined guards to the equality checker. + const temp = equalityChecker; + equalityChecker = (a, b) => { + if(!a || !b) { + return false; + } + + return temp(a, b); + } + + const deleteLeftCalc = (tokenSet: T[], predictCount: number) => { + // TODO: once we start activating multi-tokenization for real, only the + // 'reduce' component should remain. + return (predictCount > 1) + ? (tokenSet[tokenSet.length - 1]?.codepointLength ?? 0) + : tokenSet.reduce((prev, curr) => prev + curr.codepointLength, 0); + } + + const tokenSetA = userContextTokenization.slice(); + const tokenSetB = variantForSuggestions.slice(); + + let aHeadIndexInB = tokenSetB.findIndex((t) => equalityChecker(t, tokenSetA[0])); + let bHeadIndexInA = tokenSetA.findIndex((t) => equalityChecker(t, tokenSetB[0])); + + if(aHeadIndexInB == -1 && bHeadIndexInA == -1) { + // Both are full replacements. + return { + tokensToRemove: tokenSetA, + tokensToPredict: tokenSetB, + deleteLeft: deleteLeftCalc(tokenSetA, tokenSetB.length) } + } else if(aHeadIndexInB != 0 && bHeadIndexInA != 0) { + throw new Error("Leading edge of context should not differ in both tokenizations."); + } + + let tailOffset = 0; + while(equalityChecker(tokenSetA[bHeadIndexInA + tailOffset], tokenSetB[aHeadIndexInB + tailOffset])) { + tailOffset++; } - tokensToPredict.reverse(); + const tokensToRemove: T[] = tokenSetA.slice(bHeadIndexInA + tailOffset); + const tokensToPredict: T[] = tokenSetB.slice(aHeadIndexInB + tailOffset); // Can occur when backspacing to the end of a previous word. if(tokensToPredict.length == 0) { if(tokenSetA.length == 0 || tokenSetB.length == 0) { throw new Error("Invalid state - a tokenization is missing expected tokens"); } - tokensToRemove.push(tokenSetA.pop()); - tokensToPredict.push(tokenSetB.pop()); + tokensToRemove.unshift(tokenSetA[bHeadIndexInA + tailOffset - 1]); + tokensToPredict.unshift(tokenSetB[aHeadIndexInB + tailOffset - 1]); } - tokensToRemove.reverse(); - return { tokensToRemove, - tokensToPredict + tokensToPredict, + deleteLeft: deleteLeftCalc(tokensToRemove, tokensToPredict.length) } } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts index 3445abaadd9..e18174895ec 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts @@ -170,12 +170,14 @@ function buildQuickBrownFixture() { }; } +const tokenEquality = (a: ContextToken, b: ContextToken) => a.spaceId == b.spaceId; + describe('determineSuggestionRange', () => { it('adjusts the final token if no tokenization changes occur', () => { const fixture = buildQuickBrownFixture(); const noChange = fixture.variations.noChange; - const analysis = determineSuggestionRange(fixture.baseTokenization, noChange.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, noChange.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, noChange.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, noChange.range.tokensToPredict); @@ -185,7 +187,7 @@ describe('determineSuggestionRange', () => { const fixture = buildQuickBrownFixture(); const plainInsert = fixture.variations.plainInsert; - const analysis = determineSuggestionRange(fixture.baseTokenization, plainInsert.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, plainInsert.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, plainInsert.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, plainInsert.range.tokensToPredict); @@ -195,7 +197,7 @@ describe('determineSuggestionRange', () => { const fixture = buildQuickBrownFixture(); const newTokenInsert = fixture.variations.newTokenInsert; - const analysis = determineSuggestionRange(fixture.baseTokenization, newTokenInsert.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, newTokenInsert.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, newTokenInsert.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, newTokenInsert.range.tokensToPredict); @@ -205,7 +207,7 @@ describe('determineSuggestionRange', () => { const fixture = buildQuickBrownFixture(); const charReplace = fixture.variations.charReplace; - const analysis = determineSuggestionRange(fixture.baseTokenization, charReplace.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, charReplace.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, charReplace.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, charReplace.range.tokensToPredict); @@ -215,7 +217,7 @@ describe('determineSuggestionRange', () => { const fixture = buildQuickBrownFixture(); const del5Insert5 = fixture.variations.del5Insert5; - const analysis = determineSuggestionRange(fixture.baseTokenization, del5Insert5.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, del5Insert5.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, del5Insert5.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, del5Insert5.range.tokensToPredict); @@ -225,7 +227,7 @@ describe('determineSuggestionRange', () => { const fixture = buildQuickBrownFixture(); const eraseToken = fixture.variations.eraseToken; - const analysis = determineSuggestionRange(fixture.baseTokenization, eraseToken.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, eraseToken.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, eraseToken.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, eraseToken.range.tokensToPredict); @@ -235,7 +237,7 @@ describe('determineSuggestionRange', () => { const fixture = buildQuickBrownFixture(); const deleteToBound = fixture.variations.deleteToBound; - const analysis = determineSuggestionRange(fixture.baseTokenization, deleteToBound.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, deleteToBound.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, deleteToBound.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, deleteToBound.range.tokensToPredict); @@ -255,7 +257,7 @@ describe('determineSuggestionRange', () => { null ) - const analysis = determineSuggestionRange(originalQuickBrownTokenization, foxVsAlligatorTokenization); + const analysis = determineSuggestionRange(originalQuickBrownTokenization.tokens, foxVsAlligatorTokenization.tokens, tokenEquality); assert.sameOrderedMembers( analysis.tokensToRemove, @@ -279,7 +281,7 @@ describe('determineSuggestionRange', () => { null ) - const analysis = determineSuggestionRange(originalQuickBrownTokenization, dogsAndCatTokenization); + const analysis = determineSuggestionRange(originalQuickBrownTokenization.tokens, dogsAndCatTokenization.tokens, tokenEquality); assert.sameOrderedMembers( analysis.tokensToRemove, From beb2a553b1f07a43d66e394cc079c2c40a428c76 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 26 Mar 2026 09:10:56 -0500 Subject: [PATCH 26/65] change(web): simplify model.predict() calls Rather than copying over part of the existing context just to delete it, we can simplify prediction calls by just pre-deleting the current token, then applying any relevant deleteLeft transform component afterward to resulting predictions. Build-bot: skip build:web Test-bot: skip --- .../worker-thread/src/main/predict-helpers.ts | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index b9d8919b9a3..9b54159bfe2 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -356,6 +356,7 @@ export function determineContextTransition( return transition; } +// TODO: Remove this and its associated unit tests! /** * Determines where the context for prediction-generation should be rooted and how * much of the context it should replace. @@ -514,33 +515,37 @@ export function buildAndMapPredictions( ): CorrectionPredictionTuple[] { const model = transition.final.model; - // No matter the prediction, once we know the root of the prediction, we'll - // always 'replace' the same amount of text. We can handle this before the - // big 'prediction root' loop. - const { predictionContext, deleteLeft } = determineSuggestionAlignment(transition, tokenization, model); + const applicationTarget = transition.base.displayTokenization; + const { tokensToRemove, tokensToPredict } = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); - let correction = match.matchString; - let rootCost = match.totalCost; + const deleteLeft = tokensToPredict.length > 1 ? 0 : tokensToRemove.reduce((prev, curr) => prev + curr.searchModule.codepointLength, 0); + + // Exists to be extended by the 'correctionTransfrom' below. + const emptyContext: Context = { + left: '', + startOfBuffer: false, + endOfBuffer: false + }; // Replace the existing context with the correction. const correctionTransform: Transform = { - insert: correction, // insert correction string - deleteLeft: deleteLeft, + insert: match.matchString, // insert correction string + deleteLeft: 0, id: transition.transitionId // The correction should always be based on the most recent external transform/transcription ID. } + const rootCost = match.totalCost; const predictionRoot = { sample: correctionTransform, p: Math.exp(-rootCost * costFactor) }; - // Worth considering: extend Traversal to allow direct prediction lookups? - // let traversal = match.finalTraversal; // ... - let predictions = predictFromCorrections(model, [predictionRoot], predictionContext); + let predictions = predictFromCorrections(model, [predictionRoot], emptyContext); predictions.forEach((entry) => { entry.preservationTransform = tokenization.taillessTrueKeystroke; // // Will need an extra lookup layer if the suggestion is generated from within a cluster. // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization); + entry.prediction.sample.transform.deleteLeft = deleteLeft; }); return predictions; @@ -1118,13 +1123,10 @@ export function finalizeSuggestions( // // Note: may need adjustment if/when supporting phrase-level correction. if(tuple.preservationTransform) { - const presDL = tuple.preservationTransform.deleteLeft; - const mergedTransform = models.buildMergedTransform(tuple.preservationTransform, prediction.sample.transform); - // Any preserved delete-left is applied early because it directly affects the suggestion - // root; we need to remove that preserved delete-left here. - if(presDL > 0) { - mergedTransform.deleteLeft -= presDL; - } + const mergedTransform = { + ...models.buildMergedTransform(tuple.preservationTransform, {...prediction.sample.transform, deleteLeft: 0}), + deleteLeft: prediction.sample.transform.deleteLeft + }; // Temporarily and locally drops 'readonly' semantics so that we can reassign the transform. // See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers From 4c929e1408a21b87b002d423afc82e862460c518 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 27 Apr 2026 16:38:40 -0500 Subject: [PATCH 27/65] change(web): remove determineSuggestionAlignment method in favor of determineSuggestionRange --- .../worker-thread/src/main/predict-helpers.ts | 70 --------------- .../determine-suggestion-alignment.tests.ts | 90 ------------------- 2 files changed, 160 deletions(-) delete mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 9b54159bfe2..b8bf4b1570c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -356,76 +356,6 @@ export function determineContextTransition( return transition; } -// TODO: Remove this and its associated unit tests! -/** - * Determines where the context for prediction-generation should be rooted and how - * much of the context it should replace. - * @param transition - * @param lexicalModel - * @returns - */ -export function determineSuggestionAlignment( - transition: ContextTransition, - tokenization: ContextTokenization, - lexicalModel: LexicalModel -): { - /** - * The context to use directly for generating predictions from the model. - */ - predictionContext: Context, - /** - * The total number of characters to delete for generated suggestions - * in order to replace the prediction root token entirely. - */ - deleteLeft: number -} { - const transitionEdits = tokenization.transitionEdits; - const context = transition.base.context; - const postContext = transition.final.context; - const inputTransform = transition.inputDistribution[0].sample; - let deleteLeft: number; - - // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. - const wordbreak = determineModelWordbreaker(lexicalModel); - - // Is the token under construction newly-constructed / is there no pre-existing root? - if(tokenization.taillessTrueKeystroke && transitionEdits?.addedNewTokens) { - return { - // If the new token is due to whitespace or due to a different input type - // that would likely imply a tokenization boundary, infer 'new word' mode. - // Apply any part of the context change that is not considered to be up - // for correction. - predictionContext: models.applyTransform(tokenization.taillessTrueKeystroke, context), - // As the word/token being corrected/predicted didn't originally exist, - // there's no part of it to 'replace'. (Suggestions are applied to the - // pre-transform state.) - deleteLeft: 0 - }; - // If the tokenized context length is shorter... sounds like a backspace (or similar). - } else if (transitionEdits?.removedOldTokens) { - /* Ooh, we've dropped context here. Almost certainly from a backspace or - * similar effect. Even if we drop multiple tokens... well, we know exactly - * how many chars were actually deleted - `inputTransform.deleteLeft`. Since - * we replace a word being corrected/predicted, we take length of the - * remaining context's tail token in addition to however far was deleted to - * reach that state. - */ - deleteLeft = KMWString.length(wordbreak(postContext)) + inputTransform.deleteLeft; - } else { - // Suggestions are applied to the pre-input context, so get the token's original length. - // We're on the same token, so just delete its text for the replacement op. - deleteLeft = KMWString.length(wordbreak(context)); - } - - // Did the wordbreaker (or similar) append a blank token before the caret? If so, - // preserve that by preventing corrections from triggering left-deletion. - if(tokenization.tail.isEmptyToken) { - deleteLeft = 0; - } - - return { predictionContext: context, deleteLeft }; -} - /** * Given two ContextTokenizations related by context transition, this function * determines the tail-end range of the tokenization affected by the transition. diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts deleted file mode 100644 index be2d1177571..00000000000 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { assert } from 'chai'; - -import { LexicalModelTypes } from '@keymanapp/common-types'; -import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; -import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; - -import { ContextState, ContextTransition, determineSuggestionAlignment, models } from "@keymanapp/lm-worker/test-index"; - -import CasingFunction = LexicalModelTypes.CasingFunction; -import Context = LexicalModelTypes.Context; -import TrieModel = models.TrieModel; - -const plainApplyCasing: CasingFunction = function(caseToApply, text) { - switch(caseToApply) { - case 'lower': - return text.toLowerCase(); - case 'upper': - return text.toUpperCase(); - case 'initial': - return plainApplyCasing('upper', text.charAt(0)) . concat(text.substring(1)); - default: - return text; - } -}; - -const plainCasedModel = new TrieModel( - jsonFixture('models/tries/english-1000'), { - languageUsesCasing: true, - applyCasing: plainApplyCasing, - wordBreaker: defaultBreaker, - searchTermToKey: function(text: string) { - // We're dealing with very simple English text; no need to normalize or remove diacritics here. - return plainApplyCasing('lower', text); - } - } -); - -describe('determineSuggestionAlignment', () => { - it('handles standard cases well - same token, no preservationTransforms', () => { - const context: Context = { - left: 'this is techn', - startOfBuffer: true, - endOfBuffer: true - }; - const baseState = new ContextState(context, plainCasedModel); - - const transition = new ContextTransition(baseState, 0); - transition.finalize(transition.base, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); - - // transition, model - const results = determineSuggestionAlignment(transition, transition.final.displayTokenization, plainCasedModel); - - assert.deepEqual(results.predictionContext, context); - assert.equal(results.deleteLeft, "techn".length); - }); - - it('handles extension of prior token after backspace', () => { - const context: Context = { - left: 'this is tech ', - startOfBuffer: true, - endOfBuffer: true - }; - const baseState = new ContextState(context, plainCasedModel); - - const transition = baseState.analyzeTransition(context, [{sample: { insert: '', deleteLeft: 1 }, p: 1}]) - - // transition, model - const results = determineSuggestionAlignment(transition, transition.final.displayTokenization, plainCasedModel); - - assert.deepEqual(results.predictionContext, context); - assert.equal(results.deleteLeft, "tech".length + 1 /* for the deleted whitespace */); - }); - - it('handles extension of prior token after complex input with delete-left', () => { - const context: Context = { - left: 'this is tech ', - startOfBuffer: true, - endOfBuffer: true - }; - const baseState = new ContextState(context, plainCasedModel); - - const transition = baseState.analyzeTransition(context, [{sample: { insert: 'n', deleteLeft: 1 }, p: 1}]) - - // transition, model - const results = determineSuggestionAlignment(transition, transition.final.displayTokenization, plainCasedModel); - - assert.deepEqual(results.predictionContext, context); - assert.equal(results.deleteLeft, "techn".length + 1 /* for the deleted whitespace */); - }); -}); \ No newline at end of file From cdfc2db7b9dca9bef0e846b5880e731c72693372 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 18 May 2026 11:47:43 -0500 Subject: [PATCH 28/65] change(web): predict from per-token correction sequence --- .../worker-thread/src/main/predict-helpers.ts | 132 +++++++++++++----- .../predict-from-corrections.tests.ts | 101 ++------------ .../worker-custom-punctuation.tests.ts | 18 +++ 3 files changed, 126 insertions(+), 125 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index b8bf4b1570c..bd06a89e708 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -12,7 +12,7 @@ import { ContextState, determineContextSlideTransform } from './correction/conte import { ContextTransition } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; -import { getBestTokenMatches } from './correction/distance-modeler.js'; +import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js'; import { TokenResultMapping } from './correction/token-result-mapping.js'; import CasingForm = LexicalModelTypes.CasingForm; @@ -245,11 +245,24 @@ export async function correctAndEnumerateWithoutTraversals( }); } - // Remove `null` entries. - predictionRoots = predictionRoots.filter(tuple => !!tuple); + const wordbreak = determineModelWordbreaker(lexicalModel); + // Remove `null` entries, then determine suggestions. + predictionRoots.forEach((pr) => { + const postContext = models.applyTransform(pr.sample, context); + const tailTokenText = wordbreak(postContext); + const rootContext = models.applyTransform({insert: '', deleteLeft: KMWString.length(tailTokenText)}, postContext); + + const results = predictFromCorrectionSequence(lexicalModel, [{ + sample: { + insert: tailTokenText, + deleteLeft: 0, + id: pr.sample.id + }, + p: pr.p + }], rootContext); + results.forEach((r) => rawPredictions.push(r)); + }) - // Running in bulk over all suggestions, duplicate entries may be possible. - rawPredictions = predictFromCorrections(lexicalModel, predictionRoots, context); if(allowSpace) { rawPredictions.forEach((entry) => entry.preservationTransform = inputTransform); } @@ -449,13 +462,7 @@ export function buildAndMapPredictions( const { tokensToRemove, tokensToPredict } = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); const deleteLeft = tokensToPredict.length > 1 ? 0 : tokensToRemove.reduce((prev, curr) => prev + curr.searchModule.codepointLength, 0); - - // Exists to be extended by the 'correctionTransfrom' below. - const emptyContext: Context = { - left: '', - startOfBuffer: false, - endOfBuffer: false - }; + const rootContext = models.applyTransform({insert: '', deleteLeft}, transition.base.context); // Replace the existing context with the correction. const correctionTransform: Transform = { @@ -470,7 +477,7 @@ export function buildAndMapPredictions( p: Math.exp(-rootCost * costFactor) }; - let predictions = predictFromCorrections(model, [predictionRoot], emptyContext); + const predictions = predictFromCorrectionSequence(model, [predictionRoot], rootContext); predictions.forEach((entry) => { entry.preservationTransform = tokenization.taillessTrueKeystroke; // // Will need an extra lookup layer if the suggestion is generated from within a cluster. @@ -664,43 +671,94 @@ export function shouldStopSearchingEarly( * @param context * @returns */ -export function predictFromCorrections( +export function predictFromCorrectionSequence( lexicalModel: LexicalModel, corrections: ProbabilityMass[], context: Context ): CorrectionPredictionTuple[] { - let returnedPredictions: CorrectionPredictionTuple[] = []; - const wordbreak = determineModelWordbreaker(lexicalModel); + let predictionPrefixSequence: ProbabilityMass[] = []; + let tailPredictions: ProbabilityMass[]; - for(let correction of corrections) { - let predictions = lexicalModel.predict(correction.sample, context); + let currentContext = context; + let successfulPredictions = 0; - const { sample: correctionTransform, p: correctionProb } = correction; - const correctionRoot = wordbreak(models.applyTransform(correction.sample, context)); + for(let i = 0; i < corrections.length; i++) { + const correction = corrections[i].sample; + + // Step 2: predict based on the final token. + const predictions = lexicalModel.predict(correction, currentContext); + + // Failsafe: if there are no matching predictions, create a fake prediction + // matching the original text. + if(predictions.length != 0) { + successfulPredictions++; + } else { + predictions.push({ + sample: { + transform: correction, + displayAs: correction.insert + }, + // It's not found in the lexicon, so we'll take a low probability for it. + // + // Edit penalties will be applied via the correction component separately later on. + p: -Math.exp(EDIT_DISTANCE_COST_SCALE) + }); + } - let predictionSet = predictions.map((pair: ProbabilityMass) => { - // Let's not rely on the model to copy transform IDs. - // Only bother is there IS an ID to copy. - if(correctionTransform.id !== undefined) { - pair.sample.transformId = correctionTransform.id; + if(i == corrections.length - 1) { + tailPredictions = predictions; + } else { + let bestMatch = predictions.find((p) => KMWString.length(p.sample.transform.insert) == KMWString.length(correction.insert)); + if(!bestMatch) { + bestMatch = predictions[0]; } - let tuple: CorrectionPredictionTuple = { - prediction: pair, - correction: { - sample: correctionRoot, - p: correctionProb + predictionPrefixSequence = predictionPrefixSequence.concat(bestMatch); + } + + // Or maybe per prediction, in some manner? + currentContext = models.applyTransform(correction, currentContext); + } + + if(!successfulPredictions) { + return []; + } + + const predictions: CorrectionPredictionTuple[] = tailPredictions.map((p) => { + // Concat corrections + predictions for their components. + const predictionSequence = [...predictionPrefixSequence, p]; + const fullPrediction: ProbabilityMass = predictionSequence.reduce((prev, curr) => { + return { + sample: { + transform: models.buildMergedTransform(prev.sample.transform, curr.sample.transform), + displayAs: prev.sample.displayAs + curr.sample.displayAs }, - totalProb: pair.p * correctionProb, - matchLevel: SuggestionSimilarity.none + p: prev.p * curr.p }; - return tuple; - }); + }, {sample: {transform: {insert: '', deleteLeft: 0}, displayAs: ''}, p: 1}); - returnedPredictions = returnedPredictions.concat(predictionSet); - } + const fullCorrection: ProbabilityMass = corrections.reduce((prev, curr) => { + return { + sample: prev.sample + curr.sample.insert, + p: prev.p * curr.p + } + }, {sample: '', p: 1}) + + const transformId = p.sample.transform.id; + if(transformId) { + fullPrediction.sample.transform.id = transformId; + fullPrediction.sample.transformId = transformId; + } - return returnedPredictions; + return { + prediction: fullPrediction, + correction: fullCorrection, + totalProb: fullPrediction.p * fullCorrection.p, + matchLevel: SuggestionSimilarity.none, + }; + }); + + return predictions; } /** diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts index 2351ff99320..35f69975be4 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts @@ -4,7 +4,7 @@ import { assert } from 'chai'; import { deepCopy } from "keyman/common/web-utils"; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { models, predictFromCorrections, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; +import { models, predictFromCorrectionSequence, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; import CasingFunction = LexicalModelTypes.CasingFunction; import Context = LexicalModelTypes.Context; @@ -71,10 +71,10 @@ const DUMMY_MODEL_CONFIG = { languageUsesCasing: true }; -describe('predictFromCorrections', () => { +describe('predictFromCorrectionSequence', () => { it('handles a single correction prefixing multiple entries - no transform ID', () => { const context: Context = { - left: 'It', + left: '', right: '', startOfBuffer: true, endOfBuffer: true @@ -82,7 +82,7 @@ describe('predictFromCorrections', () => { const correctionDistribution: Distribution = [{ sample: { - insert: 's', + insert: 'Its', deleteLeft: 0 }, p: 0.6 @@ -112,12 +112,15 @@ describe('predictFromCorrections', () => { futureSuggestions: [ dummied_suggestions ] }); - const predictions = predictFromCorrections(model, correctionDistribution, context); + const predictions = predictFromCorrectionSequence(model, correctionDistribution, context); predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); predictions.sort(tupleDisplayOrderSort); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((s) => { + delete s.p; + return s; + })); assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); @@ -125,7 +128,7 @@ describe('predictFromCorrections', () => { it('handles a single correction prefixing multiple entries - with transform ID', () => { const context: Context = { - left: 'It', + left: '', right: '', startOfBuffer: true, endOfBuffer: true @@ -133,7 +136,7 @@ describe('predictFromCorrections', () => { const correctionDistribution: Distribution = [{ sample: { - insert: 's', + insert: 'Its', deleteLeft: 0, id: 314159 }, @@ -164,7 +167,7 @@ describe('predictFromCorrections', () => { futureSuggestions: [ dummied_suggestions ] }); - const predictions = predictFromCorrections(model, correctionDistribution, context); + const predictions = predictFromCorrectionSequence(model, correctionDistribution, context); predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); predictions.sort(tupleDisplayOrderSort); @@ -173,89 +176,11 @@ describe('predictFromCorrections', () => { assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((entry) => { entry = deepCopy(entry); entry.transformId = 314159; + delete entry.p; return entry; })); assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); }); - - it('handles multiple corrections at once', () => { - const context: Context = { - left: 'It', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - // Note: each correction is used in order in a separate model.predict call. - /** @type {Distribution} */ - const correctionDistribution: Distribution = [{ - // postContext: is - sample: { - insert: 's', - deleteLeft: 1 - }, - p: 0.4 - }, { - // postContext: its - sample: { - insert: 's', - deleteLeft: 0 - }, - p: 0.6 - } - ]; - - const dummied_suggestions: Outcome[][] = [ - // postContext: is - [{ - transform: { - insert: "is", - deleteLeft: 2 - }, - displayAs: "is", - p: 0.4 - }, { - transform: { - insert: "isn't", - deleteLeft: 2 - }, - displayAs: "isn't", - p: 0.2 - }], - // postContext: its - [{ - transform: { - insert: "it's", - deleteLeft: 2 - }, - displayAs: "it's", - p: 0.18 - }, { - transform: { - insert: "its", - deleteLeft: 2 - }, - displayAs: "its", - p: 0.02 - }] - ]; - - const model = new DummyModel({ - ...DUMMY_MODEL_CONFIG, - futureSuggestions: dummied_suggestions - }); - - const predictions = predictFromCorrections(model, correctionDistribution, context); - predictions.sort(tupleDisplayOrderSort); - - assert.sameOrderedMembers(predictions.map((entry) => entry.prediction.sample.displayAs), ["is", "it's", "isn't", "its"]); - assert.sameDeepMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.flatMap((entry) => entry)); - - assert.approximately(predictions[0].totalProb, 0.4 * 0.4, 0.00001); - assert.approximately(predictions[1].totalProb, 0.18 * 0.6, 0.00001); - assert.approximately(predictions[2].totalProb, 0.4 * 0.2, 0.00001); - assert.approximately(predictions[3].totalProb, 0.02 * 0.6, 0.00001); - }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts index 9b9ab2c3121..cd4dbd106e0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts @@ -81,6 +81,24 @@ describe('Custom Punctuation', function () { open: "'", close: "'" } + }, + // Some of the suggestions above actually wordbreak differently from + // what might be expected. So, we override the wordbreaker to ensure + // the tests run smoothly. + wordbreaker: (text) => { + const textLen = text.length; + if(text.charAt(textLen - 1) == "แš€") { + return [ + {text: text.substring(0, 1), start: 0, end: 1, length: 1}, + {text: text.substring(1, textLen-2), start: 1, end: textLen-1, length: textLen-2}, + {text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1} + ]; + } else { + return [ + {text: text.substring(0, 1), start: 0, end: 1, length: 1}, + {text: text.substring(1), start: 1, end: textLen, length: textLen-1} + ]; + } } }); From 9336ea2388dfe86fcfb66434d540bda44a0487ad Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 18 May 2026 13:12:49 -0500 Subject: [PATCH 29/65] feat(web): add new unit tests for generation of suggestions from multi-token corrections --- .../worker-thread/src/main/predict-helpers.ts | 18 +- .../predict-from-correction-sequence.tests.ts | 535 ++++++++++++++++++ .../predict-from-corrections.tests.ts | 186 ------ 3 files changed, 545 insertions(+), 194 deletions(-) create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts delete mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index bd06a89e708..36622c61c22 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -663,23 +663,25 @@ export function shouldStopSearchingEarly( * Given a generated set of corrections from the correction-search process, this * function searches the lexical model for valid predictions rooted on each. * - * While doing so, it also associates each prediction with metadata used for - * to "rank" and select the best predictions once the search is complete. This - * is performed at later stages. + * While doing so, it also associates each prediction with metadata used for to + * "rank" and select the best predictions once the search is complete. This is + * performed at later stages. * @param lexicalModel - * @param corrections - * @param context + * @param corrections Each `correction` should insert a full token's text to be + * appended to the context resulting from all preceding corrections. + * @param rootContext This context should represent all portions of the post-context + * not represented by the entries of the `corrections` array. * @returns */ export function predictFromCorrectionSequence( lexicalModel: LexicalModel, corrections: ProbabilityMass[], - context: Context + rootContext: Context ): CorrectionPredictionTuple[] { let predictionPrefixSequence: ProbabilityMass[] = []; let tailPredictions: ProbabilityMass[]; - let currentContext = context; + let currentContext = rootContext; let successfulPredictions = 0; for(let i = 0; i < corrections.length; i++) { @@ -701,7 +703,7 @@ export function predictFromCorrectionSequence( // It's not found in the lexicon, so we'll take a low probability for it. // // Edit penalties will be applied via the correction component separately later on. - p: -Math.exp(EDIT_DISTANCE_COST_SCALE) + p: Math.exp(-EDIT_DISTANCE_COST_SCALE) }); } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts new file mode 100644 index 00000000000..acbefe1e827 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -0,0 +1,535 @@ + +import { assert } from 'chai'; + +import { deepCopy } from "keyman/common/web-utils"; +import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { EDIT_DISTANCE_COST_SCALE, models, predictFromCorrectionSequence, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; + +import CasingFunction = LexicalModelTypes.CasingFunction; +import Context = LexicalModelTypes.Context; +import Distribution = LexicalModelTypes.Distribution; +import DummyModel = models.DummyModel; +import Outcome = LexicalModelTypes.Outcome; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Suggestion = LexicalModelTypes.Suggestion; +import Transform = LexicalModelTypes.Transform; + +// See: developer/src/kmc-model/model-defaults.ts, defaultApplyCasing +const applyCasing: CasingFunction = (casing, text) => { + switch(casing) { + case 'lower': + return text.toLowerCase(); + case 'upper': + return text.toUpperCase(); + case 'initial': + var headCode = text.charCodeAt(0); + // The length of the first code unit, as measured in code points. + var headUnitLength = 1; + + // Is the first character a high surrogate, indicating possible use of UTF-16 + // surrogate pairs? Also, is the string long enough for there to BE a pair? + if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) { + // It's possible, so now we check for low surrogates. + var lowSurrogateCode = text.charCodeAt(1); + + if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) { + // We have a surrogate pair; this pair is the 'first' character. + headUnitLength++; + } + } + + // Capitalizes the first code unit of the string, leaving the rest intact. + return text.substring(0, headUnitLength).toUpperCase() // head - uppercased + .concat(text.substring(headUnitLength)); // tail - lowercased + } +}; + +/** @type { import("#./models/dummy-model.js").DummyOptions } */ +const DUMMY_MODEL_CONFIG = { + punctuation: { + quotesForKeepSuggestion: { + open: '<', + close: '>' + }, + insertAfterWord: '\u00a0' // non-breaking space + }, + applyCasing: applyCasing, + searchTermToKey: (wordform: string) => { + // See: developer/src/kmc-model/model-defaults.ts, defaultCasedSearchTermToKey + return applyCasing('lower', wordform) + .normalize('NFKD') + // Remove any combining diacritics (if input is in NFKD) + .replace(/[\u0300-\u036F]/g, '') + // Replace directional quotation marks with plain apostrophes + .replace(/[โ€˜โ€™]/g, "'") + // Also double-quote marks. + .replace(/[โ€œโ€]/g, '"') + // ** Difference from model-defaults here ** + // And finally, erase single-quotation marks. + .replace(/'/, ''); + }, + languageUsesCasing: true +}; + +describe('predictFromCorrectionSequence', () => { + describe('on a single correction', () => { + it('constructs suggestions matching multiple lexical entries directly - no transform ID', () => { + const context: Context = { + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionDistribution: Distribution = [{ + sample: { + insert: 'Its', + deleteLeft: 0 + }, + p: 0.6 + } + ]; + + const dummied_suggestions: Outcome[] = [ + { + transform: { + insert: "it's", + deleteLeft: 2 + }, + displayAs: "it's", + p: 0.18 + }, { + transform: { + insert: "its", + deleteLeft: 2 + }, + displayAs: "its", + p: 0.02 + } + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: [ dummied_suggestions ] + }); + + const predictions = predictFromCorrectionSequence(model, correctionDistribution, context); + predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); + predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); + predictions.sort(tupleDisplayOrderSort); + + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((s) => { + delete s.p; + return s; + })); + + assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); + assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); + }); + + it('constructs suggestions matching multiple lexical entries directly - with transform ID', () => { + const context: Context = { + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionDistribution: Distribution = [{ + sample: { + insert: 'Its', + deleteLeft: 0, + id: 314159 + }, + p: 0.6 + } + ]; + + const dummied_suggestions: Outcome[] = [ + { + transform: { + insert: "it's", + deleteLeft: 2 + }, + displayAs: "it's", + p: 0.18 + }, { + transform: { + insert: "its", + deleteLeft: 2 + }, + displayAs: "its", + p: 0.02 + } + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: [ dummied_suggestions ] + }); + + const predictions = predictFromCorrectionSequence(model, correctionDistribution, context); + predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); + predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); + predictions.sort(tupleDisplayOrderSort); + + assert.sameOrderedMembers(predictions.map((entry) => entry.prediction.sample.displayAs), ["it's", "its"]); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((entry) => { + entry = deepCopy(entry); + entry.transformId = 314159; + delete entry.p; + return entry; + })); + + assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); + assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); + }); + }); + + describe('on a sequence of corrections', () => { + it('returns results even if some correction tokens lack predictions', () => { + const context: Context = { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionSequence: Distribution = [ + { + sample: { + insert: 'golden', + deleteLeft: 0 + }, + p: 0.1 + }, { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: 0.2 + }, { + sample: { + insert: 'app', + deleteLeft: 0 + }, + p: 0.2 + } + ]; + + const dummied_suggestion_sequences: Outcome[][] = [ + [ + { + transform: { + insert: "golden", + deleteLeft: 0 + }, + displayAs: "golden", + p: 0.1 + } + ], + [], + [ + { + transform: { + insert: "apple", + deleteLeft: 0 + }, + displayAs: "apple", + p: 0.5 + } + ] + ]; + + const expected_prediction: ProbabilityMass = { + sample: { + transform: { + insert: 'golden apple', + deleteLeft: 0 + }, + displayAs: 'golden apple' + }, p: dummied_suggestion_sequences.map((dist) => { + return dist[0] + }).reduce((accum, curr) => { + return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) + }, 1) + } + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: dummied_suggestion_sequences + }); + + const predictions = predictFromCorrectionSequence(model, correctionSequence, context); + predictions.forEach((entry) => assert.equal(entry.correction.sample, 'golden app')); + predictions.forEach((entry) => assert.equal(entry.correction.p, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + predictions.sort(tupleDisplayOrderSort); + + assert.equal(predictions[0].prediction.sample.transform.insert, 'golden apple'); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), [expected_prediction.sample]); + + assert.approximately(predictions[0].prediction.p, expected_prediction.p, 0.00001); + }); + + it('returns no results if all correction tokens lack predictions', () => { + const context: Context = { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionSequence: Distribution = [ + { + sample: { + insert: 'golden', + deleteLeft: 0 + }, + p: 0.1 + }, { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: 0.2 + }, { + sample: { + insert: 'app', + deleteLeft: 0 + }, + p: 0.2 + } + ]; + + const dummied_suggestion_sequences: Outcome[][] = [ + [], + [], + [] + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: dummied_suggestion_sequences + }); + + const predictions = predictFromCorrectionSequence(model, correctionSequence, context); + assert.deepEqual(predictions, []); + }); + + it('uses only the best suggestion for non-final corrected tokens', () => { + const context: Context = { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionSequence: Distribution = [ + { + sample: { + insert: 'g', + deleteLeft: 0 + }, + p: 0.1 + }, { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: 0.2 + }, { + sample: { + insert: 'app', + deleteLeft: 0 + }, + p: 0.2 + } + ]; + + const dummied_suggestion_sequences: Outcome[][] = [ + [ + { + transform: { + insert: "golden", + deleteLeft: 0 + }, + displayAs: "golden", + p: 0.2 + }, { + transform: { + insert: "green", + deleteLeft: 0 + }, + displayAs: "green", + p: 0.15 + }, { + transform: { + insert: "gray", + deleteLeft: 0 + }, + displayAs: "gray", + p: 0.1 + } + ], + [], + [ + { + transform: { + insert: "apple", + deleteLeft: 0 + }, + displayAs: "apple", + p: 0.5 + } + ] + ]; + + const expected_prediction: ProbabilityMass = { + sample: { + transform: { + insert: 'golden apple', + deleteLeft: 0 + }, + displayAs: 'golden apple' + }, p: dummied_suggestion_sequences.map((dist) => { + return dist[0] + }).reduce((accum, curr) => { + return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) + }, 1) + } + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: dummied_suggestion_sequences + }); + + const predictions = predictFromCorrectionSequence(model, correctionSequence, context); + // There should be no variations with 'green' or 'gray' apples. + assert.equal(predictions.length, 1); + + predictions.forEach((entry) => assert.equal(entry.correction.sample, 'g app')); + predictions.forEach((entry) => assert.equal(entry.correction.p, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + predictions.sort(tupleDisplayOrderSort); + + assert.equal(predictions[0].prediction.sample.transform.insert, 'golden apple'); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), [expected_prediction.sample]); + + assert.approximately(predictions[0].prediction.p, expected_prediction.p, 0.00001); + }); + + it('uses all suggestions generated from context-final correction-tokens', () => { + const context: Context = { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionSequence: Distribution = [ + { + sample: { + insert: 'golden', + deleteLeft: 0 + }, + p: 0.1 + }, { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: 0.2 + }, { + sample: { + insert: 'app', + deleteLeft: 0 + }, + p: 0.2 + } + ]; + + const dummied_suggestion_sequences: Outcome[][] = [ + [ + { + transform: { + insert: "golden", + deleteLeft: 0 + }, + displayAs: "golden", + p: 0.1 + } + ], + [], + [ + { + transform: { + insert: "apple", + deleteLeft: 0 + }, + displayAs: "apple", + p: 0.5 + }, { + transform: { + insert: "application", + deleteLeft: 0 + }, + displayAs: "application", + p: 0.11 + }, { + transform: { + insert: "appetizer", + deleteLeft: 0 + }, + displayAs: "appetizer", + p: 0.1 + } + ] + ]; + + const tailIndex = dummied_suggestion_sequences.length - 1; + const expected_predictions: ProbabilityMass[] = dummied_suggestion_sequences[tailIndex].map((p) => { + const expectedText = `golden ${p.transform.insert}`; + + return { + sample: { + transform: { + insert: expectedText, + deleteLeft: 0 + }, + displayAs: expectedText + }, p: dummied_suggestion_sequences.map((dist) => { + return dist[0] + }).reduce((accum, curr, index) => { + if(tailIndex == index) { + return accum * p.p; + } else { + return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) + } + }, 1) + }; + }); + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: dummied_suggestion_sequences + }); + + const predictions = predictFromCorrectionSequence(model, correctionSequence, context); + assert.equal(predictions.length, dummied_suggestion_sequences[dummied_suggestion_sequences.length - 1].length); + + predictions.forEach((entry) => assert.equal(entry.correction.sample, 'golden app')); + predictions.forEach((entry) => assert.equal(entry.correction.p, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + predictions.sort(tupleDisplayOrderSort); + + assert.sameOrderedMembers( + predictions.map((t) => t.prediction.sample.transform.insert), + ['golden apple', 'golden application', 'golden appetizer'] + ); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), expected_predictions.map((p => p.sample))); + + for(let i = 0; i < predictions.length; i++) { + assert.approximately(predictions[i].prediction.p, expected_predictions[i].p, 0.00001, `Expected probabilty mismatch at index ${i}`); + } + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts deleted file mode 100644 index 35f69975be4..00000000000 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts +++ /dev/null @@ -1,186 +0,0 @@ - -import { assert } from 'chai'; - -import { deepCopy } from "keyman/common/web-utils"; -import { LexicalModelTypes } from '@keymanapp/common-types'; - -import { models, predictFromCorrectionSequence, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; - -import CasingFunction = LexicalModelTypes.CasingFunction; -import Context = LexicalModelTypes.Context; -import Distribution = LexicalModelTypes.Distribution; -import DummyModel = models.DummyModel; -import Outcome = LexicalModelTypes.Outcome; -import Suggestion = LexicalModelTypes.Suggestion; -import Transform = LexicalModelTypes.Transform; - -// See: developer/src/kmc-model/model-defaults.ts, defaultApplyCasing -const applyCasing: CasingFunction = (casing, text) => { - switch(casing) { - case 'lower': - return text.toLowerCase(); - case 'upper': - return text.toUpperCase(); - case 'initial': - var headCode = text.charCodeAt(0); - // The length of the first code unit, as measured in code points. - var headUnitLength = 1; - - // Is the first character a high surrogate, indicating possible use of UTF-16 - // surrogate pairs? Also, is the string long enough for there to BE a pair? - if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) { - // It's possible, so now we check for low surrogates. - var lowSurrogateCode = text.charCodeAt(1); - - if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) { - // We have a surrogate pair; this pair is the 'first' character. - headUnitLength++; - } - } - - // Capitalizes the first code unit of the string, leaving the rest intact. - return text.substring(0, headUnitLength).toUpperCase() // head - uppercased - .concat(text.substring(headUnitLength)); // tail - lowercased - } -}; - -/** @type { import("#./models/dummy-model.js").DummyOptions } */ -const DUMMY_MODEL_CONFIG = { - punctuation: { - quotesForKeepSuggestion: { - open: '<', - close: '>' - }, - insertAfterWord: '\u00a0' // non-breaking space - }, - applyCasing: applyCasing, - searchTermToKey: (wordform: string) => { - // See: developer/src/kmc-model/model-defaults.ts, defaultCasedSearchTermToKey - return applyCasing('lower', wordform) - .normalize('NFKD') - // Remove any combining diacritics (if input is in NFKD) - .replace(/[\u0300-\u036F]/g, '') - // Replace directional quotation marks with plain apostrophes - .replace(/[โ€˜โ€™]/g, "'") - // Also double-quote marks. - .replace(/[โ€œโ€]/g, '"') - // ** Difference from model-defaults here ** - // And finally, erase single-quotation marks. - .replace(/'/, ''); - }, - languageUsesCasing: true -}; - -describe('predictFromCorrectionSequence', () => { - it('handles a single correction prefixing multiple entries - no transform ID', () => { - const context: Context = { - left: '', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const correctionDistribution: Distribution = [{ - sample: { - insert: 'Its', - deleteLeft: 0 - }, - p: 0.6 - } - ]; - - const dummied_suggestions: Outcome[] = [ - { - transform: { - insert: "it's", - deleteLeft: 2 - }, - displayAs: "it's", - p: 0.18 - }, { - transform: { - insert: "its", - deleteLeft: 2 - }, - displayAs: "its", - p: 0.02 - } - ]; - - const model = new DummyModel({ - ...DUMMY_MODEL_CONFIG, - futureSuggestions: [ dummied_suggestions ] - }); - - const predictions = predictFromCorrectionSequence(model, correctionDistribution, context); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); - predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); - predictions.sort(tupleDisplayOrderSort); - - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((s) => { - delete s.p; - return s; - })); - - assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); - assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); - }); - - it('handles a single correction prefixing multiple entries - with transform ID', () => { - const context: Context = { - left: '', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const correctionDistribution: Distribution = [{ - sample: { - insert: 'Its', - deleteLeft: 0, - id: 314159 - }, - p: 0.6 - } - ]; - - const dummied_suggestions: Outcome[] = [ - { - transform: { - insert: "it's", - deleteLeft: 2 - }, - displayAs: "it's", - p: 0.18 - }, { - transform: { - insert: "its", - deleteLeft: 2 - }, - displayAs: "its", - p: 0.02 - } - ]; - - const model = new DummyModel({ - ...DUMMY_MODEL_CONFIG, - futureSuggestions: [ dummied_suggestions ] - }); - - const predictions = predictFromCorrectionSequence(model, correctionDistribution, context); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); - predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); - predictions.sort(tupleDisplayOrderSort); - - assert.sameOrderedMembers(predictions.map((entry) => entry.prediction.sample.displayAs), ["it's", "its"]); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((entry) => { - entry = deepCopy(entry); - entry.transformId = 314159; - delete entry.p; - return entry; - })); - - assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); - assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); - }); -}); \ No newline at end of file From ec458c525cb46f9f8557c45794fa5c6e6aa73e3c Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 28 May 2026 15:35:55 -0500 Subject: [PATCH 30/65] fix(web): propagate transitionID correctly from corrections to predictions --- .../worker-thread/src/main/predict-helpers.ts | 23 ++++----- .../predict-from-correction-sequence.tests.ts | 47 +++++++++++++------ 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 36622c61c22..0f39a464545 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -259,7 +259,7 @@ export async function correctAndEnumerateWithoutTraversals( id: pr.sample.id }, p: pr.p - }], rootContext); + }], rootContext, pr.sample.id); results.forEach((r) => rawPredictions.push(r)); }) @@ -459,9 +459,8 @@ export function buildAndMapPredictions( const model = transition.final.model; const applicationTarget = transition.base.displayTokenization; - const { tokensToRemove, tokensToPredict } = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); + const { deleteLeft } = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); - const deleteLeft = tokensToPredict.length > 1 ? 0 : tokensToRemove.reduce((prev, curr) => prev + curr.searchModule.codepointLength, 0); const rootContext = models.applyTransform({insert: '', deleteLeft}, transition.base.context); // Replace the existing context with the correction. @@ -477,7 +476,7 @@ export function buildAndMapPredictions( p: Math.exp(-rootCost * costFactor) }; - const predictions = predictFromCorrectionSequence(model, [predictionRoot], rootContext); + const predictions = predictFromCorrectionSequence(model, [predictionRoot], rootContext, transition.transitionId); predictions.forEach((entry) => { entry.preservationTransform = tokenization.taillessTrueKeystroke; // // Will need an extra lookup layer if the suggestion is generated from within a cluster. @@ -669,14 +668,17 @@ export function shouldStopSearchingEarly( * @param lexicalModel * @param corrections Each `correction` should insert a full token's text to be * appended to the context resulting from all preceding corrections. - * @param rootContext This context should represent all portions of the post-context - * not represented by the entries of the `corrections` array. + * @param rootContext This context should represent all portions of the + * post-context not represented by the entries of the `corrections` array. + * @param transitionId Indicates the unique ID of the transition that triggered + * prediction generation. * @returns */ export function predictFromCorrectionSequence( lexicalModel: LexicalModel, corrections: ProbabilityMass[], - rootContext: Context + rootContext: Context, + transitionId: number ): CorrectionPredictionTuple[] { let predictionPrefixSequence: ProbabilityMass[] = []; let tailPredictions: ProbabilityMass[]; @@ -746,10 +748,9 @@ export function predictFromCorrectionSequence( } }, {sample: '', p: 1}) - const transformId = p.sample.transform.id; - if(transformId) { - fullPrediction.sample.transform.id = transformId; - fullPrediction.sample.transformId = transformId; + if(transitionId) { + fullPrediction.sample.transform.id = transitionId; + fullPrediction.sample.transformId = transitionId; } return { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts index acbefe1e827..06fdb220b99 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -114,13 +114,16 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: [ dummied_suggestions ] }); - const predictions = predictFromCorrectionSequence(model, correctionDistribution, context); + const transitionID = 12345; + const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); predictions.sort(tupleDisplayOrderSort); assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((s) => { delete s.p; + s.transformId = transitionID; + s.transform.id = transitionID; return s; })); @@ -136,11 +139,12 @@ describe('predictFromCorrectionSequence', () => { endOfBuffer: true }; + const transitionID = 314159; const correctionDistribution: Distribution = [{ sample: { insert: 'Its', deleteLeft: 0, - id: 314159 + id: transitionID }, p: 0.6 } @@ -169,7 +173,7 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: [ dummied_suggestions ] }); - const predictions = predictFromCorrectionSequence(model, correctionDistribution, context); + const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); predictions.sort(tupleDisplayOrderSort); @@ -177,13 +181,15 @@ describe('predictFromCorrectionSequence', () => { assert.sameOrderedMembers(predictions.map((entry) => entry.prediction.sample.displayAs), ["it's", "its"]); assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((entry) => { entry = deepCopy(entry); - entry.transformId = 314159; + entry.transformId = transitionID; + entry.transform.id = transitionID; delete entry.p; return entry; })); assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); + predictions.forEach((prediction) => assert.equal(prediction.prediction.sample.transformId, transitionID)); }); }); @@ -242,13 +248,16 @@ describe('predictFromCorrectionSequence', () => { ] ]; + const transitionID = 101; const expected_prediction: ProbabilityMass = { sample: { transform: { insert: 'golden apple', - deleteLeft: 0 + deleteLeft: 0, + id: transitionID }, - displayAs: 'golden apple' + displayAs: 'golden apple', + transformId: transitionID }, p: dummied_suggestion_sequences.map((dist) => { return dist[0] }).reduce((accum, curr) => { @@ -261,7 +270,7 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: dummied_suggestion_sequences }); - const predictions = predictFromCorrectionSequence(model, correctionSequence, context); + const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID); predictions.forEach((entry) => assert.equal(entry.correction.sample, 'golden app')); predictions.forEach((entry) => assert.equal(entry.correction.p, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); predictions.sort(tupleDisplayOrderSort); @@ -270,6 +279,7 @@ describe('predictFromCorrectionSequence', () => { assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), [expected_prediction.sample]); assert.approximately(predictions[0].prediction.p, expected_prediction.p, 0.00001); + assert.equal(predictions[0].prediction.sample.transformId, transitionID); }); it('returns no results if all correction tokens lack predictions', () => { @@ -313,7 +323,7 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: dummied_suggestion_sequences }); - const predictions = predictFromCorrectionSequence(model, correctionSequence, context); + const predictions = predictFromCorrectionSequence(model, correctionSequence, context, 3); assert.deepEqual(predictions, []); }); @@ -385,13 +395,16 @@ describe('predictFromCorrectionSequence', () => { ] ]; + const transitionID = 42; const expected_prediction: ProbabilityMass = { sample: { transform: { insert: 'golden apple', - deleteLeft: 0 + deleteLeft: 0, + id: transitionID }, - displayAs: 'golden apple' + displayAs: 'golden apple', + transformId: 42 }, p: dummied_suggestion_sequences.map((dist) => { return dist[0] }).reduce((accum, curr) => { @@ -404,7 +417,7 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: dummied_suggestion_sequences }); - const predictions = predictFromCorrectionSequence(model, correctionSequence, context); + const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID); // There should be no variations with 'green' or 'gray' apples. assert.equal(predictions.length, 1); @@ -416,6 +429,7 @@ describe('predictFromCorrectionSequence', () => { assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), [expected_prediction.sample]); assert.approximately(predictions[0].prediction.p, expected_prediction.p, 0.00001); + assert.equal(predictions[0].prediction.sample.transformId, transitionID); }); it('uses all suggestions generated from context-final correction-tokens', () => { @@ -487,6 +501,8 @@ describe('predictFromCorrectionSequence', () => { ]; const tailIndex = dummied_suggestion_sequences.length - 1; + + const transitionID = 13; const expected_predictions: ProbabilityMass[] = dummied_suggestion_sequences[tailIndex].map((p) => { const expectedText = `golden ${p.transform.insert}`; @@ -494,9 +510,11 @@ describe('predictFromCorrectionSequence', () => { sample: { transform: { insert: expectedText, - deleteLeft: 0 + deleteLeft: 0, + id: transitionID }, - displayAs: expectedText + displayAs: expectedText, + transformId: transitionID }, p: dummied_suggestion_sequences.map((dist) => { return dist[0] }).reduce((accum, curr, index) => { @@ -514,7 +532,7 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: dummied_suggestion_sequences }); - const predictions = predictFromCorrectionSequence(model, correctionSequence, context); + const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID); assert.equal(predictions.length, dummied_suggestion_sequences[dummied_suggestion_sequences.length - 1].length); predictions.forEach((entry) => assert.equal(entry.correction.sample, 'golden app')); @@ -529,6 +547,7 @@ describe('predictFromCorrectionSequence', () => { for(let i = 0; i < predictions.length; i++) { assert.approximately(predictions[i].prediction.p, expected_predictions[i].p, 0.00001, `Expected probabilty mismatch at index ${i}`); + assert.equal(predictions[i].prediction.sample.transformId, transitionID); } }); }); From 84d00f7f29aa1a9f9c1fadc2ed0b088d5f59ecc5 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 2 Jun 2026 09:17:34 -0500 Subject: [PATCH 31/65] fix(web): transitionId undefined check, extra unit test --- .../worker-thread/src/main/predict-helpers.ts | 2 +- .../predict-from-correction-sequence.tests.ts | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 0f39a464545..3263cad93b0 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -748,7 +748,7 @@ export function predictFromCorrectionSequence( } }, {sample: '', p: 1}) - if(transitionId) { + if(transitionId !== undefined) { fullPrediction.sample.transform.id = transitionId; fullPrediction.sample.transformId = transitionId; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts index 06fdb220b99..64cea5b2300 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -191,6 +191,53 @@ describe('predictFromCorrectionSequence', () => { assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); predictions.forEach((prediction) => assert.equal(prediction.prediction.sample.transformId, transitionID)); }); + + it('constructs suggestions without input (as if after a context reset)', () => { + const context: Context = { + left: 'appl', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionDistribution: Distribution = [{ + sample: { + insert: 'appl', + deleteLeft: 4 + }, + p: 1 + } + ]; + + const dummied_suggestions: Outcome[] = [ + { + transform: { + insert: "apple", + deleteLeft: 4 + }, + displayAs: "apple", + p: 0.5 + } + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: [ dummied_suggestions ] + }); + + const transitionID = 12345; + const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); + predictions.forEach((entry) => assert.equal(entry.correction.sample, 'appl')); + predictions.forEach((entry) => assert.equal(entry.correction.p, 1)); + predictions.sort(tupleDisplayOrderSort); + + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((s) => { + delete s.p; + s.transformId = transitionID; + s.transform.id = transitionID; + return s; + })); + }); }); describe('on a sequence of corrections', () => { From fe6acfdfefc3592c223feb11d9d621cf6222365a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 19 May 2026 13:09:12 -0500 Subject: [PATCH 32/65] refactor(web): expose suggestion-root parameters for use in unit tests Build-bot: skip build:web Test-bot: skip --- .../worker-thread/src/main/predict-helpers.ts | 85 +++-- ...ine-tokenized-correction-sequence.tests.ts | 361 ++++++++++++++++++ 2 files changed, 423 insertions(+), 23 deletions(-) create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 3263cad93b0..414c7ff8ebe 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -13,7 +13,6 @@ import { ContextTransition } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js'; -import { TokenResultMapping } from './correction/token-result-mapping.js'; import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; @@ -26,6 +25,7 @@ import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; import SuggestionTag = LexicalModelTypes.SuggestionTag; import Transform = LexicalModelTypes.Transform; +import { TokenResult } from './correction/tokenization-corrector.js'; /* * The functions in this file exist to provide unit-testable stateless components for the @@ -440,24 +440,55 @@ export function determineSuggestionRange( } } +/** + * Specifies the core, preprocessed data necessary for generating predictions, + * regardless of model type. + */ +export interface PredictionParameters { + /** + * The portion of context that should remain unchanged by generated suggestions + */ + rootContext: Context, + + /** + * A tokenization of the corrected part of the context, usable to generate + * suggestions. + * + * Note that each correction will be applied iteratively to the rootContext. + * That is, when suggesting based on the correction at index 1, the + * "unchanged" (root) context used for that suggestion will include the + * changes from the entry at index 0 (or possibly, a suggestion derived from it). + */ + tokenizedCorrection: ProbabilityMass[], + + /** + * A closure to be applied to the generated suggestion's metadata. + * @param entry + * @returns + */ + applyInPost: (entry: CorrectionPredictionTuple) => void +} + /** * This function takes in metadata about generated corrections (for models that - * implement Traversals) and uses that to construct predictions based upon those - * corrections. - * @param transition Context-transition data underlying the tokenization that led to the correction - * @param tokenization The tokenization from which the correction was generated. - * @param match The generated correction itself - the correction string and its cost - * @param costFactor A multiplicative factor used to adjust the cost when building prediction probabilities. + * implement Traversals) and uses that to produce the corresponding parameters + * to use for generating suggestions. + * @param transition Context-transition data underlying the tokenization that + * led to the correction + * @param tokenization The tokenization from which the correction was + * generated. + * @param match The generated correction itself - the correction string + * and its cost + * @param costFactor A multiplicative factor used to adjust the cost when + * building prediction probabilities. * @returns */ -export function buildAndMapPredictions( +export function determineTokenizedCorrectionSequence( transition: ContextTransition, tokenization: ContextTokenization, - match: Readonly, + match: Readonly, costFactor: number -): CorrectionPredictionTuple[] { - const model = transition.final.model; - +): PredictionParameters { const applicationTarget = transition.base.displayTokenization; const { deleteLeft } = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); @@ -467,7 +498,12 @@ export function buildAndMapPredictions( const correctionTransform: Transform = { insert: match.matchString, // insert correction string deleteLeft: 0, - id: transition.transitionId // The correction should always be based on the most recent external transform/transcription ID. + } + + // The correction should always be based on the most recent external + // transform/transcription ID. + if(transition.transitionId !== undefined) { + correctionTransform.id = transition.transitionId; } const rootCost = match.totalCost; @@ -476,15 +512,16 @@ export function buildAndMapPredictions( p: Math.exp(-rootCost * costFactor) }; - const predictions = predictFromCorrectionSequence(model, [predictionRoot], rootContext, transition.transitionId); - predictions.forEach((entry) => { - entry.preservationTransform = tokenization.taillessTrueKeystroke; - // // Will need an extra lookup layer if the suggestion is generated from within a cluster. - // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization); - entry.prediction.sample.transform.deleteLeft = deleteLeft; - }); - - return predictions; + return { + rootContext, + tokenizedCorrection: [predictionRoot], + applyInPost: (entry: CorrectionPredictionTuple) => { + entry.preservationTransform = tokenization.taillessTrueKeystroke; + // // Will need an extra lookup layer if the suggestion is generated from within a cluster. + // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization); + entry.prediction.sample.transform.deleteLeft = deleteLeft; + } + }; } /** @@ -598,7 +635,9 @@ export async function correctAndEnumerate( */ const costFactor = (tokenization.tail.inputCount <= 1) ? ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT : 1; - const predictions = buildAndMapPredictions(transition, tokenization, match, costFactor); + const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, match, costFactor); + const predictions = predictFromCorrectionSequence(lexicalModel, predictionPrep.tokenizedCorrection, predictionPrep.rootContext, transition.transitionId); + predictions.forEach((p) => predictionPrep.applyInPost(p)); // Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions. if(predictions.length > 0 && bestCorrectionCost === undefined) { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts new file mode 100644 index 00000000000..756a09c6559 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -0,0 +1,361 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-05-19 + * + * This file tests the prediction helper-method responsible for preparing + * corrections for multi-token prediction for our standard models, all of which + * utilize LexiconTraversals and the context-tokenization-caching subsystem. + */ + +import { assert } from 'chai'; + +import { LexicalModelTypes } from "@keymanapp/common-types"; +import * as wordBreakers from '@keymanapp/models-wordbreakers'; +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { KMWString } from 'keyman/common/web-utils'; + +import { determineTokenizedCorrectionSequence, models, ContextState, ContextToken, ContextTokenization, CorrectionPredictionTuple } from "@keymanapp/lm-worker/test-index"; + +import Context = LexicalModelTypes.Context; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; +import TrieModel = models.TrieModel; + +const testModel = new TrieModel( + jsonFixture('models/tries/english-1000'), { + wordBreaker: wordBreakers.default, + } +); + +describe('determineTokenizedCorrectionSequence', () => { + it(`properly analyzes common-case token-extension - adding a letter to an existing word`, () => { + const context: Context = { + left: 'the quick brown f', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'o', + deleteLeft: 0 + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + const transition = state.analyzeTransition(context, [trueInput]); + + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, { + matchString: 'fo', + inputSamplingCost: -Math.log(trueInput.p), + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }, + 1 + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.deepEqual(results.tokenizedCorrection, [ + { + sample: { + insert: 'fo', + deleteLeft: 0 + }, + p: trueInput.p + } + ]); + }); + + it(`properly analyzes common-case whitespace - ending a token and adding a new one`, () => { + const context: Context = { + left: 'the quick brown', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + const transition = state.analyzeTransition(context, [trueInput]); + + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, { + matchString: ' ', + inputSamplingCost: -Math.log(trueInput.p), + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }, + 1 + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.deepEqual(results.tokenizedCorrection, [ + { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: trueInput.p + } + ]); + }); + + it(`properly analyzes common-case word-start - beginning a new token`, () => { + const context: Context = { + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + const transition = state.analyzeTransition(context, [trueInput]); + + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, { + matchString: 'f', + inputSamplingCost: -Math.log(trueInput.p), + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }, + 1 + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.deepEqual(results.tokenizedCorrection, [ + { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: trueInput.p + } + ]); + }); + + it(`properly analyzes post-merge case`, () => { + let context: Context = { + left: 'the quick brown fox ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 't', + deleteLeft: 0 + }, + p: .5 + }; + + const constructingState = new ContextState(context, testModel); + const tokens = constructingState.displayTokenization.tokens; + tokens.push(ContextToken.fromRawText(testModel, 'can')); + tokens.push(ContextToken.fromRawText(testModel, '\'')); + + context = models.applyTransform({insert: 'can\'', deleteLeft: 0}, context); + + const state = new ContextState(context, testModel, new ContextTokenization(tokens)); + const transition = state.analyzeTransition(context, [trueInput]); + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, { + matchString: 'can\'t', + inputSamplingCost: -Math.log(trueInput.p), + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }, + 1 + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown fox ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.deepEqual(results.tokenizedCorrection, [ + { + sample: { + insert: 'can\'t', + deleteLeft: 0 + }, + p: trueInput.p + } + ]); + }); + + // Will be handled far better after resolving multi-tokenization handling. + it.skip(`properly analyzes post-split case`, () => { + const context: Context = { + left: 'the quick brown fox can\'', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + assert.equal(state.displayTokenization.tail.exampleInput, 'can\''); + const transition = state.analyzeTransition(context, [trueInput]); + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, { + matchString: ' ', + inputSamplingCost: -Math.log(trueInput.p), + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }, + 1 + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown fox ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.deepEqual(results.tokenizedCorrection, [ + { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: trueInput.p + } + ]); + }); + + it(`properly analyzes conplex transition - multi-token replacement`, () => { + const context: Context = { + left: 'the quick brown f', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'fast red d', + deleteLeft: 'quick brown f'.length + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + const transition = state.analyzeTransition(context, [trueInput]); + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, { + matchString: 'd', + inputSamplingCost: -Math.log(trueInput.p), + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }, + 1 + ); + + // Large-scale deletions will receive enhanced handling soon. But, for now, it's + // deleted by the `preservationTransform`, not here. + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.deepEqual(results.tokenizedCorrection, [ + { + sample: { + insert: 'd', + deleteLeft: 0 + }, + p: trueInput.p + } + ]); + + const dummiedTuple: CorrectionPredictionTuple = { + prediction: { + sample: { + transform: { insert: 'dog', deleteLeft: 0 }, + displayAs: 'dog' + }, + p: .25 + }, + correction: { + sample: 'd', + p: trueInput.p + }, + totalProb: .25 * trueInput.p + }; + + results.applyInPost(dummiedTuple); + + assert.deepEqual(dummiedTuple.preservationTransform, { + insert: trueInput.sample.insert.substring(0, KMWString.length(trueInput.sample.insert) - 1), // remove the 'd'. + deleteLeft: trueInput.sample.deleteLeft - 1 + }); + }); +}); \ No newline at end of file From 329b51425a8bc9fbaea6f5acd1a54bd60ba5b9b3 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 11 Jun 2026 11:11:00 -0500 Subject: [PATCH 33/65] fix(web): filter out corrections to whitespace/backspace inputs from standard keys --- .../predictive-text/worker-thread/src/main/predict-helpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 3263cad93b0..c61f6346ca1 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -247,6 +247,7 @@ export async function correctAndEnumerateWithoutTraversals( const wordbreak = determineModelWordbreaker(lexicalModel); // Remove `null` entries, then determine suggestions. + predictionRoots = predictionRoots.filter(tuple => !!tuple); predictionRoots.forEach((pr) => { const postContext = models.applyTransform(pr.sample, context); const tailTokenText = wordbreak(postContext); From 8d52ba47a2f6c32a0a11f7eb34660d71ad4b2173 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 6 May 2026 13:21:08 -0500 Subject: [PATCH 34/65] change(web): simplify mapWhitespacedTokenization requirements To better handle inputs that shift the word-boundary in some custom models and models released before Keyman 14.0, this PR provides generalized re-use of the whitespace-based token-transition algorithm used for our most prominently-supported models. Build-bot: skip build:web Test-bot: skip --- .../main/correction/context-tokenization.ts | 337 ++++++++++-------- 1 file changed, 182 insertions(+), 155 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 72992f84d34..175c0eb202e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -10,7 +10,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { KMWString } from 'keyman/common/web-utils'; -import { ContextToken } from './context-token.js'; +import { ContextToken, ContextTokenLike } from './context-token.js'; import { TransformUtils } from '../transformUtils.js'; import { computeDistance, EditOperation, EditTuple } from './classical-calculation.js'; import { determineModelTokenizer } from '../model-helpers.js'; @@ -339,7 +339,7 @@ export class ContextTokenization { } /** - * Given the existing tokenization and an incoming input `Transform`, this + * Given this existing tokenization and an incoming input `Transform`, this * method precomputes how both the current, pre-application tokenization will * be altered and how the incoming Transform will be tokenized. * @@ -356,158 +356,7 @@ export class ContextTokenization { transform: Transform, edgeOptions?: EdgeWindowOptions ): TokenizationTransitionEdits { - // Step 4: now that our window's been properly updated, determine what the - // input's effects on the context is. - // - // Context does not slide within this function. - // - // Assumption: this alignment cannot fail; we KNOW there's a solid - // before-and-after relationship here, and we can base it on the results of - // a prior syncToSourceWindow call. - // - // We don't wish to do the full tokenization here - we only want to check - // over the last few tokens that might reasonably shift. We also want to - // batch effects. - - // Do not mutate the original transform; it can cause unexpected assertion - // effects in unit tests. - const edgeTransform = {...transform, deleteRight: transform.deleteRight || 0}; - const edgeWindow = buildEdgeWindow(this.tokens, edgeTransform, false, edgeOptions); - const { - retokenizationText, - editBoundary, - sliceIndex: edgeSliceIndex - } = edgeWindow; - // Prevent mutation of the original return property. - const stackedDeletes = edgeWindow.deleteLengths.slice(); - - const tokenize = determineModelTokenizer(lexicalModel); - const postTokenization = tokenize({left: retokenizationText + transform.insert, startOfBuffer: true, endOfBuffer: true}).left.map(t => t.text); - if(postTokenization.length == 0) { - postTokenization.push(''); - } - const { stackedInserts, firstInsertPostIndex } = traceInsertEdits(postTokenization, transform); - - // What does the edge's retokenization look like when we remove the inserted portions? - const retokenizedEdge = postTokenization.slice(0, firstInsertPostIndex); - const insertBoundaryToken = postTokenization[firstInsertPostIndex]; - - // Note: requires that helpers have not mutated `stackedInserts`. - const uninsertedBoundaryToken = KMWString.substring(insertBoundaryToken, 0, KMWString.lastIndexOf(insertBoundaryToken, stackedInserts[0])); - - // Do not preserve empty tokens here, even if tokenization normally would produce one. - // It's redundant and replaceable for tokenization batching efforts. - if(uninsertedBoundaryToken != '') { - retokenizedEdge.push(uninsertedBoundaryToken); - } - - // We've found the root token within the root context state to which deletes (and inserts) - // may be applied. - // We've also found the last post-application token to which transform changes contributed. - // How do these indices line up - we need to properly construct and index our transforms, - // but 'merge' and 'split' edits can mess up that indexing. - - const currentTokens = this.tokens; - const preTokenization = currentTokens - .slice(edgeSliceIndex, editBoundary.tokenIndex+1) - .map(t => t.exampleInput); - - // Determine the effects of splits & merges as applied to the original - // cached context state. - const { mergeOffset, splitOffset, editPath, merges, splits } = analyzePathMergesAndSplits( - preTokenization, - postTokenization.slice(0, firstInsertPostIndex+1) - ); - - /* - * Final steps: We can now safely index the transforms. Let's do it! - * 1. Determine the first index a Transform may align to - * 2. Build the transforms - * - * Notes: - * - text applied to the end of a 'merged' token at the tail: should have - * index 0, not -1. - * - pretokenization index will mismatch by -1: -SUM(merge size - 1) - * - Ex: can + ' + t => can't - * -1 0 0 - * - text applied to the end of a 'split' token at the tail: should also - * have index 0, not 1. - * - posttokenization index will mismatch by +1: SUM(split size - 1) - * - new token after 'split': index 1 - * - Ex: can' + ? => can + ' + ? - * 0 -1 0 1 - * - * The first transform applies at the end of the retokenized zone and its - * associated index. The question: were there deletes that occurred? - */ - - const lastEditedPreTokenIndex = editBoundary.tokenIndex - edgeSliceIndex; - let shiftDeletes = false; - // first popped entry == 0 - a delete no-op. - if(stackedDeletes[stackedDeletes.length - 1] == 0) { - // the boundary indices found by both methods above differ - if(lastEditedPreTokenIndex + mergeOffset != firstInsertPostIndex + splitOffset) { - shiftDeletes = true; - } - - // there are no inserts, so we don't affect the boundary token we landed on. - if(stackedDeletes.length > 1 && transform.insert == '') { - shiftDeletes = true; - } - } - - if(shiftDeletes) { - // Do not add a zero-length delete if we're not actually altering the - // corresponding token at all. - stackedDeletes.pop(); - } - - // The first delete always applies to index 0. If the built edge window - // omits a context-final empty-string, adjust the tokenization indices - // accordingly. - const tailIndex = 0 - (stackedDeletes.length - 1) + (editBoundary.omitsEmptyToken ? -1 : 0); - // Mutates stackedInserts, stackedDeletes. - const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length); - const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex); - - // If there's an empty transform in the 0 position and we already know we're - // dropping tokens - and only deleting - we're dropping an - // otherwise-untracked empty token - make sure it's included! - const droppedFinalTransform = baseRemovedTokenCount > 0 && transform.insert == '' && TransformUtils.isEmpty(transformMap.get(0)); - // Past that, if we have more delete entries than insert entries for our transforms, we - // dropped some tokens outright. - const removedTokenCount = baseRemovedTokenCount + (droppedFinalTransform ? 1 : 0); - - // Final step: check for any unexpected boundary shifts not mappable to 'merge' / 'split' - // and not caused by transforms. All transforms always apply in sequence at the end. - const unmappedEdits: EditTuple[] = []; - for(let i = 0; i < editPath.length - transformMap.size; i++) { - const op = editPath[i].op; - switch(op) { - case 'merge': - case 'split': - // already calculated - // can fall through to the `continue;` line. - case 'match': - continue; - default: - // Should only be substitutions here. - // We may wish to add extra analysis in the future when supporting - // prediction from multiple competing tokenizations. - unmappedEdits.push(editPath[i] as EditTuple); - } - } - - return { - alignment: { - edgeWindow: {...edgeWindow, retokenization: retokenizedEdge}, - merges, - splits, - unmappedEdits, - removedTokenCount - }, - tokenizedTransform: transformMap, - }; + return mapWhitespacedTokenization(this.tokens, lexicalModel, transform, edgeOptions); } /** @@ -768,6 +617,184 @@ interface RetokenizedEdgeWindow extends EdgeWindow { retokenization: string[]; } +/** + * Given an existing tokenization and an incoming input `Transform`, this + * method precomputes how both the current, pre-application tokenization will + * be altered and how the incoming Transform will be tokenized. + * + * This function is able to operate with a reduced interface, not requiring + * the full ContextToken/ContextState/etc subsystem and its related + * SearchQuotientNode requirements. + * + * Note that this method is designed for use with languages that employ + * classical space-based wordbreaking. Do not use it for languages that need + * dictionary-based wordbreaking support! + * @param tokens + * @param lexicalModel + * @param transform + * @param edgeOptions + * @returns + */ +export function mapWhitespacedTokenization( + tokens: ContextTokenLike[], + lexicalModel: LexicalModel, + transform: Transform, + edgeOptions?: EdgeWindowOptions +): TokenizationTransitionEdits { + // Step 4: now that our window's been properly updated, determine what the + // input's effects on the context is. + // + // Context does not slide within this function. + // + // Assumption: this alignment cannot fail; we KNOW there's a solid + // before-and-after relationship here, and we can base it on the results of + // a prior syncToSourceWindow call. + // + // We don't wish to do the full tokenization here - we only want to check + // over the last few tokens that might reasonably shift. We also want to + // batch effects. + + // Do not mutate the original transform; it can cause unexpected assertion + // effects in unit tests. + const edgeTransform = {...transform, deleteRight: transform.deleteRight || 0}; + const edgeWindow = buildEdgeWindow(tokens, edgeTransform, false, edgeOptions); + const { + retokenizationText, + editBoundary, + sliceIndex: edgeSliceIndex + } = edgeWindow; + // Prevent mutation of the original return property. + const stackedDeletes = edgeWindow.deleteLengths.slice(); + + const tokenize = determineModelTokenizer(lexicalModel); + const postTokenization = tokenize({left: retokenizationText + transform.insert, startOfBuffer: true, endOfBuffer: true}).left.map(t => t.text); + if(postTokenization.length == 0) { + postTokenization.push(''); + } + const { stackedInserts, firstInsertPostIndex } = traceInsertEdits(postTokenization, transform); + + // What does the edge's retokenization look like when we remove the inserted portions? + const retokenizedEdge = postTokenization.slice(0, firstInsertPostIndex); + const insertBoundaryToken = postTokenization[firstInsertPostIndex]; + + // Note: requires that helpers have not mutated `stackedInserts`. + const uninsertedBoundaryToken = KMWString.substring(insertBoundaryToken, 0, KMWString.lastIndexOf(insertBoundaryToken, stackedInserts[0])); + + // Do not preserve empty tokens here, even if tokenization normally would produce one. + // It's redundant and replaceable for tokenization batching efforts. + if(uninsertedBoundaryToken != '') { + retokenizedEdge.push(uninsertedBoundaryToken); + } + + // We've found the root token within the root context state to which deletes (and inserts) + // may be applied. + // We've also found the last post-application token to which transform changes contributed. + // How do these indices line up - we need to properly construct and index our transforms, + // but 'merge' and 'split' edits can mess up that indexing. + + const currentTokens = tokens; + const preTokenization = currentTokens + .slice(edgeSliceIndex, editBoundary.tokenIndex+1) + .map(t => t.exampleInput); + + // Determine the effects of splits & merges as applied to the original + // cached context state. + const { mergeOffset, splitOffset, editPath, merges, splits } = analyzePathMergesAndSplits( + preTokenization, + postTokenization.slice(0, firstInsertPostIndex+1) + ); + + /* + * Final steps: We can now safely index the transforms. Let's do it! + * 1. Determine the first index a Transform may align to + * 2. Build the transforms + * + * Notes: + * - text applied to the end of a 'merged' token at the tail: should have + * index 0, not -1. + * - pretokenization index will mismatch by -1: -SUM(merge size - 1) + * - Ex: can + ' + t => can't + * -1 0 0 + * - text applied to the end of a 'split' token at the tail: should also + * have index 0, not 1. + * - posttokenization index will mismatch by +1: SUM(split size - 1) + * - new token after 'split': index 1 + * - Ex: can' + ? => can + ' + ? + * 0 -1 0 1 + * + * The first transform applies at the end of the retokenized zone and its + * associated index. The question: were there deletes that occurred? + */ + + const lastEditedPreTokenIndex = editBoundary.tokenIndex - edgeSliceIndex; + let shiftDeletes = false; + // first popped entry == 0 - a delete no-op. + if(stackedDeletes[stackedDeletes.length - 1] == 0) { + // the boundary indices found by both methods above differ + if(lastEditedPreTokenIndex + mergeOffset != firstInsertPostIndex + splitOffset) { + shiftDeletes = true; + } + + // there are no inserts, so we don't affect the boundary token we landed on. + if(stackedDeletes.length > 1 && transform.insert == '') { + shiftDeletes = true; + } + } + + if(shiftDeletes) { + // Do not add a zero-length delete if we're not actually altering the + // corresponding token at all. + stackedDeletes.pop(); + } + + // The first delete always applies to index 0. If the built edge window + // omits a context-final empty-string, adjust the tokenization indices + // accordingly. + const tailIndex = 0 - (stackedDeletes.length - 1) + (editBoundary.omitsEmptyToken ? -1 : 0); + // Mutates stackedInserts, stackedDeletes. + const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length); + const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex); + + // If there's an empty transform in the 0 position and we already know we're + // dropping tokens - and only deleting - we're dropping an + // otherwise-untracked empty token - make sure it's included! + const droppedFinalTransform = baseRemovedTokenCount > 0 && transform.insert == '' && TransformUtils.isEmpty(transformMap.get(0)); + // Past that, if we have more delete entries than insert entries for our transforms, we + // dropped some tokens outright. + const removedTokenCount = baseRemovedTokenCount + (droppedFinalTransform ? 1 : 0); + + // Final step: check for any unexpected boundary shifts not mappable to 'merge' / 'split' + // and not caused by transforms. All transforms always apply in sequence at the end. + const unmappedEdits: EditTuple[] = []; + for(let i = 0; i < editPath.length - transformMap.size; i++) { + const op = editPath[i].op; + switch(op) { + case 'merge': + case 'split': + // already calculated + // can fall through to the `continue;` line. + case 'match': + continue; + default: + // Should only be substitutions here. + // We may wish to add extra analysis in the future when supporting + // prediction from multiple competing tokenizations. + unmappedEdits.push(editPath[i] as EditTuple); + } + } + + return { + alignment: { + edgeWindow: {...edgeWindow, retokenization: retokenizedEdge}, + merges, + splits, + unmappedEdits, + removedTokenCount + }, + tokenizedTransform: transformMap, + }; +} + /** * Constructs a window on one side of the represented context that is aligned to * existing tokenization. @@ -782,7 +809,7 @@ interface RetokenizedEdgeWindow extends EdgeWindow { * @returns */ export function buildEdgeWindow( - currentTokens: ContextToken[], + currentTokens: ContextTokenLike[], // Requires deleteRight be explicitly set. transform: Transform & { deleteRight: number }, applyAtFront: boolean, From e76b338853f3823a56b2a0e9202d3d424d5ef7ea Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 28 May 2026 16:16:32 -0500 Subject: [PATCH 35/65] refactor(web): define common buildCorrectionSequence method used for all model types Build-bot: skip build:web Test-bot: skip --- .../worker-thread/src/main/predict-helpers.ts | 51 ++++++++++++------- ...ine-tokenized-correction-sequence.tests.ts | 2 +- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 98b3086709f..b2bd4297fa6 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -13,6 +13,7 @@ import { ContextTransition } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js'; +import { TokenResult } from './correction/tokenization-corrector.js'; import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; @@ -25,7 +26,6 @@ import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; import SuggestionTag = LexicalModelTypes.SuggestionTag; import Transform = LexicalModelTypes.Transform; -import { TokenResult } from './correction/tokenization-corrector.js'; /* * The functions in this file exist to provide unit-testable stateless components for the @@ -470,6 +470,34 @@ export interface PredictionParameters { applyInPost: (entry: CorrectionPredictionTuple) => void } +export function buildCorrectionSequence( + transitionEffects: ReturnType, + context: Context, + match: Readonly, + costFactor: number +) { + const { deleteLeft } = transitionEffects; + + const rootContext = models.applyTransform({insert: '', deleteLeft}, context); + + // Replace the existing context with the correction. + const correctionTransform: Transform = { + insert: match.matchString, // insert correction string + deleteLeft: 0, + } + + const rootCost = match.totalCost; + const predictionRoot = { + sample: correctionTransform, + p: Math.exp(-rootCost * costFactor) + }; + + return { + rootContext, + tokenizedCorrection: [predictionRoot] + }; +} + /** * This function takes in metadata about generated corrections (for models that * implement Traversals) and uses that to produce the corresponding parameters @@ -491,31 +519,20 @@ export function determineTokenizedCorrectionSequence( costFactor: number ): PredictionParameters { const applicationTarget = transition.base.displayTokenization; - const { deleteLeft } = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); - - const rootContext = models.applyTransform({insert: '', deleteLeft}, transition.base.context); + const transitionParams = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); - // Replace the existing context with the correction. - const correctionTransform: Transform = { - insert: match.matchString, // insert correction string - deleteLeft: 0, - } + const suggestionParams = buildCorrectionSequence(transitionParams, transition.base.context, match, costFactor); // The correction should always be based on the most recent external // transform/transcription ID. if(transition.transitionId !== undefined) { - correctionTransform.id = transition.transitionId; + suggestionParams.tokenizedCorrection.forEach((t) => t.sample.id = transition.transitionId); } - const rootCost = match.totalCost; - const predictionRoot = { - sample: correctionTransform, - p: Math.exp(-rootCost * costFactor) - }; + const { deleteLeft } = transitionParams; return { - rootContext, - tokenizedCorrection: [predictionRoot], + ...suggestionParams, applyInPost: (entry: CorrectionPredictionTuple) => { entry.preservationTransform = tokenization.taillessTrueKeystroke; // // Will need an extra lookup layer if the suggestion is generated from within a cluster. diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts index 756a09c6559..ddf838d4586 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -286,7 +286,7 @@ describe('determineTokenizedCorrectionSequence', () => { ]); }); - it(`properly analyzes conplex transition - multi-token replacement`, () => { + it(`properly analyzes complex transition - multi-token replacement`, () => { const context: Context = { left: 'the quick brown f', right: '', From 4cfac7b19a45dce2fc97ba27b66c514ee8d76cb8 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 6 May 2026 15:07:43 -0500 Subject: [PATCH 36/65] change(web): rework traversalless prediction, add mild whitespace-correction Build-bot: skip build:web Test-bot: skip --- .../templates/src/tokenization.ts | 4 + .../worker-thread/src/main/model-helpers.ts | 3 +- .../worker-thread/src/main/predict-helpers.ts | 135 ++++++++---------- 3 files changed, 66 insertions(+), 76 deletions(-) diff --git a/web/src/engine/predictive-text/templates/src/tokenization.ts b/web/src/engine/predictive-text/templates/src/tokenization.ts index fd8ed28d5ca..47ef927fa5b 100644 --- a/web/src/engine/predictive-text/templates/src/tokenization.ts +++ b/web/src/engine/predictive-text/templates/src/tokenization.ts @@ -95,6 +95,10 @@ export function tokenize( currentIndex = nextIndex; } + if(tokenization.left.length == 0) { + tokenization.left.push({text: '', isWhitespace: false}); + } + // New step 2: handle any rejoins needed. // Handle any desired special handling for directly-pre-caret scenarios - where for this diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts index 7008a74be55..03078bcfde5 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts @@ -71,7 +71,8 @@ export function determineModelTokenizer(model: LexicalModel) { if(model.wordbreaker) { return models.tokenize(model.wordbreaker, context); } else { - return null; + // Not ideal for pre-14.0 models, but it'll do for now. + return models.tokenize(wordBreakers.default, context); } } } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index b2bd4297fa6..9272cf3c47b 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -6,7 +6,7 @@ import { searchForProperty, WordBreakProperty } from '@keymanapp/models-wordbrea import { TransformUtils } from './transformUtils.js'; import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTokenLike } from './correction/context-token.js'; -import { ContextTokenization } from './correction/context-tokenization.js'; +import { ContextTokenization, mapWhitespacedTokenization } from './correction/context-tokenization.js'; import { ContextTracker } from './correction/context-tracker.js'; import { ContextState, determineContextSlideTransform } from './correction/context-state.js'; import { ContextTransition } from './correction/context-transition.js'; @@ -189,89 +189,72 @@ export function tupleDisplayOrderSort(a: CorrectionPredictionTuple, b: Correctio return b.totalProb - a.totalProb; } -export async function correctAndEnumerateWithoutTraversals( +export function correctAndEnumerateWithoutTraversals( lexicalModel: LexicalModel, - transformDistribution: Distribution, + corrections: Distribution, context: Context -): Promise<{ - /** - * For models that support correction-search caching, this provides the - * cached object corresponding to this method's operation. - * - * Otherwise, is `null`. - */ - postContextState?: ContextState; - - /** - * The suggestions generated based on the user's input state. - */ - rawPredictions: CorrectionPredictionTuple[]; - - /** - * The id of a prior ContextTransition event that triggered a Suggestion found - * at the end of the Context. Will be undefined if no edits have occurred - * since the Suggestion was applied. - */ - revertableTransitionId?: number -}> { - const inputTransform = transformDistribution[0].sample; - let rawPredictions: CorrectionPredictionTuple[] = []; +): CorrectionPredictionTuple[] { + let returnedPredictions: CorrectionPredictionTuple[] = []; + + const tokenizer = determineModelTokenizer(lexicalModel); + const tokenization = tokenizer(context); // issue at present if no tokens exist! + + for(let correction of corrections) { + // Step 1: determine tokenization effects. We can't use the + // ContextTokenization pattern due to the model's lack of LexiconTraversal + // support, though. + const transformId = correction.sample.id; + + const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample); + const tokenizedCorrection = tokenizationMapping.tokenizedTransform; + // tokenizationMapping.alignment.edgeWindow.editBoundary. + const tokenizedCorrectionEntries = [...tokenizedCorrection.values()]; + const deleteLeft = tokenizedCorrectionEntries.reduce((total, curr) => total + curr.deleteLeft, 0); + + const rootContext = models.applyTransform({insert: '', deleteLeft}, context); + + // Start: determine the proper root for the first tokenized correction component. + const edgeWindow = tokenizationMapping.alignment.edgeWindow; + const nonEmptyBoundaryRoot = edgeWindow.editBoundary.text == '' ? tokenization.left[edgeWindow.editBoundary.tokenIndex-1]?.text ?? '' : edgeWindow.editBoundary.text; + const appliedBoundaryTail = nonEmptyBoundaryRoot + tokenizedCorrectionEntries[0].insert; + + // If the previous text can be tokenized into multiple tokens, the first correction's text should stand alone. + // If not, the correction root should incorporate `nonEmptyBoundaryRoot` as a prefix. + const tailTransform = tokenizedCorrectionEntries[0]; + if(tokenizer({left: appliedBoundaryTail, startOfBuffer: false, endOfBuffer: false}).left.length > 1) { + tokenizedCorrectionEntries[0] = { + ...tailTransform, + deleteLeft: 0 + } + } else { + tokenizedCorrectionEntries[0] = { + insert: nonEmptyBoundaryRoot + tailTransform.insert, + deleteLeft: 0 + } + } - let predictionRoots: ProbabilityMass[]; + const preservationTransform = tokenizedCorrectionEntries.slice(0, -1).reduce((accum, curr) => { + return models.buildMergedTransform(accum, {...curr, deleteLeft: 0}); + }, { insert: '', deleteLeft, id: correction.sample.id}); - // Only allow new-word suggestions if space was the most likely keypress. - const allowSpace = TransformUtils.isWhitespace(inputTransform); - const allowBksp = TransformUtils.isBackspace(inputTransform); - // Generates raw prediction distributions for each valid input. Can only 'correct' - // against the final input. - // - // This is the old, 12.0-13.0 'correction' style. - if(allowSpace) { - // Detect start of new word; prevent whitespace loss here. - predictionRoots = [{sample: inputTransform, p: 1.0}]; - } else { - predictionRoots = transformDistribution.map((alt) => { - let transform = alt.sample; - - // Filter out special keys unless they're expected. - if(TransformUtils.isWhitespace(transform) && !allowSpace) { - return null; - } else if(TransformUtils.isBackspace(transform) && !allowBksp) { - return null; + const predictions = predictFromCorrectionSequence(lexicalModel, tokenizedCorrectionEntries.map((e) =>{ + return { + sample: e, + p: correction.p } + }), rootContext, transformId); - return alt; + predictions.forEach((p) => { + p.preservationTransform = preservationTransform; + if(transformId) { + p.prediction.sample.transformId = transformId; + } + returnedPredictions.push(p); }); } - const wordbreak = determineModelWordbreaker(lexicalModel); - // Remove `null` entries, then determine suggestions. - predictionRoots = predictionRoots.filter(tuple => !!tuple); - predictionRoots.forEach((pr) => { - const postContext = models.applyTransform(pr.sample, context); - const tailTokenText = wordbreak(postContext); - const rootContext = models.applyTransform({insert: '', deleteLeft: KMWString.length(tailTokenText)}, postContext); - - const results = predictFromCorrectionSequence(lexicalModel, [{ - sample: { - insert: tailTokenText, - deleteLeft: 0, - id: pr.sample.id - }, - p: pr.p - }], rootContext, pr.sample.id); - results.forEach((r) => rawPredictions.push(r)); - }) - - if(allowSpace) { - rawPredictions.forEach((entry) => entry.preservationTransform = inputTransform); - } - - return { - postContextState: null, - rawPredictions: rawPredictions - }; + return returnedPredictions; } /** @@ -585,7 +568,9 @@ export async function correctAndEnumerate( // It's mostly here to support models compiled before Keyman 14.0, which was // when the `LexiconTraversal` pattern was established. if(!contextTracker) { - return correctAndEnumerateWithoutTraversals(lexicalModel, transformDistribution, context); + return { + rawPredictions: correctAndEnumerateWithoutTraversals(lexicalModel, transformDistribution, context) + }; } // 'else': the current, 14.0+ pattern, which is able to leverage From ef8593b049d6a0bc8afbede39bffb65fbb012f66 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 8 May 2026 15:45:19 -0500 Subject: [PATCH 37/65] fix(web): adjust tokenization unit test expectations to match --- .../templates/tokenization.tests.ts | 8 ++--- .../worker-custom-punctuation.tests.ts | 33 +++++++++++++------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts index 3bc636c4128..0aa4f9551ed 100644 --- a/web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts @@ -175,7 +175,7 @@ describe('Tokenization functions', function() { }); it('properly handles empty-context cases', function() { - // Wordbreaking on a empty space => no word. + // Wordbreaking on a empty space => no word, but empty initial token. let context = { left: '', startOfBuffer: true, right: '', endOfBuffer: true @@ -184,7 +184,7 @@ describe('Tokenization functions', function() { let tokenization = models.tokenize(wordBreakers.default, context); let expectedResult: models.Tokenization = { - left: [], + left: [{text: '', isWhitespace: false}], right: [], caretSplitsToken: false }; @@ -193,11 +193,11 @@ describe('Tokenization functions', function() { }); it('properly handles null context cases', function() { - // Wordbreaking on a empty space => no word. + // Wordbreaking on a empty space => no word, but empty initial token. let tokenization = models.tokenize(wordBreakers.default, null); let expectedResult: models.Tokenization = { - left: [], + left: [{text: '', isWhitespace: false}], right: [], caretSplitsToken: false }; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts index cd4dbd106e0..01f1a6ac96a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts @@ -87,17 +87,30 @@ describe('Custom Punctuation', function () { // the tests run smoothly. wordbreaker: (text) => { const textLen = text.length; - if(text.charAt(textLen - 1) == "แš€") { - return [ - {text: text.substring(0, 1), start: 0, end: 1, length: 1}, - {text: text.substring(1, textLen-2), start: 1, end: textLen-1, length: textLen-2}, - {text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1} - ]; + if(text.charAt(0) == "แš›") { // ensure the prior token component (the 'แš›') wordbreaks. + if(text.charAt(textLen - 1) == "แš€") { // ensure the insert-after component word-breaks. + return [ + {text: text.substring(0, 1), start: 0, end: 1, length: 1}, + {text: text.substring(1, textLen-2), start: 1, end: textLen-1, length: textLen-2}, + {text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1} + ]; + } else { + return [ + {text: text.substring(0, 1), start: 0, end: 1, length: 1}, + {text: text.substring(1), start: 1, end: textLen, length: textLen-1} + ]; + } } else { - return [ - {text: text.substring(0, 1), start: 0, end: 1, length: 1}, - {text: text.substring(1), start: 1, end: textLen, length: textLen-1} - ]; + if(text.charAt(textLen - 1) == "แš€") { + return [ + {text: text.substring(0, textLen-2), start: 0, end: textLen-1, length: textLen-1}, + {text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1} + ]; + } else { + return [ + {text: text, start: 0, end: textLen, length: textLen} + ]; + } } } }); From ba6536d4c7c38d0c55a1fbf0c1d76fb8b478c3d3 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 21 May 2026 15:22:05 -0500 Subject: [PATCH 38/65] refactor(web): DRY out & spin-off correction-sequence construction Aims: - improve consistency between Traversal-based models and legacy/custom models without Traversals - improve ability to unit-test construction of the correction-sequence directly, as a smaller, individual unit --- .../worker-thread/src/main/predict-helpers.ts | 84 +++++++++-------- ...ine-tokenized-correction-sequence.tests.ts | 2 +- ...raversalless-correction-sequences.tests.ts | 90 +++++++++++++++++++ 3 files changed, 132 insertions(+), 44 deletions(-) create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 9272cf3c47b..5c3f3109e8f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -189,72 +189,65 @@ export function tupleDisplayOrderSort(a: CorrectionPredictionTuple, b: Correctio return b.totalProb - a.totalProb; } -export function correctAndEnumerateWithoutTraversals( +export function determineTraversallessCorrectionSequences( lexicalModel: LexicalModel, corrections: Distribution, context: Context -): CorrectionPredictionTuple[] { - let returnedPredictions: CorrectionPredictionTuple[] = []; +): PredictionParameters[] { + let returnedPredictionData: PredictionParameters[] = []; const tokenizer = determineModelTokenizer(lexicalModel); + const wordbreak = determineModelWordbreaker(lexicalModel); + const tokenization = tokenizer(context); // issue at present if no tokens exist! + const tokenMapper = (t: models.Token) => { + return { + exampleInput: t.text + } as ContextTokenLike; + } for(let correction of corrections) { // Step 1: determine tokenization effects. We can't use the // ContextTokenization pattern due to the model's lack of LexiconTraversal // support, though. const transformId = correction.sample.id; + const postContext = models.applyTransform(correction.sample, context); + const postTokenization = tokenizer(postContext); + + const transitionEffects = determineSuggestionRange(tokenization.left.map(tokenMapper), postTokenization.left.map(tokenMapper), (a, b) => a.exampleInput == b.exampleInput); + const match: TokenResult = { + matchString: wordbreak(postContext), + inputSamplingCost: -Math.log(correction.p), + knownCost: 0, + totalCost: -Math.log(correction.p) + }; + + const suggestionParams = buildCorrectionSequence(transitionEffects, context, match, 1); const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample); const tokenizedCorrection = tokenizationMapping.tokenizedTransform; - // tokenizationMapping.alignment.edgeWindow.editBoundary. const tokenizedCorrectionEntries = [...tokenizedCorrection.values()]; - const deleteLeft = tokenizedCorrectionEntries.reduce((total, curr) => total + curr.deleteLeft, 0); - - const rootContext = models.applyTransform({insert: '', deleteLeft}, context); - - // Start: determine the proper root for the first tokenized correction component. - const edgeWindow = tokenizationMapping.alignment.edgeWindow; - const nonEmptyBoundaryRoot = edgeWindow.editBoundary.text == '' ? tokenization.left[edgeWindow.editBoundary.tokenIndex-1]?.text ?? '' : edgeWindow.editBoundary.text; - const appliedBoundaryTail = nonEmptyBoundaryRoot + tokenizedCorrectionEntries[0].insert; - - // If the previous text can be tokenized into multiple tokens, the first correction's text should stand alone. - // If not, the correction root should incorporate `nonEmptyBoundaryRoot` as a prefix. - const tailTransform = tokenizedCorrectionEntries[0]; - if(tokenizer({left: appliedBoundaryTail, startOfBuffer: false, endOfBuffer: false}).left.length > 1) { - tokenizedCorrectionEntries[0] = { - ...tailTransform, - deleteLeft: 0 - } - } else { - tokenizedCorrectionEntries[0] = { - insert: nonEmptyBoundaryRoot + tailTransform.insert, - deleteLeft: 0 - } - } + const { tokensToRemove, tokensToPredict } = transitionEffects; + const deleteLeft = tokensToPredict.length > 1 ? 0 : tokensToRemove.reduce((prev, curr) => prev + curr.codepointLength, 0); + // IF: array has multiple entries, then build the preservation-transform as below, including the deleteLeft. + // If not, don't make one! const preservationTransform = tokenizedCorrectionEntries.slice(0, -1).reduce((accum, curr) => { return models.buildMergedTransform(accum, {...curr, deleteLeft: 0}); }, { insert: '', deleteLeft, id: correction.sample.id}); - - const predictions = predictFromCorrectionSequence(lexicalModel, tokenizedCorrectionEntries.map((e) =>{ - return { - sample: e, - p: correction.p - } - }), rootContext, transformId); - - predictions.forEach((p) => { - p.preservationTransform = preservationTransform; - if(transformId) { - p.prediction.sample.transformId = transformId; + returnedPredictionData.push({ + ...suggestionParams, + applyInPost: (p) => { + p.preservationTransform = preservationTransform; + if(transformId) { + p.prediction.sample.transformId = transformId; + } } - returnedPredictions.push(p); - }); + }) } - return returnedPredictions; + return returnedPredictionData; } /** @@ -568,8 +561,13 @@ export async function correctAndEnumerate( // It's mostly here to support models compiled before Keyman 14.0, which was // when the `LexiconTraversal` pattern was established. if(!contextTracker) { + const predictionData = determineTraversallessCorrectionSequences(lexicalModel, transformDistribution, context); return { - rawPredictions: correctAndEnumerateWithoutTraversals(lexicalModel, transformDistribution, context) + rawPredictions: predictionData.flatMap((entry) => { + const predictions = predictFromCorrectionSequence(lexicalModel, entry.tokenizedCorrection, entry.rootContext, transformDistribution[0]?.sample.id); + predictions.forEach((p) => entry.applyInPost(p)); + return predictions; + }) }; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts index ddf838d4586..e562f34d079 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -236,7 +236,7 @@ describe('determineTokenizedCorrectionSequence', () => { }); // Will be handled far better after resolving multi-tokenization handling. - it.skip(`properly analyzes post-split case`, () => { + it.skip(`properly analyzes post-split new-wordbreak case`, () => { const context: Context = { left: 'the quick brown fox can\'', right: '', diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts new file mode 100644 index 00000000000..994c918d1a8 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -0,0 +1,90 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-05-18 + * + * This file tests the prediction helper-method responsible for preparing + * corrections for multi-token prediction for some custom and all legacy models. + */ + +import { assert } from 'chai'; + +import { LexicalModelTypes } from "@keymanapp/common-types"; +import * as wordBreakers from '@keymanapp/models-wordbreakers'; + +import { determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; + +import Context = LexicalModelTypes.Context; +import DummyModel = models.DummyModel; +import DummyOptions = models.DummyOptions; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; + + +/* + * This file's tests use these parts of a lexical model: + * - model.wordbreaker + * - model.toKey + * - model.applyCasing + * - model.punctuation + */ + +const DUMMY_MODEL_CONFIG: DummyOptions = { + punctuation: { + quotesForKeepSuggestion: { + open: '<', + close: '>' + }, + insertAfterWord: '\u00a0' // non-breaking space + }, + wordbreaker: wordBreakers.default +}; + +const testModel = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + // No suggestions needed here, so we don't define any. +}); + +describe('determineTraversallessCorrectionSequences', () => { + it(`creates an 'exact'-match suggestion based on primary input and current context`, () => { + const context: Context = { + left: 'iphon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e', + deleteLeft: 0 + }, + p: 1 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokenizedCorrection, [{ + sample: { + insert: 'iphone', + deleteLeft: 0 + }, + p: 1 + }]); + }); +}); \ No newline at end of file From 3d355f054b662a56f731ee29d492643cd3b1c09c Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 22 May 2026 09:09:18 -0500 Subject: [PATCH 39/65] fix(web): add codepointLength prop to match new base-branch reqts --- .../worker-thread/src/main/predict-helpers.ts | 3 ++- .../determine-traversalless-correction-sequences.tests.ts | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 5c3f3109e8f..5b8f3a78131 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -202,7 +202,8 @@ export function determineTraversallessCorrectionSequences( const tokenization = tokenizer(context); // issue at present if no tokens exist! const tokenMapper = (t: models.Token) => { return { - exampleInput: t.text + exampleInput: t.text, + codepointLength: KMWString.length(t.text) } as ContextTokenLike; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts index 994c918d1a8..8ddc10d94d8 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -46,9 +46,9 @@ const testModel = new DummyModel({ }); describe('determineTraversallessCorrectionSequences', () => { - it(`creates an 'exact'-match suggestion based on primary input and current context`, () => { + it(`processes standard-case corrections correctly - text appended to existing token`, () => { const context: Context = { - left: 'iphon', + left: 'I want an iPhon', right: '', startOfBuffer: true, endOfBuffer: true @@ -72,7 +72,7 @@ describe('determineTraversallessCorrectionSequences', () => { ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined }, { casingForm: undefined, - left: '', + left: 'I want an ', right: '', startOfBuffer: true, endOfBuffer: true @@ -81,7 +81,7 @@ describe('determineTraversallessCorrectionSequences', () => { assert.deepEqual(entry.tokenizedCorrection, [{ sample: { - insert: 'iphone', + insert: 'iPhone', deleteLeft: 0 }, p: 1 From 808c598771aa6b1c9988321b0f646e40ba717f1a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 22 May 2026 15:18:17 -0500 Subject: [PATCH 40/65] feat(web): add unit tests for multi-token traversalless-model correction-sequence construction --- .../worker-thread/src/main/predict-helpers.ts | 6 +- ...raversalless-correction-sequences.tests.ts | 320 +++++++++++++++++- 2 files changed, 320 insertions(+), 6 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 5b8f3a78131..ab1b0f5b832 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -228,14 +228,12 @@ export function determineTraversallessCorrectionSequences( const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample); const tokenizedCorrection = tokenizationMapping.tokenizedTransform; const tokenizedCorrectionEntries = [...tokenizedCorrection.values()]; - const { tokensToRemove, tokensToPredict } = transitionEffects; - const deleteLeft = tokensToPredict.length > 1 ? 0 : tokensToRemove.reduce((prev, curr) => prev + curr.codepointLength, 0); // IF: array has multiple entries, then build the preservation-transform as below, including the deleteLeft. // If not, don't make one! const preservationTransform = tokenizedCorrectionEntries.slice(0, -1).reduce((accum, curr) => { - return models.buildMergedTransform(accum, {...curr, deleteLeft: 0}); - }, { insert: '', deleteLeft, id: correction.sample.id}); + return { insert: accum.insert + curr.insert, deleteLeft: accum.deleteLeft + curr.deleteLeft }; + }, { insert: '', deleteLeft: 0, id: correction.sample.id}); returnedPredictionData.push({ ...suggestionParams, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts index 8ddc10d94d8..9b14ecb1046 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -11,8 +11,9 @@ import { assert } from 'chai'; import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; +import { KMWString } from 'keyman/common/web-utils'; -import { determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; +import { CorrectionPredictionTuple, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import DummyModel = models.DummyModel; @@ -46,6 +47,48 @@ const testModel = new DummyModel({ }); describe('determineTraversallessCorrectionSequences', () => { + it(`processes common-case corrections correctly - on context reset with existing text`, () => { + const context = { + left: 'appl', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: '', + deleteLeft: 0 + }, + p: 1 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokenizedCorrection, [{ + sample: { + insert: 'appl', + deleteLeft: 0 + }, + p: trueInput.p + }]); + }); + it(`processes standard-case corrections correctly - text appended to existing token`, () => { const context: Context = { left: 'I want an iPhon', @@ -84,7 +127,280 @@ describe('determineTraversallessCorrectionSequences', () => { insert: 'iPhone', deleteLeft: 0 }, - p: 1 + p: trueInput.p + }]); + }); + + it(`properly analyzes common-case token-extension - adding a letter to an existing word`, () => { + const context: Context = { + left: 'the quick brown f', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'o', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokenizedCorrection, [{ + sample: { + insert: 'fo', + deleteLeft: 0 + }, + p: trueInput.p }]); }); + + it(`properly analyzes common-case whitespace - ending a token and adding a new one`, () => { + const context: Context = { + left: 'the quick brown', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the quick brown', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokenizedCorrection, [{ + sample: { + insert: '', // empty token after a whitespace. + deleteLeft: 0 + }, + p: trueInput.p + }]); + }); + + + it(`properly analyzes common-case word-start - beginning a new token`, () => { + const context: Context = { + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokenizedCorrection, [{ + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: trueInput.p + }]); + }); + + it(`properly analyzes post-merge case`, () => { + let context: Context = { + left: 'the quick brown fox can\'', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 't', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the quick brown fox ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokenizedCorrection, [{ + sample: { + insert: 'can\'t', + deleteLeft: 0 + }, + p: trueInput.p + }]); + }); + + it(`properly analyzes post-split new-wordbreak case`, () => { + const context: Context = { + left: 'the quick brown fox can\'', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + // assert.deepEqual( + // { + // ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + // }, { + // casingForm: undefined, + // // Proper logic requires full multi-token awareness; predictions are currently + // // based on just the last token. + // left: 'the quick brown fox can\'', + // right: '', + // startOfBuffer: true, + // endOfBuffer: true + // } + // ); + + assert.deepEqual(entry.tokenizedCorrection, [{ + sample: { + insert: '', // empty token after a whitespace. + deleteLeft: 0 + }, + p: trueInput.p + }]); + }); + + it(`properly analyzes complex transition - multi-token replacement`, () => { + const context: Context = { + left: 'the quick brown f', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'fast red d', + deleteLeft: 'quick brown f'.length + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + // Large-scale deletions will receive enhanced handling soon. But, for now, it's + // deleted by the `preservationTransform`, not here. + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokenizedCorrection, [{ + sample: { + insert: 'd', + deleteLeft: 0 + }, + p: trueInput.p + }]); + + const dummiedTuple: CorrectionPredictionTuple = { + prediction: { + sample: { + transform: { insert: 'dog', deleteLeft: 0 }, + displayAs: 'dog' + }, + p: .25 + }, + correction: { + sample: 'd', + p: trueInput.p + }, + totalProb: .25 * trueInput.p + }; + + entry.applyInPost(dummiedTuple); + + assert.deepEqual(dummiedTuple.preservationTransform, { + insert: trueInput.sample.insert.substring(0, KMWString.length(trueInput.sample.insert) - 1), // remove the 'd'. + deleteLeft: trueInput.sample.deleteLeft - 1 + }); + }); }); \ No newline at end of file From 0a85adc01a07ce3426d2d83e7629b988f438bd13 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 27 May 2026 15:40:12 -0500 Subject: [PATCH 41/65] refactor(web): rework application of single-char correction penalty Build-bot: skip build:web Test-bot: skip --- .../main/correction/token-result-mapping.ts | 4 + .../main/correction/tokenization-corrector.ts | 3 + .../worker-thread/src/main/predict-helpers.ts | 70 ++++++++------- ...ine-tokenized-correction-sequence.tests.ts | 89 +++++++++---------- ...raversalless-correction-sequences.tests.ts | 54 ++++++----- 5 files changed, 110 insertions(+), 110 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts index c85e9e9b6d4..862ac1f740f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts @@ -75,6 +75,10 @@ export class TokenResultMapping implements CorrectionResultMapping, return this.node; } + get inputCount(): number { + return this.matchingSpace.inputCount; + } + get inputSequence(): ProbabilityMass[] { return this.node.priorInput; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index bd2ccacb00e..07b6f8ae16f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -34,6 +34,7 @@ import { MAX_EDIT_THRESHOLD_FACTOR } from "./search-quotient-spur.js"; export type TokenResult = { matchString: string, inputSamplingCost: number, + inputCount: number, knownCost: number, totalCost: number } @@ -196,6 +197,7 @@ export class TokenizationCorrector implements CorrectionSearchable { @@ -216,14 +215,25 @@ export function determineTraversallessCorrectionSequences( const postTokenization = tokenizer(postContext); const transitionEffects = determineSuggestionRange(tokenization.left.map(tokenMapper), postTokenization.left.map(tokenMapper), (a, b) => a.exampleInput == b.exampleInput); - const match: TokenResult = { - matchString: wordbreak(postContext), - inputSamplingCost: -Math.log(correction.p), - knownCost: 0, - totalCost: -Math.log(correction.p) - }; + transitionEffects.transitionId = correction.sample.id; + + // Build _multiple_ tokens to function as fake `TokenizationCorrector` + // results in order to generate a correction-root sequence. + const correctionRoots = transitionEffects.tokensToPredict.map((token, index) => { + const match: TokenResult = { + matchString: token.exampleInput, + // Key part: should be > 1 if the token is longer than just the most recent insert. + // Any extra tokens are only affected by the current input. + inputCount: index == 0 ? Math.max(1, KMWString.length(token.exampleInput) + 1 - KMWString.length(correction.sample.insert)) : 1, + inputSamplingCost: -Math.log(correction.p), + knownCost: 0, + totalCost: -Math.log(correction.p) + }; - const suggestionParams = buildCorrectionSequence(transitionEffects, context, match, 1); + return match; + }); + + const suggestionParams = buildCorrectionSequence(transitionEffects, context, correctionRoots[correctionRoots.length - 1]); const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample); const tokenizedCorrection = tokenizationMapping.tokenizedTransform; @@ -449,7 +459,6 @@ export function buildCorrectionSequence( transitionEffects: ReturnType, context: Context, match: Readonly, - costFactor: number ) { const { deleteLeft } = transitionEffects; @@ -461,6 +470,21 @@ export function buildCorrectionSequence( deleteLeft: 0, } + /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost + * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if + * there are significantly more likely words. We only need this to allow very minor fat-finger + * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on + * key borders. + * + * Technically, the probabilities this produces won't be normalized as-is... but there's no + * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when + * to apply it could become tricky, so it's simpler to leave out. + * + * Worst-case, it's possible to temporarily add normalization if a code deep-dive + * is needed in the future. + */ + const costFactor = (match.inputCount <= 1) ? ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT : 1; + const rootCost = match.totalCost; const predictionRoot = { sample: correctionTransform, @@ -483,20 +507,17 @@ export function buildCorrectionSequence( * generated. * @param match The generated correction itself - the correction string * and its cost - * @param costFactor A multiplicative factor used to adjust the cost when - * building prediction probabilities. * @returns */ export function determineTokenizedCorrectionSequence( transition: ContextTransition, tokenization: ContextTokenization, - match: Readonly, - costFactor: number + match: Readonly ): PredictionParameters { const applicationTarget = transition.base.displayTokenization; const transitionParams = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); - const suggestionParams = buildCorrectionSequence(transitionParams, transition.base.context, match, costFactor); + const suggestionParams = buildCorrectionSequence(transitionParams, transition.base.context, match); // The correction should always be based on the most recent external // transform/transcription ID. @@ -620,28 +641,13 @@ export async function correctAndEnumerate( continue; } - /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost - * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if - * there are significantly more likely words. We only need this to allow very minor fat-finger - * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on - * key borders. - * - * Technically, the probabilities this produces won't be normalized as-is... but there's no - * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when - * to apply it could become tricky, so it's simpler to leave out. - * - * Worst-case, it's possible to temporarily add normalization if a code deep-dive - * is needed in the future. - */ - const costFactor = (tokenization.tail.inputCount <= 1) ? ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT : 1; - - const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, match, costFactor); + const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, match); const predictions = predictFromCorrectionSequence(lexicalModel, predictionPrep.tokenizedCorrection, predictionPrep.rootContext, transition.transitionId); predictions.forEach((p) => predictionPrep.applyInPost(p)); // Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions. - if(predictions.length > 0 && bestCorrectionCost === undefined) { - bestCorrectionCost = match.totalCost * costFactor; + if(predictions.length > 0 && (bestCorrectionCost === undefined || bestCorrectionCost > match.totalCost)) { + bestCorrectionCost = match.totalCost; } // If we're getting the same prediction again, it's lower-cost. Update! diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts index e562f34d079..7623bfffe51 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -15,7 +15,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { KMWString } from 'keyman/common/web-utils'; -import { determineTokenizedCorrectionSequence, models, ContextState, ContextToken, ContextTokenization, CorrectionPredictionTuple } from "@keymanapp/lm-worker/test-index"; +import { determineTokenizedCorrectionSequence, models, ContextState, ContextToken, ContextTokenization, CorrectionPredictionTuple, ModelCompositor } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; @@ -54,10 +54,10 @@ describe('determineTokenizedCorrectionSequence', () => { transition.final.displayTokenization, { matchString: 'fo', inputSamplingCost: -Math.log(trueInput.p), + inputCount: 2, knownCost: 0, totalCost: -Math.log(trueInput.p) - }, - 1 + } ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -104,10 +104,10 @@ describe('determineTokenizedCorrectionSequence', () => { transition.final.displayTokenization, { matchString: ' ', inputSamplingCost: -Math.log(trueInput.p), + inputCount: 1, knownCost: 0, totalCost: -Math.log(trueInput.p) - }, - 1 + } ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -118,15 +118,12 @@ describe('determineTokenizedCorrectionSequence', () => { endOfBuffer: true }); - assert.deepEqual(results.tokenizedCorrection, [ - { - sample: { - insert: ' ', - deleteLeft: 0 - }, - p: trueInput.p - } - ]); + assert.equal(results.tokenizedCorrection.length, 1); + assert.deepEqual(results.tokenizedCorrection[0].sample, { + insert: ' ', + deleteLeft: 0 + }); + assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); }); it(`properly analyzes common-case word-start - beginning a new token`, () => { @@ -154,10 +151,10 @@ describe('determineTokenizedCorrectionSequence', () => { transition.final.displayTokenization, { matchString: 'f', inputSamplingCost: -Math.log(trueInput.p), + inputCount: 1, knownCost: 0, totalCost: -Math.log(trueInput.p) - }, - 1 + } ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -168,15 +165,13 @@ describe('determineTokenizedCorrectionSequence', () => { endOfBuffer: true }); - assert.deepEqual(results.tokenizedCorrection, [ - { - sample: { - insert: 'f', - deleteLeft: 0 - }, - p: trueInput.p - } - ]); + + assert.equal(results.tokenizedCorrection.length, 1); + assert.deepEqual(results.tokenizedCorrection[0].sample, { + insert: 'f', + deleteLeft: 0 + }); + assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); }); it(`properly analyzes post-merge case`, () => { @@ -210,10 +205,10 @@ describe('determineTokenizedCorrectionSequence', () => { transition.final.displayTokenization, { matchString: 'can\'t', inputSamplingCost: -Math.log(trueInput.p), + inputCount: 5, knownCost: 0, totalCost: -Math.log(trueInput.p) - }, - 1 + } ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -261,10 +256,10 @@ describe('determineTokenizedCorrectionSequence', () => { transition.final.displayTokenization, { matchString: ' ', inputSamplingCost: -Math.log(trueInput.p), + inputCount: 1, knownCost: 0, totalCost: -Math.log(trueInput.p) - }, - 1 + } ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -275,15 +270,13 @@ describe('determineTokenizedCorrectionSequence', () => { endOfBuffer: true }); - assert.deepEqual(results.tokenizedCorrection, [ - { - sample: { - insert: ' ', - deleteLeft: 0 - }, - p: trueInput.p - } - ]); + + assert.equal(results.tokenizedCorrection.length, 1); + assert.deepEqual(results.tokenizedCorrection[0].sample, { + insert: ' ', + deleteLeft: 0 + }); + assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); }); it(`properly analyzes complex transition - multi-token replacement`, () => { @@ -310,10 +303,10 @@ describe('determineTokenizedCorrectionSequence', () => { transition.final.displayTokenization, { matchString: 'd', inputSamplingCost: -Math.log(trueInput.p), + inputCount: 1, knownCost: 0, totalCost: -Math.log(trueInput.p) - }, - 1 + } ); // Large-scale deletions will receive enhanced handling soon. But, for now, it's @@ -326,15 +319,13 @@ describe('determineTokenizedCorrectionSequence', () => { endOfBuffer: true }); - assert.deepEqual(results.tokenizedCorrection, [ - { - sample: { - insert: 'd', - deleteLeft: 0 - }, - p: trueInput.p - } - ]); + + assert.equal(results.tokenizedCorrection.length, 1); + assert.deepEqual(results.tokenizedCorrection[0].sample, { + insert: 'd', + deleteLeft: 0 + }); + assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); const dummiedTuple: CorrectionPredictionTuple = { prediction: { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts index 9b14ecb1046..11a4e841311 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -13,7 +13,7 @@ import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { KMWString } from 'keyman/common/web-utils'; -import { CorrectionPredictionTuple, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; +import { CorrectionPredictionTuple, ModelCompositor, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import DummyModel = models.DummyModel; @@ -204,13 +204,12 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.deepEqual(entry.tokenizedCorrection, [{ - sample: { - insert: '', // empty token after a whitespace. - deleteLeft: 0 - }, - p: trueInput.p - }]); + assert.equal(entry.tokenizedCorrection.length, 1); + assert.deepEqual(entry.tokenizedCorrection[0].sample, { + insert: '', + deleteLeft: 0 + }); + assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); }); @@ -246,13 +245,12 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.deepEqual(entry.tokenizedCorrection, [{ - sample: { - insert: 'f', - deleteLeft: 0 - }, - p: trueInput.p - }]); + assert.equal(entry.tokenizedCorrection.length, 1); + assert.deepEqual(entry.tokenizedCorrection[0].sample, { + insert: 'f', + deleteLeft: 0 + }); + assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); }); it(`properly analyzes post-merge case`, () => { @@ -330,13 +328,12 @@ describe('determineTraversallessCorrectionSequences', () => { // } // ); - assert.deepEqual(entry.tokenizedCorrection, [{ - sample: { - insert: '', // empty token after a whitespace. - deleteLeft: 0 - }, - p: trueInput.p - }]); + assert.equal(entry.tokenizedCorrection.length, 1); + assert.deepEqual(entry.tokenizedCorrection[0].sample, { + insert: '', + deleteLeft: 0 + }); + assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); }); it(`properly analyzes complex transition - multi-token replacement`, () => { @@ -373,13 +370,12 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.deepEqual(entry.tokenizedCorrection, [{ - sample: { - insert: 'd', - deleteLeft: 0 - }, - p: trueInput.p - }]); + assert.equal(entry.tokenizedCorrection.length, 1); + assert.deepEqual(entry.tokenizedCorrection[0].sample, { + insert: 'd', + deleteLeft: 0 + }); + assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); const dummiedTuple: CorrectionPredictionTuple = { prediction: { From 1a7bb31bda0c9378548bca3d1a1c6d15a1e5aacc Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 30 Apr 2026 08:28:43 -0500 Subject: [PATCH 42/65] feat(web): tokenize input corrections and provide for multi-token predictions Build-bot: skip build:web Test-bot: skip --- .../correction/tokenization-result-mapping.ts | 4 +- .../worker-thread/src/main/predict-helpers.ts | 85 ++++++++++--------- ...ine-tokenized-correction-sequence.tests.ts | 48 +++++++---- ...raversalless-correction-sequences.tests.ts | 6 +- 4 files changed, 83 insertions(+), 60 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts index 32e0fb48fce..c5588424afe 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts @@ -5,13 +5,13 @@ export class TokenizationResultMapping implements CorrectionResultMapping; - constructor(tokenization: TokenResult[], corrector: TokenizationCorrector) { + constructor(tokenization: TokenResult[], corrector?: TokenizationCorrector) { this.matchingSpace = corrector; this.matchedResult = tokenization; } get spaceId(): number { - return this.matchingSpace.tokenization.spaceId; + return this.matchingSpace?.tokenization.spaceId; } // /** diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 196a6ac235c..67b04137bbc 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -14,6 +14,8 @@ import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js'; import { TokenResult } from './correction/tokenization-corrector.js'; +import { TokenizationCorrector } from './correction/tokenization-corrector.js'; +import { TokenizationResultMapping } from './correction/tokenization-result-mapping.js'; import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; @@ -233,7 +235,8 @@ export function determineTraversallessCorrectionSequences( return match; }); - const suggestionParams = buildCorrectionSequence(transitionEffects, context, correctionRoots[correctionRoots.length - 1]); + // But, for now, only actually use the last one. + const suggestionParams = buildCorrectionSequence(transitionEffects, context, new TokenizationResultMapping([correctionRoots[correctionRoots.length - 1]], null)); const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample); const tokenizedCorrection = tokenizationMapping.tokenizedTransform; @@ -377,12 +380,8 @@ export function determineSuggestionRange( return temp(a, b); } - const deleteLeftCalc = (tokenSet: T[], predictCount: number) => { - // TODO: once we start activating multi-tokenization for real, only the - // 'reduce' component should remain. - return (predictCount > 1) - ? (tokenSet[tokenSet.length - 1]?.codepointLength ?? 0) - : tokenSet.reduce((prev, curr) => prev + curr.codepointLength, 0); + const deleteLeftCalc = (tokenSet: T[]) => { + return tokenSet.reduce((prev, curr) => prev + curr.codepointLength, 0); } const tokenSetA = userContextTokenization.slice(); @@ -396,7 +395,7 @@ export function determineSuggestionRange( return { tokensToRemove: tokenSetA, tokensToPredict: tokenSetB, - deleteLeft: deleteLeftCalc(tokenSetA, tokenSetB.length) + deleteLeft: deleteLeftCalc(tokenSetA) } } else if(aHeadIndexInB != 0 && bHeadIndexInA != 0) { throw new Error("Leading edge of context should not differ in both tokenizations."); @@ -422,7 +421,7 @@ export function determineSuggestionRange( return { tokensToRemove, tokensToPredict, - deleteLeft: deleteLeftCalc(tokensToRemove, tokensToPredict.length) + deleteLeft: deleteLeftCalc(tokensToRemove) } } @@ -456,44 +455,49 @@ export interface PredictionParameters { } export function buildCorrectionSequence( - transitionEffects: ReturnType, + transitionEffects: SuggestionReplacement, context: Context, - match: Readonly, + tokenizationCorrection: TokenizationResultMapping ) { const { deleteLeft } = transitionEffects; const rootContext = models.applyTransform({insert: '', deleteLeft}, context); // Replace the existing context with the correction. - const correctionTransform: Transform = { - insert: match.matchString, // insert correction string - deleteLeft: 0, - } + const tokenizedCorrections = tokenizationCorrection.matchedResult.map((correction, i) => { + /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost + * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if + * there are significantly more likely words. We only need this to allow very minor fat-finger + * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on + * key borders. + * + * Technically, the probabilities this produces won't be normalized as-is... but there's no + * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when + * to apply it could become tricky, so it's simpler to leave out. + * + * Worst-case, it's possible to temporarily add normalization if a code deep-dive + * is needed in the future. + */ + const costFactor = (correction.inputCount <= 1) ? ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT : 1; + + const entry = { + sample: { + insert: correction.matchString, // insert correction string + deleteLeft: 0, + } as Transform, + p: Math.exp(-correction.totalCost * costFactor) + }; - /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost - * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if - * there are significantly more likely words. We only need this to allow very minor fat-finger - * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on - * key borders. - * - * Technically, the probabilities this produces won't be normalized as-is... but there's no - * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when - * to apply it could become tricky, so it's simpler to leave out. - * - * Worst-case, it's possible to temporarily add normalization if a code deep-dive - * is needed in the future. - */ - const costFactor = (match.inputCount <= 1) ? ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT : 1; - - const rootCost = match.totalCost; - const predictionRoot = { - sample: correctionTransform, - p: Math.exp(-rootCost * costFactor) - }; + if(transitionEffects.transitionId !== undefined) { + entry.sample.id = transitionEffects.transitionId; + } + + return entry; + }); return { rootContext, - tokenizedCorrection: [predictionRoot] + tokenizedCorrection: tokenizedCorrections }; } @@ -512,10 +516,11 @@ export function buildCorrectionSequence( export function determineTokenizedCorrectionSequence( transition: ContextTransition, tokenization: ContextTokenization, - match: Readonly + match: TokenizationResultMapping ): PredictionParameters { const applicationTarget = transition.base.displayTokenization; const transitionParams = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); + transitionParams.transitionId = transition.transitionId; const suggestionParams = buildCorrectionSequence(transitionParams, transition.base.context, match); @@ -641,7 +646,11 @@ export async function correctAndEnumerate( continue; } - const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, match); + const suggestionRange = determineSuggestionRange(transition.base.displayTokenization.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); + suggestionRange.transitionId = transition.transitionId; + const corrector = new TokenizationCorrector(tokenization, suggestionRange.tokensToPredict.length, () => true); + const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, new TokenizationResultMapping([match], corrector)); + const predictions = predictFromCorrectionSequence(lexicalModel, predictionPrep.tokenizedCorrection, predictionPrep.rootContext, transition.transitionId); predictions.forEach((p) => predictionPrep.applyInPost(p)); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts index 7623bfffe51..ee8e2ba9109 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -15,7 +15,16 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { KMWString } from 'keyman/common/web-utils'; -import { determineTokenizedCorrectionSequence, models, ContextState, ContextToken, ContextTokenization, CorrectionPredictionTuple, ModelCompositor } from "@keymanapp/lm-worker/test-index"; +import { + determineTokenizedCorrectionSequence, + models, + ContextState, + ContextToken, + ContextTokenization, + CorrectionPredictionTuple, + ModelCompositor, + TokenizationResultMapping +} from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; @@ -51,13 +60,14 @@ describe('determineTokenizedCorrectionSequence', () => { const results = determineTokenizedCorrectionSequence( transition, - transition.final.displayTokenization, { + transition.final.displayTokenization, + new TokenizationResultMapping([{ matchString: 'fo', inputSamplingCost: -Math.log(trueInput.p), inputCount: 2, knownCost: 0, totalCost: -Math.log(trueInput.p) - } + }], null) ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -101,13 +111,14 @@ describe('determineTokenizedCorrectionSequence', () => { const results = determineTokenizedCorrectionSequence( transition, - transition.final.displayTokenization, { + transition.final.displayTokenization, + new TokenizationResultMapping([{ matchString: ' ', inputSamplingCost: -Math.log(trueInput.p), inputCount: 1, knownCost: 0, totalCost: -Math.log(trueInput.p) - } + }], null) ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -148,13 +159,14 @@ describe('determineTokenizedCorrectionSequence', () => { const results = determineTokenizedCorrectionSequence( transition, - transition.final.displayTokenization, { + transition.final.displayTokenization, + new TokenizationResultMapping([{ matchString: 'f', inputSamplingCost: -Math.log(trueInput.p), inputCount: 1, knownCost: 0, totalCost: -Math.log(trueInput.p) - } + }], null) ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -202,13 +214,14 @@ describe('determineTokenizedCorrectionSequence', () => { const results = determineTokenizedCorrectionSequence( transition, - transition.final.displayTokenization, { + transition.final.displayTokenization, + new TokenizationResultMapping([{ matchString: 'can\'t', inputSamplingCost: -Math.log(trueInput.p), inputCount: 5, knownCost: 0, totalCost: -Math.log(trueInput.p) - } + }], null) ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -253,13 +266,14 @@ describe('determineTokenizedCorrectionSequence', () => { const results = determineTokenizedCorrectionSequence( transition, - transition.final.displayTokenization, { + transition.final.displayTokenization, + new TokenizationResultMapping([{ matchString: ' ', inputSamplingCost: -Math.log(trueInput.p), inputCount: 1, knownCost: 0, totalCost: -Math.log(trueInput.p) - } + }], null) ); assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { @@ -300,26 +314,26 @@ describe('determineTokenizedCorrectionSequence', () => { const results = determineTokenizedCorrectionSequence( transition, - transition.final.displayTokenization, { + transition.final.displayTokenization, + new TokenizationResultMapping([{ matchString: 'd', inputSamplingCost: -Math.log(trueInput.p), inputCount: 1, knownCost: 0, totalCost: -Math.log(trueInput.p) - } + }], null) ); - // Large-scale deletions will receive enhanced handling soon. But, for now, it's - // deleted by the `preservationTransform`, not here. assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { casingForm: undefined, - left: 'the quick brown ', + left: 'the ', right: '', startOfBuffer: true, endOfBuffer: true }); - + // Coming up next - actually providing ALL correction elements, not just the final one. + // We're not _quite_ ready for that yet, though. assert.equal(results.tokenizedCorrection.length, 1); assert.deepEqual(results.tokenizedCorrection[0].sample, { insert: 'd', diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts index 11a4e841311..f9c7dccd550 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -361,15 +361,15 @@ describe('determineTraversallessCorrectionSequences', () => { ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined }, { casingForm: undefined, - // Large-scale deletions will receive enhanced handling soon. But, for now, it's - // deleted by the `preservationTransform`, not here. - left: 'the quick brown ', + left: 'the ', right: '', startOfBuffer: true, endOfBuffer: true } ); + // Coming up next - actually providing ALL correction elements, not just the final one. + // We're not _quite_ ready for that yet, though. assert.equal(entry.tokenizedCorrection.length, 1); assert.deepEqual(entry.tokenizedCorrection[0].sample, { insert: 'd', From 9a26be51c0de4fd3a37ce39e5aef8c3f4e4f28ca Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 12 May 2026 11:57:47 -0500 Subject: [PATCH 43/65] fix(web): fix bugs in createDefaultKeep, extend unit testing It turns out that #15766 did not perfectly address all cases for generation of default "keep" suggestions. This PR will remedy the situation. Build-bot: skip build:web Test-bot: skip --- .../worker-thread/src/main/predict-helpers.ts | 27 ++- .../create-default-keep.tests.ts | 209 +++++++++++++++++- 2 files changed, 225 insertions(+), 11 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 67b04137bbc..71635be8bdd 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -973,7 +973,7 @@ export function processSimilarity( /** * Generates metadata for a new 'keep' suggestion based solely upon the existing - * context. + * context and the most likely input. * * This method is designed for use when no appropriate 'keep' suggestion was * generated by the correction-search process. @@ -989,19 +989,28 @@ export function createDefaultKeep( ): CorrectionPredictionTuple { const { sample: inputTransform, p: inputTransformProb } = trueInput; const wordbreak = determineModelWordbreaker(lexicalModel); + const tokenizer = determineModelTokenizer(lexicalModel); const postContext = models.applyTransform(inputTransform, context); const truePrefix = wordbreak(postContext); - // Generate a full-word 'keep' replacement like other suggestions when one is not otherwise - // produced; we want to replace the full token in the same manner used for other suggestions. - const basePrefixLength = KMWString.length(truePrefix) - KMWString.length(inputTransform.insert) + inputTransform.deleteLeft; - const keepTransform = { - insert: truePrefix, - deleteLeft: basePrefixLength - }; + const tokenization = tokenizer(context); + const postTokenization = tokenizer(postContext); + + const tokenMapper = (t: models.Token) => { + return { + exampleInput: t.text, + codepointLength: KMWString.length(t.text) + } as ContextTokenLike; + } + const transitionEffects = determineSuggestionRange(tokenization.left.map(tokenMapper), postTokenization.left.map(tokenMapper), (a, b) => a.exampleInput == b.exampleInput); + + const keepTransition = { + insert: transitionEffects.tokensToPredict.reduce((accum, curr) => accum + curr.exampleInput, ''), + deleteLeft: transitionEffects.tokensToRemove.reduce((accum, curr) => accum + curr.codepointLength, 0) + } + let keepSuggestion = models.transformToSuggestion(keepTransition); - let keepSuggestion = models.transformToSuggestion(keepTransform); // This is the one case where the transform doesn't insert the full word; we need to override the displayAs param. keepSuggestion.displayAs = truePrefix; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts index 6b5981c8343..052c349291c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts @@ -91,8 +91,8 @@ const testModelWithCasing = new DummyModel({ // No suggestions needed here, so we don't define any. }); -describe('produceKeep', () => { - it(`creates an 'exact'-match suggestion based on primary input and current context`, () => { +describe('createDefaultKeep', () => { + it(`creates an 'exact'-match suggestion based on simple primary input`, () => { const context: Context = { left: 'iphon', right: '', @@ -132,4 +132,209 @@ describe('produceKeep', () => { const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); assert.deepEqual(tuple, expectedKeep); }); + + it(`creates an 'exact'-match suggestion based on full word after a backspace`, () => { + const context: Context = { + left: 'iphone ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: '', + deleteLeft: 1 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'iphone', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphone', + deleteLeft: 7 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex deletion`, () => { + const context: Context = { + left: 'iphone a', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e', + deleteLeft: 3 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'iphone', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphone', + deleteLeft: 8 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex insertion`, () => { + const context: Context = { + left: 'iphon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'es and', + deleteLeft: 0 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'and', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphones and', + deleteLeft: 5 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex replacement`, () => { + const context: Context = { + left: 'iphone ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 's', + deleteLeft: 1 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'iphones', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphones', + deleteLeft: 7 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an empty 'exact'-match suggestion after adding a wordbreak`, () => { + const context: Context = { + left: 'iphon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e ', + deleteLeft: 0 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: '', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphone ', + deleteLeft: 5 + }, + displayAs: '<>', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); }); \ No newline at end of file From 389586fbdc65ddc4465c690cdbb9656a20b63f5b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 12 Jun 2026 08:32:19 -0500 Subject: [PATCH 44/65] change(web): cover .deleteLeft in unit testing --- .../src/main/correction/context-token.ts | 2 +- .../determine-suggestion-range.tests.ts | 42 +++++++++++++++---- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index ce1d8f9830a..e21b9709b0b 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -37,7 +37,7 @@ function textToCharTransforms(text: string, transformId?: number): Transform[] { /** - * Implements an interface similar to ContextToken that is useful for handling + * Defines an interface compatible with ContextToken that is useful for handling * cases that should not be considered correctable. */ export interface ContextTokenLike { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts index e18174895ec..a6b5bcfaeb7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts @@ -107,7 +107,10 @@ function buildQuickBrownFixture() { null ); + const deleteLeftCalc = (tokens: ContextToken[]) => tokens.reduce((accum, curr) => accum + curr.codepointLength, 0); + return { + deleteLeftCalc, baseTokenization, variations: { noChange: { @@ -115,7 +118,8 @@ function buildQuickBrownFixture() { tokenization: baseTokenization, range: { tokensToRemove: [baseTokenization.tail], - tokensToPredict: [baseTokenization.tail] + tokensToPredict: [baseTokenization.tail], + deleteLeft: deleteLeftCalc([baseTokenization.tail]) } }, plainInsert: { @@ -123,7 +127,8 @@ function buildQuickBrownFixture() { tokenization: plainInsertTokenization, range: { tokensToRemove: [baseTokenization.tail], - tokensToPredict: [plainInsertTokenization.tail] + tokensToPredict: [plainInsertTokenization.tail], + deleteLeft: deleteLeftCalc([baseTokenization.tail]) } }, newTokenInsert: { @@ -131,7 +136,8 @@ function buildQuickBrownFixture() { tokenization: newTokenInsertTokenization, range: { tokensToRemove: [] as ContextToken[], - tokensToPredict: [newTokenInsertTokenization.tail] + tokensToPredict: [newTokenInsertTokenization.tail], + deleteLeft: deleteLeftCalc([]) } }, charReplace: { @@ -139,7 +145,8 @@ function buildQuickBrownFixture() { tokenization: charReplaceTokenization, range: { tokensToRemove: [baseTokenization.tail], - tokensToPredict: [charReplaceTokenization.tail] + tokensToPredict: [charReplaceTokenization.tail], + deleteLeft: deleteLeftCalc([baseTokenization.tail]) } }, eraseToken: { @@ -147,7 +154,8 @@ function buildQuickBrownFixture() { tokenization: eraseTokenTokenization, range: { tokensToRemove: [baseTokenization.tail], - tokensToPredict: [eraseTokenTokenization.tail] + tokensToPredict: [eraseTokenTokenization.tail], + deleteLeft: deleteLeftCalc([baseTokenization.tail]) } }, del5Insert5: { @@ -155,7 +163,8 @@ function buildQuickBrownFixture() { tokenization: del5Insert5Tokenization, range: { tokensToRemove: baseTokenization.tokens.slice(baseTokenCount-3), - tokensToPredict: [del5Insert5Tokenization.tail] + tokensToPredict: [del5Insert5Tokenization.tail], + deleteLeft: deleteLeftCalc(baseTokenization.tokens.slice(baseTokenCount-3)) } }, deleteToBound: { @@ -163,7 +172,8 @@ function buildQuickBrownFixture() { tokenization: deleteToBoundTokenization, range: { tokensToRemove: baseTokenization.tokens.slice(baseTokenCount-3), - tokensToPredict: [deleteToBoundTokenization.tail] + tokensToPredict: [deleteToBoundTokenization.tail], + deleteLeft: deleteLeftCalc(baseTokenization.tokens.slice(baseTokenCount-3)) } } } @@ -181,6 +191,7 @@ describe('determineSuggestionRange', () => { assert.sameOrderedMembers(analysis.tokensToRemove, noChange.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, noChange.range.tokensToPredict); + assert.equal(analysis.deleteLeft, noChange.range.deleteLeft); }); it('adjusts the final token after a simple same-token insert', () => { @@ -191,6 +202,7 @@ describe('determineSuggestionRange', () => { assert.sameOrderedMembers(analysis.tokensToRemove, plainInsert.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, plainInsert.range.tokensToPredict); + assert.equal(analysis.deleteLeft, plainInsert.range.deleteLeft); }); it('adjusts the final token after a simple word-breaking insert', () => { @@ -201,6 +213,7 @@ describe('determineSuggestionRange', () => { assert.sameOrderedMembers(analysis.tokensToRemove, newTokenInsert.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, newTokenInsert.range.tokensToPredict); + assert.equal(analysis.deleteLeft, newTokenInsert.range.deleteLeft); }); it('adjusts the final token after a simple same-token character replacement', () => { @@ -211,6 +224,7 @@ describe('determineSuggestionRange', () => { assert.sameOrderedMembers(analysis.tokensToRemove, charReplace.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, charReplace.range.tokensToPredict); + assert.equal(analysis.deleteLeft, charReplace.range.deleteLeft); }); it('handles deletion of two tokens + alteration of the token before', () => { @@ -221,6 +235,7 @@ describe('determineSuggestionRange', () => { assert.sameOrderedMembers(analysis.tokensToRemove, del5Insert5.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, del5Insert5.range.tokensToPredict); + assert.equal(analysis.deleteLeft, del5Insert5.range.deleteLeft); }); it('handles deletion of chars up to closest whitespace', () => { @@ -231,6 +246,7 @@ describe('determineSuggestionRange', () => { assert.sameOrderedMembers(analysis.tokensToRemove, eraseToken.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, eraseToken.range.tokensToPredict); + assert.equal(analysis.deleteLeft, eraseToken.range.deleteLeft); }); it('handles deletion up to boundary of ancestor non-whitespace token', () => { @@ -241,10 +257,11 @@ describe('determineSuggestionRange', () => { assert.sameOrderedMembers(analysis.tokensToRemove, deleteToBound.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, deleteToBound.range.tokensToPredict); + assert.equal(analysis.deleteLeft, deleteToBound.range.deleteLeft); }); it('handles large variation in intermediate tokens', () => { - const originalQuickBrownTokenization = buildQuickBrownFixture().baseTokenization; + const { deleteLeftCalc, baseTokenization: originalQuickBrownTokenization } = buildQuickBrownFixture(); const rawText = ['beyond', ' ', 'the', ' ', 'hungry', ' ', 'green', ' ', 'alligator']; // the quick brown fox jumped | // Final whitespace is immediately before index 10. @@ -267,10 +284,15 @@ describe('determineSuggestionRange', () => { analysis.tokensToPredict, tokensToAppend ); + + // TODO: Once we allow multiple tokens to contribute to deleteLeft, replace + // RHS with + // originalQuickBrownTokenization.tokens.slice(transitionSliceIndex). + assert.equal(analysis.deleteLeft, deleteLeftCalc([originalQuickBrownTokenization.tail])); }); it('handles insertion of many extra new tokens at once', () => { - const originalQuickBrownTokenization = buildQuickBrownFixture().baseTokenization; + const { deleteLeftCalc, baseTokenization: originalQuickBrownTokenization } = buildQuickBrownFixture(); const originalTokenCount = originalQuickBrownTokenization.tokens.length; const rawText = ['dogs', ' ', 'and', ' ', 'the', ' ', 'sleeping', ' ', 'cat']; const tokensToAppend = rawText.map((t) => ContextToken.fromRawText(plainModel, t, false)); @@ -291,5 +313,7 @@ describe('determineSuggestionRange', () => { analysis.tokensToPredict, tokensToAppend ); + + assert.equal(analysis.deleteLeft, deleteLeftCalc([originalQuickBrownTokenization.tail])); }); }); \ No newline at end of file From 415e0038799b819263be4d0e94a3375d77f1bf4b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 12 May 2026 14:15:17 -0500 Subject: [PATCH 45/65] refactor(web): refactor intermediate composited prediction type This reorganizes the type formerly known as CorrectionPredictionTuple, preparing it to share similarities with a new incoming type handling an earlier, tokenized intermediate stage that will be needed for some aspects of suggestion generation. Build-bot: skip build:web Test-bot: skip --- .../main/correction/tokenization-corrector.ts | 8 +- .../src/main/model-compositor.ts | 5 +- .../worker-thread/src/main/predict-helpers.ts | 242 ++++---- .../early-correction-search-stopping.tests.ts | 29 +- .../prediction-helpers/auto-correct.tests.ts | 544 ++++++++++-------- .../create-default-keep.tests.ts | 140 +++-- ...ine-tokenized-correction-sequence.tests.ts | 25 +- ...raversalless-correction-sequences.tests.ts | 25 +- .../predict-from-correction-sequence.tests.ts | 66 +-- .../suggestion-deduplication.tests.ts | 90 +-- .../suggestion-finalization.tests.ts | 98 ++-- .../suggestion-similarity.tests.ts | 188 +++--- .../worker-model-compositor.tests.ts | 4 + 13 files changed, 805 insertions(+), 659 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 07b6f8ae16f..835f9eef388 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -58,6 +58,7 @@ export class TokenizationCorrector implements CorrectionSearchable; private tokenCostMap: Map; private tokenLookupMap: Map; @@ -172,13 +173,16 @@ export class TokenizationCorrector implements CorrectionSearchable { // New issue: this mangles the space IDs! We almost certainly need some // sort of proper map to the source token. const searchModule = new QuotientNodeFinalizer(token.searchModule, index == orderedTokens.length - 1); this.tokenLookupMap.set(searchModule.spaceId, token); - if(!filterClosure(token)) { + const passesFilter = filterClosure(token); + modelsCorrectables ||= passesFilter; + if(!passesFilter) { this._uncorrectables.push(searchModule); } else if(index == tailCorrectionLength - 1) { // The sole assignment case for this field. It may only be assigned for @@ -189,6 +193,8 @@ export class TokenizationCorrector implements CorrectionSearchable { transitionId?: number } -/** - * Collates information related to suggestions during the suggestion generation - * process. - */ -export type CorrectionPredictionTuple = { +export interface CompositedPredictionData { /** * The potential Suggestion (or Keep) */ - prediction: ProbabilityMass, + prediction: Suggestion | Keep; /** * The correction upon which the Suggestion (or Keep) is based */ - correction: ProbabilityMass, + correction: string +} + +export interface PredictionProbabilities { + /** + * The probability of the word itself, separate from corrections, as + * determined by the LexicalModel itself. + */ + prediction: number; + + /** + * The probability of text-correction steps taken to build the correction upon + * which the prediction is based. + */ + correction: number; + + /** + * The likelihood of the represented prediction, combining both the + * `prediction` and `correction` components into a single value. + */ + total: number; +} + +/** + * Tracks common intermediate prediction data, such as its underlying probabilities and its similarity to the actual context. + */ +export interface PredictionMetadata { + /** + * Tracks the relevant probability components contributing to a generated + * prediction. + */ + probabilities: PredictionProbabilities; + /** - * The likelihood of the prediction - its lexical-model likelihood multiplied - * by the keystroke-sequence + correction likelihood. + * Indicates that the 'suggestion' represents context changes that qualify for + * auto-selection. */ - totalProb: number; + autoSelectable: boolean; + /** * How directly the prediction matches the current token in the context. * @@ -137,12 +166,26 @@ export type CorrectionPredictionTuple = { * available upon initial construction of this type. */ matchLevel?: SuggestionSimilarity; + /** * Text from the triggering input that should _not_ be affected by the * prediction. */ preservationTransform?: Transform; -}; +} + +export interface IntermediateCompositedPrediction { + /** + * Contains the fully composited predictive-text Suggestion and its underlying correction string. + */ + components: CompositedPredictionData; + /** + * Tracks common intermediate prediction data, such as its underlying probabilities and its similarity to the actual context. + */ + metadata: PredictionMetadata; +} + +type IntermediatePrediction = IntermediateCompositedPrediction; /** * An enum to be used when categorizing the level of similarity between @@ -180,15 +223,15 @@ export enum SuggestionSimilarity { exact = 3 } -export function tupleDisplayOrderSort(a: CorrectionPredictionTuple, b: CorrectionPredictionTuple) { +export function tupleDisplayOrderSort(a: IntermediatePrediction, b: IntermediatePrediction) { // Similarity distance - const simDist = (b.matchLevel ?? 0) - (a.matchLevel ?? 0); + const simDist = (b.metadata.matchLevel ?? 0) - (a.metadata.matchLevel ?? 0); if(simDist != 0) { return simDist; } // Probability distance - return b.totalProb - a.totalProb; + return b.metadata.probabilities.total - a.metadata.probabilities.total; } export function determineTraversallessCorrectionSequences( @@ -251,9 +294,9 @@ export function determineTraversallessCorrectionSequences( returnedPredictionData.push({ ...suggestionParams, applyInPost: (p) => { - p.preservationTransform = preservationTransform; - if(transformId) { - p.prediction.sample.transformId = transformId; + p.metadata.preservationTransform = preservationTransform; + if(transformId !== undefined) { + p.components.prediction.transformId = transformId; } } }) @@ -451,7 +494,7 @@ export interface PredictionParameters { * @param entry * @returns */ - applyInPost: (entry: CorrectionPredictionTuple) => void + applyInPost: (entry: IntermediateCompositedPrediction) => void } export function buildCorrectionSequence( @@ -534,11 +577,11 @@ export function determineTokenizedCorrectionSequence( return { ...suggestionParams, - applyInPost: (entry: CorrectionPredictionTuple) => { - entry.preservationTransform = tokenization.taillessTrueKeystroke; + applyInPost: (entry: IntermediateCompositedPrediction) => { + entry.metadata.preservationTransform = tokenization.taillessTrueKeystroke; // // Will need an extra lookup layer if the suggestion is generated from within a cluster. // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization); - entry.prediction.sample.transform.deleteLeft = deleteLeft; + entry.components.prediction.transform.deleteLeft = deleteLeft; } }; } @@ -569,7 +612,7 @@ export async function correctAndEnumerate( /** * The suggestions generated based on the user's input state. */ - rawPredictions: CorrectionPredictionTuple[]; + rawPredictions: IntermediateCompositedPrediction[]; /** * The id of a prior ContextTransition event that triggered a Suggestion found @@ -625,9 +668,8 @@ export async function correctAndEnumerate( const searchModules = tokenizations.map(t => t.tail.searchModule); // Only run the correction search when corrections are enabled. - let rawPredictions: CorrectionPredictionTuple[] = []; + let rawPredictions: IntermediateCompositedPrediction[] = []; let bestCorrectionCost: number; - const correctionPredictionMap: Record> = {}; for await(const match of getBestTokenMatches(searchModules, timer)) { // Corrections obtained: now to predict from them! const tokenization = tokenizations.find(t => t.spaceId == match.spaceId); @@ -659,14 +701,6 @@ export async function correctAndEnumerate( bestCorrectionCost = match.totalCost; } - // If we're getting the same prediction again, it's lower-cost. Update! - let oldPredictionSet = correctionPredictionMap[match.matchString]; - if(oldPredictionSet) { - rawPredictions = rawPredictions.filter((entry) => !oldPredictionSet.find((match) => entry.prediction.sample == match.sample)); - } - - correctionPredictionMap[match.matchString] = predictions.map((entry) => entry.prediction); - rawPredictions = rawPredictions.concat(predictions); if(shouldStopSearchingEarly(bestCorrectionCost, match.totalCost, rawPredictions)) { @@ -687,7 +721,7 @@ export async function correctAndEnumerate( export function shouldStopSearchingEarly( bestCorrectionCost: number, currentCorrectionCost: number, - rawPredictions: CorrectionPredictionTuple[] + rawPredictions: IntermediateCompositedPrediction[] ) { if(currentCorrectionCost >= bestCorrectionCost + CORRECTION_SEARCH_THRESHOLDS.MAX_SEARCH_THRESHOLD) { return true; @@ -703,7 +737,7 @@ export function shouldStopSearchingEarly( // If the best suggestion from the search's current tier fails to beat the worst // pending suggestion from previous tiers, assume all further corrections will // similarly fail to win; terminate the search-loop. - if(rawPredictions[ModelCompositor.MAX_SUGGESTIONS-1].totalProb > Math.exp(-currentCorrectionCost)) { + if(rawPredictions[ModelCompositor.MAX_SUGGESTIONS-1].metadata.probabilities.total > Math.exp(-currentCorrectionCost)) { return true; } } @@ -733,7 +767,7 @@ export function predictFromCorrectionSequence( corrections: ProbabilityMass[], rootContext: Context, transitionId: number -): CorrectionPredictionTuple[] { +): IntermediateCompositedPrediction[] { let predictionPrefixSequence: ProbabilityMass[] = []; let tailPredictions: ProbabilityMass[]; @@ -782,7 +816,7 @@ export function predictFromCorrectionSequence( return []; } - const predictions: CorrectionPredictionTuple[] = tailPredictions.map((p) => { + const predictions: IntermediateCompositedPrediction[] = tailPredictions.map((p) => { // Concat corrections + predictions for their components. const predictionSequence = [...predictionPrefixSequence, p]; const fullPrediction: ProbabilityMass = predictionSequence.reduce((prev, curr) => { @@ -808,10 +842,19 @@ export function predictFromCorrectionSequence( } return { - prediction: fullPrediction, - correction: fullCorrection, - totalProb: fullPrediction.p * fullCorrection.p, - matchLevel: SuggestionSimilarity.none, + components: { + prediction: fullPrediction.sample, + correction: fullCorrection.sample + }, + metadata: { + probabilities: { + prediction: fullPrediction.p, + correction: fullCorrection.p, + total: fullPrediction.p * fullCorrection.p + }, + autoSelectable: correctionValidForAutoSelect(fullCorrection.sample), + matchLevel: SuggestionSimilarity.none + } }; }); @@ -853,17 +896,17 @@ export function applySuggestionCasing(suggestion: Suggestion, baseWord: string, */ export function dedupeSuggestions( lexicalModel: LexicalModel, - rawPredictions: CorrectionPredictionTuple[], + rawPredictions: IntermediateCompositedPrediction[], context: Context ) { const wordbreak = determineModelWordbreaker(lexicalModel); - let suggestionDistribMap: {[key: string]: CorrectionPredictionTuple} = {}; - let suggestionDistribution: CorrectionPredictionTuple[] = []; + let suggestionDistribMap: {[key: string]: IntermediateCompositedPrediction} = {}; + let suggestionDistribution: IntermediateCompositedPrediction[] = []; // Deduplicator + annotator of 'keep' suggestions. for(let tuple of rawPredictions) { - const predictedWord = wordbreak(models.applyTransform(tuple.prediction.sample.transform, context)); + const predictedWord = wordbreak(models.applyTransform(tuple.components.prediction.transform, context)); // Assumption: suggestions that have the same net result should have the // same displayAs string. (We could try to pick the one with highest net @@ -873,7 +916,7 @@ export function dedupeSuggestions( // Merge 'em! const existingSuggestion = suggestionDistribMap[predictedWord]; if(existingSuggestion) { - existingSuggestion.totalProb += tuple.totalProb; + existingSuggestion.metadata.probabilities.total += tuple.metadata.probabilities.total; } else { suggestionDistribMap[predictedWord] = tuple; } @@ -901,15 +944,16 @@ export function dedupeSuggestions( * current text * - any other suggestion * + * @param lexicalModel * @param suggestionDistribution - * @param context - * @param trueInput inputTransform + its assigned probability + * @param baseContext + * @param finalContext * @returns true if an existing suggestion fulfills the role of 'keep'; * otherwise, false. */ export function processSimilarity( lexicalModel: LexicalModel, - suggestionDistribution: CorrectionPredictionTuple[], + suggestionDistribution: IntermediateCompositedPrediction[], context: Context, trueInput: ProbabilityMass ): boolean { @@ -929,38 +973,38 @@ export function processSimilarity( for(let tuple of suggestionDistribution) { // Don't set it unnecessarily; this can have side-effects in some automated tests. if(inputTransform.id !== undefined) { - tuple.prediction.sample.transformId = inputTransform.id; + tuple.components.prediction.transformId = inputTransform.id; } - const predictedWord = wordbreak(models.applyTransform(tuple.prediction.sample.transform, context)); + const predictedWord = wordbreak(models.applyTransform(tuple.components.prediction.transform, context)); // Is the suggestion an exact match (or, "similar enough") to the // actually-typed context? If so, we wish to note this fact and to // prioritize such a suggestion over suggestions that are not. - if(keyed(tuple.correction.sample) == keyedPrefix) { + if(keyed(tuple.components.correction) == keyedPrefix) { if(predictedWord == truePrefix) { // Exact match: it's a perfect 'keep' suggestion. - tuple.matchLevel = SuggestionSimilarity.exact; - keepOption = toAnnotatedSuggestion(lexicalModel, tuple.prediction.sample, 'keep', models.QuoteBehavior.noQuotes); + tuple.metadata.matchLevel = SuggestionSimilarity.exact; + keepOption = toAnnotatedSuggestion(lexicalModel, tuple.components.prediction, 'keep', models.QuoteBehavior.noQuotes); // Indicates that this suggestion exists directly within the lexical // model as a valid suggestion. (We actively display it if it's an // exact match, but hide it if not, only preserving it for reversions // if/when needed.) keepOption.matchesModel = true; - Object.assign(tuple.prediction.sample, keepOption); - keepOption = tuple.prediction.sample as Outcome; + Object.assign(tuple.components.prediction, keepOption); + keepOption = tuple.components.prediction as Outcome; } else if(keyCased(predictedWord) == lowercasedPrefix) { // Case-insensitive match. No diacritic differences; the ONLY difference is casing. - tuple.matchLevel = SuggestionSimilarity.sameText; + tuple.metadata.matchLevel = SuggestionSimilarity.sameText; } else if(keyed(predictedWord) == keyedPrefix) { // Diacritic-insensitive / exact-key match. - tuple.matchLevel = SuggestionSimilarity.sameKey; + tuple.metadata.matchLevel = SuggestionSimilarity.sameKey; } else { - tuple.matchLevel = SuggestionSimilarity.none; + tuple.metadata.matchLevel = SuggestionSimilarity.none; } } else { - tuple.matchLevel = SuggestionSimilarity.none; + tuple.metadata.matchLevel = SuggestionSimilarity.none; } } @@ -986,7 +1030,7 @@ export function createDefaultKeep( lexicalModel: LexicalModel, context: Context, trueInput: ProbabilityMass -): CorrectionPredictionTuple { +): IntermediateCompositedPrediction { const { sample: inputTransform, p: inputTransformProb } = trueInput; const wordbreak = determineModelWordbreaker(lexicalModel); const tokenizer = determineModelTokenizer(lexicalModel); @@ -1022,19 +1066,19 @@ export function createDefaultKeep( // Insert our synthetic keepOption as a prediction tuple. return { - // Product of the two p's below. - totalProb: inputTransformProb * MAX_PROB, - prediction: { - sample: keepOption, - // We always show the keep option if it doesn't directly match, - // so max probability is fine. - p: MAX_PROB, - }, - correction: { - sample: truePrefix, - p: inputTransformProb * MAX_PROB + components: { + prediction: keepOption, + correction: truePrefix }, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: MAX_PROB, + correction: inputTransformProb, + total: inputTransformProb * MAX_PROB + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; } @@ -1067,12 +1111,12 @@ export function correctionValidForAutoSelect(correction: string) { return false; } -export function predictionAutoSelect(suggestionDistribution: CorrectionPredictionTuple[]) { +export function predictionAutoSelect(suggestionDistribution: IntermediateCompositedPrediction[]) { if(suggestionDistribution.length == 0) { return; } - const keepOption = suggestionDistribution[0].prediction.sample as Outcome; + const keepOption = suggestionDistribution[0].components.prediction as Outcome; if(keepOption.tag == 'keep' && keepOption.matchesModel) { // Auto-select it for auto-acceptance; we don't correct away from perfectly-valid // lexical entries, even if they are comparatively low-frequency. @@ -1086,19 +1130,19 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio if(suggestionDistribution.length == 1) { // Prevent auto-acceptance when the root doesn't meet validation criteria. - if(!correctionValidForAutoSelect(suggestionDistribution[0].correction.sample)) { + if(!suggestionDistribution[0].metadata.autoSelectable) { return; } // Mark for auto-acceptance; there are no alternatives. - suggestionDistribution[0].prediction.sample.autoAccept = true; + suggestionDistribution[0].components.prediction.autoAccept = true; return; } // Is it reasonable to auto-accept any of our suggestions? const bestSuggestion = suggestionDistribution[0]; - const baseCorrection = bestSuggestion.correction.sample; + const baseCorrection = bestSuggestion.components.correction; if(baseCorrection.length == 0) { // If the correction is rooted on an empty root, there's no basis for // auto-correcting to this suggestion. @@ -1107,8 +1151,8 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio // Find the highest probability for any correction that led to a valid prediction. // No need to full-on re-sort everything, though. - const bestCorrection = suggestionDistribution.reduce((prev, current) => prev?.correction.p > current.correction.p ? prev : current, null).correction; - if(bestCorrection.p > bestSuggestion.correction.p) { + const bestCorrectionP = suggestionDistribution.reduce((prev, current) => Math.max(prev, current.metadata.probabilities.correction), 0); + if(bestCorrectionP > bestSuggestion.metadata.probabilities.correction) { // Here, the best suggestion didn't come from the best correction. // Is it actually reasonable to auto-correct? We're probably just very // biased toward its frequency. (Maybe a threshold should be considered?) @@ -1119,28 +1163,28 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio // - such as replacing `cant` with `can't` if the latter is much more frequent - // we may wish to group matchLevel values below by 'mapping' them with an appropriate // function. (Both on the next line and within the reduce functor.) - const bestSuggestionTier = bestSuggestion.matchLevel; + const bestSuggestionTier = bestSuggestion.metadata.matchLevel; // compare best vs other probabilities of compatible tier. const probSum = suggestionDistribution.reduce((accum, current) => { // If the suggestion is from a different similarity tier, do not count it against // the required auto-select probability ratio threshold. That threshold should // only apply within the suggestion's tier. - return accum + (current.matchLevel == bestSuggestionTier ? current.totalProb : 0) + return accum + (current.metadata.matchLevel == bestSuggestionTier ? current.metadata.probabilities.total : 0) }, 0); - const proportionOfBest = bestSuggestion.totalProb / probSum; + const proportionOfBest = bestSuggestion.metadata.probabilities.total / probSum; if(proportionOfBest < AUTOSELECT_PROPORTION_THRESHOLD) { return; } - if(!correctionValidForAutoSelect(bestSuggestion.correction.sample)) { + if(!bestSuggestion.metadata.autoSelectable) { return; } // compare correction-cost aspects? We disable if the base correction is lower than best, // but should we do other comparisons too? - bestSuggestion.prediction.sample.autoAccept = true; + bestSuggestion.components.prediction.autoAccept = true; } /** @@ -1161,7 +1205,7 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio */ export function finalizeSuggestions( lexicalModel: LexicalModel, - deduplicatedSuggestionTuples: CorrectionPredictionTuple[], + deduplicatedSuggestionTuples: IntermediateCompositedPrediction[], context: Context, inputTransform: Transform, verbose?: boolean @@ -1170,42 +1214,44 @@ export function finalizeSuggestions( const tokenize = determineModelTokenizer(lexicalModel); const suggestions = deduplicatedSuggestionTuples.map((tuple) => { - const prediction = tuple.prediction; + const prediction = tuple.components.prediction; // If this is a suggestion after any form of wordbreak input, make sure we preserve any components // from prior tokens! // // Note: may need adjustment if/when supporting phrase-level correction. - if(tuple.preservationTransform) { + if(tuple.metadata.preservationTransform) { const mergedTransform = { - ...models.buildMergedTransform(tuple.preservationTransform, {...prediction.sample.transform, deleteLeft: 0}), - deleteLeft: prediction.sample.transform.deleteLeft + ...models.buildMergedTransform(tuple.metadata.preservationTransform, {...prediction.transform, deleteLeft: 0}), + deleteLeft: prediction.transform.deleteLeft }; // Temporarily and locally drops 'readonly' semantics so that we can reassign the transform. // See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers - let mutableSuggestion = prediction.sample as {-readonly [transform in keyof Suggestion]: Suggestion[transform]}; + let mutableSuggestion = prediction as {-readonly [transform in keyof Suggestion]: Suggestion[transform]}; // Assignment via by-reference behavior, as suggestion is an object mutableSuggestion.transform = mergedTransform; } // Is sometimes not set during unit tests. - if(prediction.sample.transformId !== undefined) { - prediction.sample.transform.id = prediction.sample.transformId; + if(prediction.transformId !== undefined) { + prediction.transform.id = prediction.transformId; } + const probs = tuple.metadata.probabilities; + if(!verbose) { return { - ...prediction.sample, - p: tuple.totalProb + ...prediction, + p: probs.total }; } else { const sample: Outcome = { - ...prediction.sample, - p: tuple.totalProb, - "lexical-p": prediction.p, - "correction-p": tuple.correction.p + ...prediction, + p: probs.total, + "lexical-p": probs.prediction, + "correction-p": probs.correction } return sample; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts index bf45e3b94e1..430d9c6c7e0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts @@ -1,6 +1,16 @@ import { assert } from 'chai'; -import { CORRECTION_SEARCH_THRESHOLDS, CorrectionPredictionTuple, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index"; +import { CORRECTION_SEARCH_THRESHOLDS, IntermediateCompositedPrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index"; + +function mockIntermediatePrediction(value: number) { + return { + metadata: { + probabilities: { + total: value + } + } + } as IntermediateCompositedPrediction +} describe('correction-search: shouldStopSearchingEarly', () => { it('stops early once new corrections are less likely than currently discovered predictions', () => { @@ -12,12 +22,7 @@ describe('correction-search: shouldStopSearchingEarly', () => { assert.equal(predictionProbs.length, ModelCompositor.MAX_SUGGESTIONS, "test setup no longer valid"); // The only part for each entry we actually care about here: .totalProb. - /** @type {import('#./predict-helpers.js').CorrectionPredictionTuple[]} */ - const predictions = predictionProbs.map((entry) => { - return { - totalProb: entry - } as CorrectionPredictionTuple - }); + const predictions = predictionProbs.map((entry) => mockIntermediatePrediction(entry)); // Thresholding is performed in log-space. // 0.0501 and 0.0499 are offset on each side of 0.05, the last value in the array defined above. @@ -33,8 +38,8 @@ describe('correction-search: shouldStopSearchingEarly', () => { // // Can technically run the method with an empty array, but the actual scenario would have // at least one prediction present in the "found predictions" array. - assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, [{ totalProb: Math.exp(-1) } as CorrectionPredictionTuple])); - assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, [{ totalProb: Math.exp(-1) } as CorrectionPredictionTuple])); + assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, [mockIntermediatePrediction(Math.exp(-1))])); + assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, [mockIntermediatePrediction(Math.exp(-1))])); }); it('stops checking corrections earlier when enough predictions have been found', () => { @@ -43,11 +48,7 @@ describe('correction-search: shouldStopSearchingEarly', () => { // The only part for each entry we actually care about here: .totalProb. /** @type {import('#./predict-helpers.js').CorrectionPredictionTuple[]} */ - const predictions = predictionProbs.map((entry) => { - return { - totalProb: entry - } as CorrectionPredictionTuple - }); + const predictions = predictionProbs.map((entry) => mockIntermediatePrediction(entry)); const baseCost = 1; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts index d32326e8436..b55886bb42f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts @@ -1,6 +1,6 @@ import { assert } from 'chai'; -import { AUTOSELECT_PROPORTION_THRESHOLD, CorrectionPredictionTuple, predictionAutoSelect, SuggestionSimilarity, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; +import { AUTOSELECT_PROPORTION_THRESHOLD, IntermediateCompositedPrediction, predictionAutoSelect, SuggestionSimilarity, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; /* * Preconditions: * - there should always be a 'keep' option. Now, whether or not that option @@ -9,7 +9,7 @@ import { AUTOSELECT_PROPORTION_THRESHOLD, CorrectionPredictionTuple, predictionA */ describe('predictionAutoSelect', () => { it(`does not throw when no suggestions are available`, () => { - const predictions: CorrectionPredictionTuple[] = []; + const predictions: IntermediateCompositedPrediction[] = []; const originalPredictions = [].concat(predictions); assert.doesNotThrow(() => predictionAutoSelect(predictions)); @@ -17,14 +17,10 @@ describe('predictionAutoSelect', () => { }); it(`selects solitary 'keep' suggestion that does match the model`, () => { - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ { - correction: { - sample: 'apple', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'e', @@ -33,9 +29,16 @@ describe('predictionAutoSelect', () => { matchesModel: true, displayAs: 'apple' }, - p: 1 + correction: 'apple', }, - totalProb: 1 + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 + }, + autoSelectable: true + } } ]; @@ -43,19 +46,15 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isOk(autoselected); }); it(`does not select suggestions if the root correction has no letters`, () => { - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ { - correction: { - sample: '5', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { tag: 'keep', transform: { insert: '5', @@ -64,17 +63,20 @@ describe('predictionAutoSelect', () => { matchesModel: false, displayAs: '5' }, - p: 0.01 + correction: '5' }, - totalProb: 0.01 + metadata: { + probabilities: { + prediction: 0.01, + correction: 1, + total: 0.01 + }, + autoSelectable: false + } }, { - correction: { - sample: '5', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { transform: { insert: '5th', deleteLeft: 0 @@ -82,9 +84,16 @@ describe('predictionAutoSelect', () => { matchesModel: true, displayAs: '5th' }, - p: 0.8 + correction: '5' }, - totalProb: 0.8 + metadata: { + probabilities: { + prediction: 0.8, + correction: 1, + total: 0.8 + }, + autoSelectable: false + } } ]; @@ -92,19 +101,15 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); it(`does not select solitary 'keep' suggestion that doesn't match the model`, () => { - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ { - correction: { - sample: 'appl', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'l', @@ -113,9 +118,16 @@ describe('predictionAutoSelect', () => { matchesModel: false, displayAs: '"appl"' }, - p: 1 + correction: 'appl' }, - totalProb: 1 + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 + }, + autoSelectable: true + } } ]; @@ -123,18 +135,14 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); it(`selects 'keep' suggestion that does match the model over any alternatives`, () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const keepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'n', @@ -143,65 +151,81 @@ describe('predictionAutoSelect', () => { matchesModel: true, displayAs: 'thin' }, - p: .05 + correction: 'thin' }, - totalProb: .04 + metadata: { + probabilities: { + prediction: .05, + correction: .8, + total: .05 * .8 + }, + autoSelectable: true + } } - const highestNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const highestNonKeepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'nk', deleteLeft: 0 }, displayAs: 'think' }, - p: .55 + correction: 'thin' }, - totalProb: .44 + metadata: { + probabilities: { + prediction: .55, + correction: .8, + total: .55 * .8 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ng', deleteLeft: 0 }, displayAs: 'thing' }, - p: .4 + correction: 'thin' }, - totalProb: .32 + metadata: { + probabilities: { + prediction: .4, + correction: .8, + total: .4 * .8 + }, + autoSelectable: true + } }, { - correction: { - sample: 'thic', - p: .2 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ck', deleteLeft: 0 }, displayAs: 'thick' }, - p: 1 + correction: 'thic' }, - totalProb: .2 + metadata: { + probabilities: { + prediction: 1, + correction: .2, + total: 1 * .2 + }, + autoSelectable: true + } } ]; @@ -209,18 +233,14 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.equal(autoselected, keepSuggestion); }); it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const keepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'n', @@ -229,40 +249,50 @@ describe('predictionAutoSelect', () => { displayAs: '"thin"', matchesModel: false }, - p: .05 + correction: 'thin' }, - totalProb: .04 + metadata: { + probabilities: { + prediction: .05, + correction: .8, + total: .8 * .05 + }, + autoSelectable: true + } } // To 'win', a suggestion (currently) needs at least twice the probability of the sum of all alternatives. // This threshold may be subject to change. // // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts. - const onlyNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const onlyNonKeepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'nk', deleteLeft: 0 }, displayAs: 'think' }, - p: .01 + correction: 'thin' }, - totalProb: .008 + metadata: { + probabilities: { + prediction: .01, + correction: .8, + total: .01 * .8 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ keepSuggestion, onlyNonKeepSuggestion ]; - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isBelow(onlyNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); + const totalProb = predictions.reduce((accum, current) => accum + current.metadata.probabilities.total, 0); + assert.isBelow(onlyNonKeepSuggestion.metadata.probabilities.total, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); predictions.sort(tupleDisplayOrderSort); @@ -270,18 +300,14 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.equal(autoselected, onlyNonKeepSuggestion); }); it(`does not select non-'keep' without sufficient winning probability`, () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const keepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'n', @@ -290,74 +316,90 @@ describe('predictionAutoSelect', () => { displayAs: '"thin"', matchesModel: false }, - p: .05 + correction: 'thin' }, - totalProb: .04 + metadata: { + probabilities: { + prediction: .05, + correction: .8, + total: .05 * .8 + }, + autoSelectable: true + } } // To 'win', a suggestion (currently) needs at least twice the probability of the sum of all alternatives. // This threshold may be subject to change. // // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts. - const highestNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const highestNonKeepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'nk', deleteLeft: 0 }, displayAs: 'think' }, - p: .55 + correction: 'thin' }, - totalProb: .44 + metadata: { + probabilities: { + prediction: .55, + correction: .8, + total: .55 * .8 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ng', deleteLeft: 0 }, displayAs: 'thing' }, - p: .4 + correction: 'thin' }, - totalProb: .32 + metadata: { + probabilities: { + prediction: .4, + correction: .8, + total: .4 * .8 + }, + autoSelectable: true + } }, { - correction: { - sample: 'thic', - p: .2 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ck', deleteLeft: 0 }, displayAs: 'thick' }, - p: 1 + correction: 'thic' }, - totalProb: .2 + metadata: { + probabilities: { + prediction: 1, + correction: .2, + total: 1 * .2 + }, + autoSelectable: true + } } ]; - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isBelow(highestNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); + const totalProb = predictions.reduce((accum, current) => accum + current.metadata.probabilities.total, 0); + assert.isBelow(highestNonKeepSuggestion.metadata.probabilities.total, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); predictions.sort(tupleDisplayOrderSort); @@ -365,18 +407,14 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); it(`does select non-'keep' with sufficient winning probability`, () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const keepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'n', @@ -385,87 +423,99 @@ describe('predictionAutoSelect', () => { displayAs: '"thin"', matchesModel: false }, - p: .05 + correction: 'thin' }, - totalProb: .04 + metadata: { + probabilities: { + prediction: .05, + correction: .8, + total: .05 * .8 + }, + autoSelectable: true + } } - const highestNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .9 - }, - prediction: { - sample: { + const highestNonKeepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'nk', deleteLeft: 0 }, displayAs: 'think' }, - p: .75 + correction: 'thin' }, - totalProb: .675 + metadata: { + probabilities: { + prediction: .75, + correction: .9, + total: .75 * .9 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { - correction: { - sample: 'thin', - p: .9 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ng', deleteLeft: 0 }, displayAs: 'thing' }, - p: .2 + correction: 'thin' }, - totalProb: .18 + metadata: { + probabilities: { + prediction: .2, + correction: .9, + total: .2 * .9 + }, + autoSelectable: true + } }, { - correction: { - sample: 'thic', - p: .1 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ck', deleteLeft: 0 }, displayAs: 'thick' }, - p: 1 + correction: 'thic' }, - totalProb: .1 + metadata: { + probabilities: { + prediction: 1, + correction: .1, + total: 1 * .1 + }, + autoSelectable: true + } } ]; - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isAbove(highestNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); + const totalProb = predictions.reduce((accum, current) => accum + current.metadata.probabilities.total, 0); + assert.isAbove(highestNonKeepSuggestion.metadata.probabilities.total, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); const originalPredictions = [].concat(predictions); assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.equal(autoselected, highestNonKeepSuggestion); }); it('ignores non key-matched suggestions when key-matched suggestions exist', () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'cant', - p: 1 - }, - prediction: { - sample: { + const keepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 't', @@ -474,51 +524,64 @@ describe('predictionAutoSelect', () => { displayAs: '"cant"', matchesModel: false }, - p: 1 + correction: 'cant' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: true, + matchLevel: SuggestionSimilarity.exact + } } - const expectedSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'cant', - p: 1 - }, - prediction: { - sample: { + const expectedSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: '\'t', deleteLeft: 0 }, displayAs: "can't" }, - p: .2 + correction: 'cant' }, - totalProb: .2, - matchLevel: SuggestionSimilarity.sameKey + metadata: { + probabilities: { + prediction: .2, + correction: 1, + total: .2 * 1 + }, + autoSelectable: true, + matchLevel: SuggestionSimilarity.sameKey + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ keepSuggestion, expectedSuggestion, { - correction: { - sample: 'cant', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'teen', deleteLeft: 0 }, displayAs: 'canteen' }, - p: .8 + correction: 'cant' }, - totalProb: .8, - matchLevel: SuggestionSimilarity.none + metadata: { + probabilities: { + prediction: .8, + correction: 1, + total: .8 * 1 + }, + autoSelectable: true, + matchLevel: SuggestionSimilarity.none + } } ]; @@ -527,20 +590,16 @@ describe('predictionAutoSelect', () => { assert.sameDeepMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.equal(autoselected, expectedSuggestion); }); // The idea: avoid "over-correcting" when a potential correction has a // super-high-frequency word. it('does not auto-select suggestion if its root correction is not most likely', () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thi', - p: .7 - }, - prediction: { - sample: { + const keepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'i', @@ -549,61 +608,74 @@ describe('predictionAutoSelect', () => { displayAs: '"thi"', matchesModel: false }, - p: .05 + correction: 'thi' }, - totalProb: .035 + metadata: { + probabilities: { + prediction: .05, + correction: .7, + total: .05 * .7 + }, + autoSelectable: true + } } - const highestCorrectionSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thi', - p: .7 - }, - prediction: { - sample: { + const highestCorrectionSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'in', deleteLeft: 0 }, displayAs: 'thin' }, - p: .1 + correction: 'thi', }, - totalProb: .07 + metadata: { + probabilities: { + prediction: .1, + correction: .7, + total: .1 * .7 + }, + autoSelectable: true + } }; - const highestNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'the', - p: .3 - }, - prediction: { - sample: { + const highestNonKeepSuggestion: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'e', deleteLeft: 0 }, displayAs: 'the' }, - p: 1 + correction: 'the' }, - totalProb: .3 + metadata: { + probabilities: { + prediction: 1, + correction: .3, + total: 1 * .3 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: IntermediateCompositedPrediction[] = [ keepSuggestion, highestNonKeepSuggestion, highestCorrectionSuggestion ]; - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isAbove(highestNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); + const totalProb = predictions.reduce((accum, current) => accum + current.metadata.probabilities.total, 0); + assert.isAbove(highestNonKeepSuggestion.metadata.probabilities.total, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); const originalPredictions = [].concat(predictions); assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts index 052c349291c..afc0d4d232f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts @@ -12,7 +12,7 @@ import { assert } from 'chai'; import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; -import { CorrectionPredictionTuple, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index"; +import { IntermediateCompositedPrediction, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index"; import CasingFunction = LexicalModelTypes.CasingFunction; import Context = LexicalModelTypes.Context; @@ -108,13 +108,9 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: CorrectionPredictionTuple = { - correction: { - sample: 'iphone', - p: 1 - }, - prediction: { - sample: { + const expectedKeep: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'iphone', deleteLeft: 5 @@ -123,10 +119,17 @@ describe('createDefaultKeep', () => { matchesModel: false, tag: 'keep' }, - p: 1 + correction: 'iphone' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); @@ -149,13 +152,9 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: CorrectionPredictionTuple = { - correction: { - sample: 'iphone', - p: 1 - }, - prediction: { - sample: { + const expectedKeep: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'iphone', deleteLeft: 7 @@ -164,10 +163,17 @@ describe('createDefaultKeep', () => { matchesModel: false, tag: 'keep' }, - p: 1 + correction: 'iphone' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); @@ -190,13 +196,9 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: CorrectionPredictionTuple = { - correction: { - sample: 'iphone', - p: 1 - }, - prediction: { - sample: { + const expectedKeep: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'iphone', deleteLeft: 8 @@ -205,10 +207,17 @@ describe('createDefaultKeep', () => { matchesModel: false, tag: 'keep' }, - p: 1 + correction: 'iphone' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); @@ -231,13 +240,9 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: CorrectionPredictionTuple = { - correction: { - sample: 'and', - p: 1 - }, - prediction: { - sample: { + const expectedKeep: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'iphones and', deleteLeft: 5 @@ -246,10 +251,17 @@ describe('createDefaultKeep', () => { matchesModel: false, tag: 'keep' }, - p: 1 + correction: 'and' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); @@ -272,13 +284,9 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: CorrectionPredictionTuple = { - correction: { - sample: 'iphones', - p: 1 - }, - prediction: { - sample: { + const expectedKeep: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'iphones', deleteLeft: 7 @@ -287,10 +295,17 @@ describe('createDefaultKeep', () => { matchesModel: false, tag: 'keep' }, - p: 1 + correction: 'iphones' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); @@ -313,13 +328,9 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: CorrectionPredictionTuple = { - correction: { - sample: '', - p: 1 - }, - prediction: { - sample: { + const expectedKeep: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'iphone ', deleteLeft: 5 @@ -328,10 +339,17 @@ describe('createDefaultKeep', () => { matchesModel: false, tag: 'keep' }, - p: 1 + correction: '' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts index ee8e2ba9109..5cd7b67072b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -21,7 +21,7 @@ import { ContextState, ContextToken, ContextTokenization, - CorrectionPredictionTuple, + IntermediateCompositedPrediction, ModelCompositor, TokenizationResultMapping } from "@keymanapp/lm-worker/test-index"; @@ -341,24 +341,27 @@ describe('determineTokenizedCorrectionSequence', () => { }); assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); - const dummiedTuple: CorrectionPredictionTuple = { - prediction: { - sample: { + const dummiedTuple: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'dog', deleteLeft: 0 }, displayAs: 'dog' }, - p: .25 - }, - correction: { - sample: 'd', - p: trueInput.p + correction: 'd' }, - totalProb: .25 * trueInput.p + metadata: { + probabilities: { + prediction: .25, + correction: trueInput.p, + total: .25 * trueInput.p + }, + autoSelectable: true + } }; results.applyInPost(dummiedTuple); - assert.deepEqual(dummiedTuple.preservationTransform, { + assert.deepEqual(dummiedTuple.metadata.preservationTransform, { insert: trueInput.sample.insert.substring(0, KMWString.length(trueInput.sample.insert) - 1), // remove the 'd'. deleteLeft: trueInput.sample.deleteLeft - 1 }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts index f9c7dccd550..a1872641d3c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -13,7 +13,7 @@ import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { KMWString } from 'keyman/common/web-utils'; -import { CorrectionPredictionTuple, ModelCompositor, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; +import { IntermediateCompositedPrediction, ModelCompositor, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import DummyModel = models.DummyModel; @@ -377,24 +377,27 @@ describe('determineTraversallessCorrectionSequences', () => { }); assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); - const dummiedTuple: CorrectionPredictionTuple = { - prediction: { - sample: { + const dummiedTuple: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'dog', deleteLeft: 0 }, displayAs: 'dog' }, - p: .25 + correction: 'd' }, - correction: { - sample: 'd', - p: trueInput.p - }, - totalProb: .25 * trueInput.p + metadata: { + probabilities: { + prediction: .25, + correction: trueInput.p, + total: .25 * trueInput.p + }, + autoSelectable: true + } }; entry.applyInPost(dummiedTuple); - assert.deepEqual(dummiedTuple.preservationTransform, { + assert.deepEqual(dummiedTuple.metadata.preservationTransform, { insert: trueInput.sample.insert.substring(0, KMWString.length(trueInput.sample.insert) - 1), // remove the 'd'. deleteLeft: trueInput.sample.deleteLeft - 1 }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts index 64cea5b2300..f236baaa5d2 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -116,19 +116,19 @@ describe('predictFromCorrectionSequence', () => { const transitionID = 12345; const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); - predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); + predictions.forEach((entry) => assert.equal(entry.components.correction, 'Its')); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 0.6)); predictions.sort(tupleDisplayOrderSort); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((s) => { + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((s) => { delete s.p; s.transformId = transitionID; s.transform.id = transitionID; return s; })); - assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); - assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); + assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001); + assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001); }); it('constructs suggestions matching multiple lexical entries directly - with transform ID', () => { @@ -174,12 +174,12 @@ describe('predictFromCorrectionSequence', () => { }); const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); - predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); + predictions.forEach((entry) => assert.equal(entry.components.correction, 'Its')); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 0.6)); predictions.sort(tupleDisplayOrderSort); - assert.sameOrderedMembers(predictions.map((entry) => entry.prediction.sample.displayAs), ["it's", "its"]); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((entry) => { + assert.sameOrderedMembers(predictions.map((entry) => entry.components.prediction.displayAs), ["it's", "its"]); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((entry) => { entry = deepCopy(entry); entry.transformId = transitionID; entry.transform.id = transitionID; @@ -187,9 +187,9 @@ describe('predictFromCorrectionSequence', () => { return entry; })); - assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); - assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); - predictions.forEach((prediction) => assert.equal(prediction.prediction.sample.transformId, transitionID)); + assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001); + assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001); + predictions.forEach((prediction) => assert.equal(prediction.components.prediction.transformId, transitionID)); }); it('constructs suggestions without input (as if after a context reset)', () => { @@ -227,11 +227,11 @@ describe('predictFromCorrectionSequence', () => { const transitionID = 12345; const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'appl')); - predictions.forEach((entry) => assert.equal(entry.correction.p, 1)); + predictions.forEach((entry) => assert.equal(entry.components.correction, 'appl')); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 1)); predictions.sort(tupleDisplayOrderSort); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((s) => { + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((s) => { delete s.p; s.transformId = transitionID; s.transform.id = transitionID; @@ -318,15 +318,15 @@ describe('predictFromCorrectionSequence', () => { }); const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'golden app')); - predictions.forEach((entry) => assert.equal(entry.correction.p, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + predictions.forEach((entry) => assert.equal(entry.components.correction, 'golden app')); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); predictions.sort(tupleDisplayOrderSort); - assert.equal(predictions[0].prediction.sample.transform.insert, 'golden apple'); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), [expected_prediction.sample]); + assert.equal(predictions[0].components.prediction.transform.insert, 'golden apple'); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), [expected_prediction.sample]); - assert.approximately(predictions[0].prediction.p, expected_prediction.p, 0.00001); - assert.equal(predictions[0].prediction.sample.transformId, transitionID); + assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction.p, 0.00001); + assert.equal(predictions[0].components.prediction.transformId, transitionID); }); it('returns no results if all correction tokens lack predictions', () => { @@ -468,15 +468,15 @@ describe('predictFromCorrectionSequence', () => { // There should be no variations with 'green' or 'gray' apples. assert.equal(predictions.length, 1); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'g app')); - predictions.forEach((entry) => assert.equal(entry.correction.p, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + predictions.forEach((entry) => assert.equal(entry.components.correction, 'g app')); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); predictions.sort(tupleDisplayOrderSort); - assert.equal(predictions[0].prediction.sample.transform.insert, 'golden apple'); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), [expected_prediction.sample]); + assert.equal(predictions[0].components.prediction.transform.insert, 'golden apple'); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), [expected_prediction.sample]); - assert.approximately(predictions[0].prediction.p, expected_prediction.p, 0.00001); - assert.equal(predictions[0].prediction.sample.transformId, transitionID); + assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction.p, 0.00001); + assert.equal(predictions[0].components.prediction.transformId, transitionID); }); it('uses all suggestions generated from context-final correction-tokens', () => { @@ -582,19 +582,19 @@ describe('predictFromCorrectionSequence', () => { const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID); assert.equal(predictions.length, dummied_suggestion_sequences[dummied_suggestion_sequences.length - 1].length); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'golden app')); - predictions.forEach((entry) => assert.equal(entry.correction.p, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + predictions.forEach((entry) => assert.equal(entry.components.correction, 'golden app')); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); predictions.sort(tupleDisplayOrderSort); assert.sameOrderedMembers( - predictions.map((t) => t.prediction.sample.transform.insert), + predictions.map((t) => t.components.prediction.transform.insert), ['golden apple', 'golden application', 'golden appetizer'] ); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), expected_predictions.map((p => p.sample))); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), expected_predictions.map((p => p.sample))); for(let i = 0; i < predictions.length; i++) { - assert.approximately(predictions[i].prediction.p, expected_predictions[i].p, 0.00001, `Expected probabilty mismatch at index ${i}`); - assert.equal(predictions[i].prediction.sample.transformId, transitionID); + assert.approximately(predictions[i].metadata.probabilities.prediction, expected_predictions[i].p, 0.00001, `Expected probabilty mismatch at index ${i}`); + assert.equal(predictions[i].components.prediction.transformId, transitionID); } }); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts index bfd2eeefafc..4a2afcf60dc 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts @@ -4,7 +4,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { deepCopy } from 'keyman/common/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { CorrectionPredictionTuple, dedupeSuggestions, models } from "@keymanapp/lm-worker/test-index"; +import { IntermediateCompositedPrediction, dedupeSuggestions, models } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import DummyModel = models.DummyModel; @@ -24,77 +24,89 @@ const testModel = new DummyModel({ * @returns */ const build_its_is_set = () => { - const its: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const its: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 0 }, displayAs: 'its' }, - p: 0.2 + correction: 'its' }, - totalProb: 0.16 - // matchLevel does not yet exist. + metadata: { + probabilities: { + prediction: .2, + correction: .8, + total: .2 * .8 + }, + autoSelectable: true + // matchLevel does not yet exist. + } }; - const it_is: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const it_is: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: '\'s', deleteLeft: 0 }, displayAs: 'it\'s' }, - p: 0.8 + correction: 'its' }, - totalProb: 0.64 + metadata: { + probabilities: { + prediction: .8, + correction: .8, + total: .8 * .8 + }, + autoSelectable: true + } }; - const is: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 1 }, displayAs: 'is' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; - const is_not: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is_not: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'sn\'t', deleteLeft: 1 }, displayAs: 'isn\'t' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; return { @@ -145,7 +157,7 @@ describe('dedupeSuggestions', () => { // There's no mathematically safe way to combine the components if the // underlying correction sources differ between duplicated suggestions, // though it's mathematically safe to combine their product. - expected.forEach((entry) => entry.totalProb *= (entry.prediction.sample.transform.insert == '\'s') ? 3 : 2); + expected.forEach((entry) => entry.metadata.probabilities.total *= (entry.components.prediction.transform.insert == '\'s') ? 3 : 2); assert.deepEqual(deduplicated, expected); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts index 387bf6dab93..9ba1faec742 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts @@ -5,7 +5,7 @@ import { deepCopy } from 'keyman/common/web-utils'; import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { CorrectionPredictionTuple, finalizeSuggestions, models } from "@keymanapp/lm-worker/test-index"; +import { IntermediateCompositedPrediction, finalizeSuggestions, models } from "@keymanapp/lm-worker/test-index"; import DummyModel = models.DummyModel; import Outcome = LexicalModelTypes.Outcome; @@ -39,6 +39,7 @@ const testModelWithoutSpacing = new DummyModel({ } }); + /** * Builds a fresh copy of test values useful for suggestion-similarity * testing. @@ -47,78 +48,89 @@ const testModelWithoutSpacing = new DummyModel({ */ const build_its_is_set = (verbose?: string) => { const verboseFlag = (verbose == 'verbose' ? true : false); - - const its: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const its: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 0 }, displayAs: 'its' }, - p: 0.2 + correction: 'its' }, - totalProb: 0.16 - // matchLevel does not yet exist. + metadata: { + probabilities: { + prediction: .2, + correction: .8, + total: .2 * .8 + }, + autoSelectable: true + // matchLevel does not yet exist. + } }; - const it_is: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const it_is: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: '\'s', deleteLeft: 0 }, displayAs: 'it\'s' }, - p: 0.8 + correction: 'its' }, - totalProb: 0.64 + metadata: { + probabilities: { + prediction: .8, + correction: .8, + total: .8 * .8 + }, + autoSelectable: true + } }; - const is: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 1 }, displayAs: 'is' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; - const is_not: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is_not: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'sn\'t', deleteLeft: 1 }, displayAs: 'isn\'t' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; const baseDefinitions = { @@ -132,13 +144,13 @@ const build_its_is_set = (verbose?: string) => { const expected = unfinalized.map((entry) => { const mapped: Outcome = { - ...deepCopy(entry.prediction.sample), - p: entry.totalProb + ...deepCopy(entry.components.prediction), + p: entry.metadata.probabilities.total }; if(verboseFlag) { - mapped['correction-p'] = entry.correction.p; - mapped['lexical-p'] = entry.prediction.p; + mapped['correction-p'] = entry.metadata.probabilities.correction; + mapped['lexical-p'] = entry.metadata.probabilities.prediction; } return mapped; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts index 6cfa731429d..6a97039d245 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts @@ -5,7 +5,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { deepCopy } from 'keyman/common/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { CorrectionPredictionTuple, models, processSimilarity, SuggestionSimilarity, toAnnotatedSuggestion } from "@keymanapp/lm-worker/test-index"; +import { IntermediateCompositedPrediction, models, processSimilarity, SuggestionSimilarity, toAnnotatedSuggestion } from "@keymanapp/lm-worker/test-index"; import CasingFunction = LexicalModelTypes.CasingFunction; import Context = LexicalModelTypes.Context; @@ -109,77 +109,89 @@ const testModelWithCasing = new DummyModel({ * @returns */ const build_its_is_set = () => { - const its: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const its: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 0 }, displayAs: 'its' }, - p: 0.2 + correction: 'its' }, - totalProb: 0.16 - // matchLevel does not yet exist. + metadata: { + probabilities: { + prediction: .2, + correction: .8, + total: .2 * .8 + }, + autoSelectable: true + // matchLevel does not yet exist. + } }; - const it_is: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const it_is: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: '\'s', deleteLeft: 0 }, displayAs: 'it\'s' }, - p: 0.8 + correction: 'its' }, - totalProb: 0.64 + metadata: { + probabilities: { + prediction: .8, + correction: .8, + total: .8 * .8 + }, + autoSelectable: true + } }; - const is: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 1 }, displayAs: 'is' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; - const is_not: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is_not: IntermediateCompositedPrediction = { + components: { + prediction: { transform: { insert: 'sn\'t', deleteLeft: 1 }, displayAs: 'isn\'t' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; return { @@ -210,32 +222,22 @@ describe('processSimilarity', () => { const testSet = build_its_is_set(); const distribution = [...Object.values(testSet)]; - const expectation: CorrectionPredictionTuple[] = [ - { - ...testSet.its, - matchLevel: SuggestionSimilarity.exact - }, { - ...testSet.it_is, - matchLevel: SuggestionSimilarity.sameKey - }, { - ...testSet.is, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is_not, - matchLevel: SuggestionSimilarity.none - } - ]; + const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; + expectation[0].metadata.matchLevel = SuggestionSimilarity.exact; // its + expectation[1].metadata.matchLevel = SuggestionSimilarity.sameKey; // it_is + expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is + expectation[3].metadata.matchLevel = SuggestionSimilarity.none; // is_not const its = testSet.its; const original_its = deepCopy(its); - const keep_its = toAnnotatedSuggestion(testModelWithCasing, original_its.prediction.sample, 'keep', QuoteBehavior.noQuotes); + const keep_its = toAnnotatedSuggestion(testModelWithCasing, original_its.components.prediction, 'keep', QuoteBehavior.noQuotes); keep_its.matchesModel = true; processSimilarity(testModelWithCasing, distribution, context, trueInput); assert.sameDeepMembers(distribution, expectation); - assert.equal(its.prediction.sample.tag, 'keep'); - assert.deepEqual(its.prediction.sample, keep_its); + assert.equal(its.components.prediction.tag, 'keep'); + assert.deepEqual(its.components.prediction, keep_its); }); it(`selects contraction as 'more similar' than same-keyed non-contraction when context is contraction`, () => { @@ -257,32 +259,22 @@ describe('processSimilarity', () => { const testSet = build_its_is_set(); const distribution = [...Object.values(testSet)]; - const expectation: CorrectionPredictionTuple[] = [ - { - ...testSet.its, - matchLevel: SuggestionSimilarity.sameKey - }, { - ...testSet.it_is, - matchLevel: SuggestionSimilarity.exact - }, { - ...testSet.is, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is_not, - matchLevel: SuggestionSimilarity.none - } - ]; + const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; + expectation[0].metadata.matchLevel = SuggestionSimilarity.sameKey; // its + expectation[1].metadata.matchLevel = SuggestionSimilarity.exact; // it_is + expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is + expectation[3].metadata.matchLevel = SuggestionSimilarity.none; // is_not const it_is = testSet.it_is; const original_it_is = deepCopy(it_is); - const keep_it_is = toAnnotatedSuggestion(testModelWithCasing, original_it_is.prediction.sample, 'keep', QuoteBehavior.noQuotes); + const keep_it_is = toAnnotatedSuggestion(testModelWithCasing, original_it_is.components.prediction, 'keep', QuoteBehavior.noQuotes); keep_it_is.matchesModel = true; processSimilarity(testModelWithCasing, distribution, context, trueInput); assert.sameDeepMembers(distribution, expectation); - assert.equal(it_is.prediction.sample.tag, 'keep'); - assert.deepEqual(it_is.prediction.sample, keep_it_is); + assert.equal(it_is.components.prediction.tag, 'keep'); + assert.deepEqual(it_is.components.prediction, keep_it_is); }); describe('with casing', () => { @@ -314,34 +306,22 @@ describe('processSimilarity', () => { // Have the predictions replace existing context parts with the lowercased equivalents. Object.values(testSet).forEach((entry) => { - const transform = entry.prediction.sample.transform; + const transform = entry.components.prediction.transform; transform.insert = transform.deleteLeft == 0 ? `it${transform.insert}` : `i${transform.insert}`; transform.deleteLeft = 2; }); const distribution = [...Object.values(testSet)]; - const expectation: CorrectionPredictionTuple[] = [ - { - ...testSet.its, - matchLevel: SuggestionSimilarity.sameKey - }, { - ...testSet.it_is, - // case mismatch, detectable because we have access to a lowercasing/uppercasing function. - matchLevel: SuggestionSimilarity.sameText - }, { - ...testSet.is, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is_not, - matchLevel: SuggestionSimilarity.none - } - ]; - + const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; + expectation[0].metadata.matchLevel = SuggestionSimilarity.sameKey; // its + expectation[1].metadata.matchLevel = SuggestionSimilarity.sameText; // it_is + expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is + expectation[3].metadata.matchLevel = SuggestionSimilarity.none; // is_not processSimilarity(testModelWithCasing, distribution, context, trueInput); // Because we mucked with the casing here, there is no perfect 'keep' match. - const keep = distribution.find((entry) => entry.prediction.sample.tag == 'keep'); + const keep = distribution.find((entry) => entry.components.prediction.tag == 'keep'); assert.isNotOk(keep); assert.sameDeepMembers(distribution, expectation); }); @@ -368,34 +348,20 @@ describe('processSimilarity', () => { // Have the predictions replace existing context parts with the lowercased equivalents. Object.values(testSet).forEach((entry) => { - const transform = entry.prediction.sample.transform; + const transform = entry.components.prediction.transform; transform.insert = transform.deleteLeft == 0 ? `it${transform.insert}` : `i${transform.insert}`; transform.deleteLeft = 2; }); const distribution = [...Object.values(testSet)]; - const expectation: CorrectionPredictionTuple[] = [ - { - ...testSet.its, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.it_is, - // case mismatch, detectable because we have access to a lowercasing/uppercasing function. - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is_not, - matchLevel: SuggestionSimilarity.none - } - ]; + const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; + expectation.forEach((entry) => entry.metadata.matchLevel = SuggestionSimilarity.none); processSimilarity(testModelWithoutCasing, distribution, context, trueInput); // Because we mucked with the casing here, there is no perfect 'keep' match. - const keep = distribution.find((entry) => entry.prediction.sample.tag == 'keep'); + const keep = distribution.find((entry) => entry.components.prediction.tag == 'keep'); assert.isNotOk(keep); assert.sameDeepMembers(distribution, expectation); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts index 928b6e75c47..8c20df9626e 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts @@ -869,6 +869,9 @@ describe('ModelCompositor', function() { deleteLeft: 1 } + // Future adjustment: add the 'baseSuggestion' to DummyModel so that it actually + // returns the suggestion again. + // `new models.DummyModel(..., futureSuggestions: [[baseSuggestion]])` let model = new models.DummyModel({punctuation: englishPunctuation}); let compositor = new ModelCompositor(model, true); @@ -883,6 +886,7 @@ describe('ModelCompositor', function() { // As this test is a bit... 'hard-wired', we only get the 'keep' suggestion. // It should still be accurate, though. + // Can be fixed via the "Future adjustment" noted above. assert.equal(suggestions.length, 1); let expectedTransform = { From 618e6bd9b0d1cbaf0fa381c427e08f1fa5390e64 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 12 May 2026 14:15:17 -0500 Subject: [PATCH 46/65] change(web): support multi-token suggestion similarity Build-bot: skip build:web Test-bot: skip --- .../src/main/model-compositor.ts | 2 +- .../worker-thread/src/main/predict-helpers.ts | 68 +++++++++---------- .../suggestion-similarity.tests.ts | 10 +-- 3 files changed, 37 insertions(+), 43 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index a58f863edc4..efb400f684d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -174,7 +174,7 @@ export class ModelCompositor { const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, rawPredictions, context); // Needs "casing" to be applied first. - const hasExistingKeep = processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, transformDistribution[0]); + const hasExistingKeep = processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, postContext); // If no existing suggestion directly matches the user-visible version of // the token, also add a 'keep' suggestion (with `.matchesModel = false`) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 914f31cb95c..73c13747606 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -954,55 +954,49 @@ export function dedupeSuggestions( export function processSimilarity( lexicalModel: LexicalModel, suggestionDistribution: IntermediateCompositedPrediction[], - context: Context, - trueInput: ProbabilityMass + baseContext: Context, + finalContext: Context ): boolean { - const { sample: inputTransform } = trueInput; const wordbreak = determineModelWordbreaker(lexicalModel); - const postContext = models.applyTransform(inputTransform, context); - const truePrefix = wordbreak(postContext); - const keyed = (text: string) => lexicalModel.toKey ? lexicalModel.toKey(text) : text; const keyCased = (text: string) => lexicalModel.applyCasing ? lexicalModel.applyCasing('lower', text) : text; - const keyedPrefix = keyed(truePrefix); - const lowercasedPrefix = keyCased(truePrefix); + const keyedTarget = keyed(finalContext.left); + const lowercasedTarget = keyCased(finalContext.left); let keepOption: Outcome; - for(let tuple of suggestionDistribution) { - // Don't set it unnecessarily; this can have side-effects in some automated tests. - if(inputTransform.id !== undefined) { - tuple.components.prediction.transformId = inputTransform.id; - } + // If there are no suggestions found, we can't validate that the underlying + // correction was an empty token. + let allCorrectionsEmpty: boolean = suggestionDistribution.length > 0 + ? true + : wordbreak(finalContext) == ''; - const predictedWord = wordbreak(models.applyTransform(tuple.components.prediction.transform, context)); + for(let tuple of suggestionDistribution) { + const appliedContext = models.applyTransform(tuple.components.prediction.transform, baseContext); + allCorrectionsEmpty &&= tuple.components.correction == ''; // Is the suggestion an exact match (or, "similar enough") to the // actually-typed context? If so, we wish to note this fact and to // prioritize such a suggestion over suggestions that are not. - if(keyed(tuple.components.correction) == keyedPrefix) { - if(predictedWord == truePrefix) { - // Exact match: it's a perfect 'keep' suggestion. - tuple.metadata.matchLevel = SuggestionSimilarity.exact; - keepOption = toAnnotatedSuggestion(lexicalModel, tuple.components.prediction, 'keep', models.QuoteBehavior.noQuotes); - - // Indicates that this suggestion exists directly within the lexical - // model as a valid suggestion. (We actively display it if it's an - // exact match, but hide it if not, only preserving it for reversions - // if/when needed.) - keepOption.matchesModel = true; - Object.assign(tuple.components.prediction, keepOption); - keepOption = tuple.components.prediction as Outcome; - } else if(keyCased(predictedWord) == lowercasedPrefix) { - // Case-insensitive match. No diacritic differences; the ONLY difference is casing. - tuple.metadata.matchLevel = SuggestionSimilarity.sameText; - } else if(keyed(predictedWord) == keyedPrefix) { - // Diacritic-insensitive / exact-key match. - tuple.metadata.matchLevel = SuggestionSimilarity.sameKey; - } else { - tuple.metadata.matchLevel = SuggestionSimilarity.none; - } + if(appliedContext.left == finalContext.left) { + // Exact match: it's a perfect 'keep' suggestion. + tuple.metadata.matchLevel = SuggestionSimilarity.exact; + keepOption = toAnnotatedSuggestion(lexicalModel, tuple.components.prediction, 'keep', models.QuoteBehavior.noQuotes); + + // Indicates that this suggestion exists directly within the lexical + // model as a valid suggestion. (We actively display it if it's an + // exact match, but hide it if not, only preserving it for reversions + // if/when needed.) + keepOption.matchesModel = true; + Object.assign(tuple.components.prediction, keepOption); + keepOption = tuple.components.prediction as Outcome; + } else if(keyCased(appliedContext.left) == lowercasedTarget) { + // Case-insensitive match. No diacritic differences; the ONLY difference is casing. + tuple.metadata.matchLevel = SuggestionSimilarity.sameText; + } else if(keyed(appliedContext.left) == keyedTarget) { + // Diacritic-insensitive / exact-key match. + tuple.metadata.matchLevel = SuggestionSimilarity.sameKey; } else { tuple.metadata.matchLevel = SuggestionSimilarity.none; } @@ -1012,7 +1006,7 @@ export function processSimilarity( // // No actual 'keep' needed if the current context token is empty, so we say we // have a 'keep' for that case, even though there isn't really one. - return !!(keepOption || truePrefix == ''); + return !!(keepOption || allCorrectionsEmpty); } /** diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts index 6a97039d245..fe3a4816135 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts @@ -77,7 +77,7 @@ const testModelWithoutCasing = new DummyModel({ .replace(/[โ€œโ€]/g, '"') // ** Difference from model-defaults here ** // And finally, erase single-quotation marks. - .replace(/'/, ''); + .replace(/'/g, ''); } // No suggestions needed here, so we don't define any. }); @@ -233,7 +233,7 @@ describe('processSimilarity', () => { const keep_its = toAnnotatedSuggestion(testModelWithCasing, original_its.components.prediction, 'keep', QuoteBehavior.noQuotes); keep_its.matchesModel = true; - processSimilarity(testModelWithCasing, distribution, context, trueInput); + processSimilarity(testModelWithCasing, distribution, context, models.applyTransform(trueInput.sample, context)); assert.sameDeepMembers(distribution, expectation); assert.equal(its.components.prediction.tag, 'keep'); @@ -270,7 +270,7 @@ describe('processSimilarity', () => { const keep_it_is = toAnnotatedSuggestion(testModelWithCasing, original_it_is.components.prediction, 'keep', QuoteBehavior.noQuotes); keep_it_is.matchesModel = true; - processSimilarity(testModelWithCasing, distribution, context, trueInput); + processSimilarity(testModelWithCasing, distribution, context, models.applyTransform(trueInput.sample, context)); assert.sameDeepMembers(distribution, expectation); assert.equal(it_is.components.prediction.tag, 'keep'); @@ -318,7 +318,7 @@ describe('processSimilarity', () => { expectation[1].metadata.matchLevel = SuggestionSimilarity.sameText; // it_is expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is expectation[3].metadata.matchLevel = SuggestionSimilarity.none; // is_not - processSimilarity(testModelWithCasing, distribution, context, trueInput); + processSimilarity(testModelWithCasing, distribution, context, models.applyTransform(trueInput.sample, context)); // Because we mucked with the casing here, there is no perfect 'keep' match. const keep = distribution.find((entry) => entry.components.prediction.tag == 'keep'); @@ -358,7 +358,7 @@ describe('processSimilarity', () => { const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; expectation.forEach((entry) => entry.metadata.matchLevel = SuggestionSimilarity.none); - processSimilarity(testModelWithoutCasing, distribution, context, trueInput); + processSimilarity(testModelWithoutCasing, distribution, context, models.applyTransform(trueInput.sample, context)); // Because we mucked with the casing here, there is no perfect 'keep' match. const keep = distribution.find((entry) => entry.components.prediction.tag == 'keep'); From b12c912211d7357c95021420a0549bd5d2b45e9f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 4 May 2026 14:39:18 -0500 Subject: [PATCH 47/65] change(web): add tokenized prediction intermediate type for whitespace correction support Converts early uses of CompositedPredictionData to TokenizedPredictionData to facilitate important token-based aspects of whitespace correction support, such as case-handling. Build-bot: skip build:web Test-bot: skip --- .../src/main/model-compositor.ts | 33 +-- .../worker-thread/src/main/predict-helpers.ts | 262 ++++++++++++------ 2 files changed, 183 insertions(+), 112 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index efb400f684d..b4c9dd92c55 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -1,15 +1,13 @@ import * as models from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { TransformUtils } from './transformUtils.js'; -import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; -import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; +import { applySuggestionCasing, compositeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; +import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTracker } from './correction/context-tracker.js'; import { DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL } from './correction/distance-modeler.js'; import { ExecutionTimer } from './correction/execution-timer.js'; -import CasingForm = LexicalModelTypes.CasingForm; import Configuration = LexicalModelTypes.Configuration; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; @@ -125,24 +123,6 @@ export class ModelCompositor { const transformId = inputTransform.id; this.initContextTracker(context, transformId); - const allowBksp = TransformUtils.isBackspace(inputTransform); - const allowWhitespace = TransformUtils.isWhitespace(inputTransform); - - const postContext = models.applyTransform(inputTransform, context); - - // TODO: It would be best for the correctAndEnumerate method to return the - // suggestion's prefix, as it already has lots of logic oriented to this. - // The context-tracker used there with v14+ models can determine this more - // robustly. - const truePrefix = this.wordbreak(postContext); - // Only use of `truePrefix`. - const basePrefix = (allowBksp || allowWhitespace) ? truePrefix : this.wordbreak(context); - - // Used to restore whitespaces if operations would remove them. - const currentCasing: CasingForm = lexicalModel.languageUsesCasing - ? detectCurrentCasing(lexicalModel, postContext) - : null; - // Section 1: determine 'prediction roots' - enumerate corrections from most to least likely, // searching for results that yield viable predictions from the model. @@ -160,9 +140,9 @@ export class ModelCompositor { // Properly capitalizes the suggestions based on the existing context casing state. // This may result in duplicates if multiple casing options exist within the // lexicon for a word. (Example: "Apple" the company vs "apple" the fruit.) - for(let tuple of rawPredictions) { - if(currentCasing && currentCasing != 'lower') { - applySuggestionCasing(tuple.components.prediction, basePrefix, this.lexicalModel, currentCasing); + if(lexicalModel.languageUsesCasing) { + for(let tuple of rawPredictions) { + tuple.components.forEach((component) => applySuggestionCasing(component, this.lexicalModel)); } } @@ -171,9 +151,10 @@ export class ModelCompositor { // We want to dedupe before trimming the list so that we can present a full set // of viable distinct suggestions if available. - const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, rawPredictions, context); + const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, compositeIntermediatePredictions(rawPredictions), context); // Needs "casing" to be applied first. + const postContext = postContextState?.context ?? models.applyTransform(inputTransform, context); const hasExistingKeep = processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, postContext); // If no existing suggestion directly matches the user-visible version of diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 73c13747606..03523c3c1c2 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -4,7 +4,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { searchForProperty, WordBreakProperty } from '@keymanapp/models-wordbreakers'; import { TransformUtils } from './transformUtils.js'; -import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; +import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTokenLike } from './correction/context-token.js'; import { ContextTokenization, mapWhitespacedTokenization } from './correction/context-tokenization.js'; import { ContextTracker } from './correction/context-tracker.js'; @@ -112,6 +112,21 @@ export interface SuggestionReplacement { transitionId?: number } +export interface TokenizedPredictionData { + /** + * The potential Suggestion + */ + prediction: Suggestion, + /** + * The correction upon which the Suggestion is based + */ + correction: string, + /** + * The unkeyed original string underlying the correction/prediction. + */ + casingRoot: string +} + export interface CompositedPredictionData { /** * The potential Suggestion (or Keep) @@ -174,6 +189,19 @@ export interface PredictionMetadata { preservationTransform?: Transform; } +export interface IntermediateTokenizedPrediction { + /** + * Contains the tokenized components to be used to construct a full + * predictive-text Suggestion, as well as data about the source for each + * component. + */ + components: TokenizedPredictionData[]; + /** + * Tracks common intermediate prediction data, such as its underlying probabilities and its similarity to the actual context. + */ + metadata: PredictionMetadata; +} + export interface IntermediateCompositedPrediction { /** * Contains the fully composited predictive-text Suggestion and its underlying correction string. @@ -185,7 +213,7 @@ export interface IntermediateCompositedPrediction { metadata: PredictionMetadata; } -type IntermediatePrediction = IntermediateCompositedPrediction; +type IntermediatePrediction = IntermediateCompositedPrediction | IntermediateTokenizedPrediction; /** * An enum to be used when categorizing the level of similarity between @@ -280,7 +308,6 @@ export function determineTraversallessCorrectionSequences( // But, for now, only actually use the last one. const suggestionParams = buildCorrectionSequence(transitionEffects, context, new TokenizationResultMapping([correctionRoots[correctionRoots.length - 1]], null)); - const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample); const tokenizedCorrection = tokenizationMapping.tokenizedTransform; const tokenizedCorrectionEntries = [...tokenizedCorrection.values()]; @@ -296,7 +323,7 @@ export function determineTraversallessCorrectionSequences( applyInPost: (p) => { p.metadata.preservationTransform = preservationTransform; if(transformId !== undefined) { - p.components.prediction.transformId = transformId; + p.components.forEach((entry) => entry.prediction.transformId = transformId); } } }) @@ -487,27 +514,37 @@ export interface PredictionParameters { * "unchanged" (root) context used for that suggestion will include the * changes from the entry at index 0 (or possibly, a suggestion derived from it). */ - tokenizedCorrection: ProbabilityMass[], + tokens: { + correction: ProbabilityMass, + casingRoot: string, + autoSelectable: boolean + }[], + + deleteLeft: number; /** * A closure to be applied to the generated suggestion's metadata. * @param entry * @returns */ - applyInPost: (entry: IntermediateCompositedPrediction) => void + applyInPost: (entry: IntermediateTokenizedPrediction) => void; } export function buildCorrectionSequence( transitionEffects: SuggestionReplacement, context: Context, tokenizationCorrection: TokenizationResultMapping -) { +): Omit { const { deleteLeft } = transitionEffects; const rootContext = models.applyTransform({insert: '', deleteLeft}, context); // Replace the existing context with the correction. - const tokenizedCorrections = tokenizationCorrection.matchedResult.map((correction, i) => { + const orderedTokens = tokenizationCorrection.matchingSpace?.orderedTokens; + const tokens: PredictionParameters['tokens'] = []; + + for(let i = 0; i < tokenizationCorrection.matchedResult.length; i++) { + const correction = tokenizationCorrection.matchedResult[i]; /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if * there are significantly more likely words. We only need this to allow very minor fat-finger @@ -535,12 +572,17 @@ export function buildCorrectionSequence( entry.sample.id = transitionEffects.transitionId; } - return entry; - }); + tokens.push({ + correction: entry, + casingRoot: orderedTokens ? orderedTokens[i].exampleInput : entry.sample.insert, + autoSelectable: correctionValidForAutoSelect(entry.sample.insert) + }); + } return { rootContext, - tokenizedCorrection: tokenizedCorrections + tokens, + deleteLeft }; } @@ -570,18 +612,18 @@ export function determineTokenizedCorrectionSequence( // The correction should always be based on the most recent external // transform/transcription ID. if(transition.transitionId !== undefined) { - suggestionParams.tokenizedCorrection.forEach((t) => t.sample.id = transition.transitionId); + suggestionParams.tokens.map((t) => t.correction.sample.id = transition.transitionId); } const { deleteLeft } = transitionParams; return { ...suggestionParams, - applyInPost: (entry: IntermediateCompositedPrediction) => { + applyInPost: (entry: IntermediateTokenizedPrediction) => { entry.metadata.preservationTransform = tokenization.taillessTrueKeystroke; // // Will need an extra lookup layer if the suggestion is generated from within a cluster. // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization); - entry.components.prediction.transform.deleteLeft = deleteLeft; + entry.components[0].prediction.transform.deleteLeft = deleteLeft; } }; } @@ -612,7 +654,7 @@ export async function correctAndEnumerate( /** * The suggestions generated based on the user's input state. */ - rawPredictions: IntermediateCompositedPrediction[]; + rawPredictions: IntermediateTokenizedPrediction[]; /** * The id of a prior ContextTransition event that triggered a Suggestion found @@ -632,7 +674,7 @@ export async function correctAndEnumerate( const predictionData = determineTraversallessCorrectionSequences(lexicalModel, transformDistribution, context); return { rawPredictions: predictionData.flatMap((entry) => { - const predictions = predictFromCorrectionSequence(lexicalModel, entry.tokenizedCorrection, entry.rootContext, transformDistribution[0]?.sample.id); + const predictions = predictFromCorrectionSequence(lexicalModel, entry); predictions.forEach((p) => entry.applyInPost(p)); return predictions; }) @@ -668,7 +710,7 @@ export async function correctAndEnumerate( const searchModules = tokenizations.map(t => t.tail.searchModule); // Only run the correction search when corrections are enabled. - let rawPredictions: IntermediateCompositedPrediction[] = []; + let rawPredictions: IntermediateTokenizedPrediction[] = []; let bestCorrectionCost: number; for await(const match of getBestTokenMatches(searchModules, timer)) { // Corrections obtained: now to predict from them! @@ -693,7 +735,7 @@ export async function correctAndEnumerate( const corrector = new TokenizationCorrector(tokenization, suggestionRange.tokensToPredict.length, () => true); const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, new TokenizationResultMapping([match], corrector)); - const predictions = predictFromCorrectionSequence(lexicalModel, predictionPrep.tokenizedCorrection, predictionPrep.rootContext, transition.transitionId); + const predictions = predictFromCorrectionSequence(lexicalModel, predictionPrep); predictions.forEach((p) => predictionPrep.applyInPost(p)); // Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions. @@ -721,7 +763,7 @@ export async function correctAndEnumerate( export function shouldStopSearchingEarly( bestCorrectionCost: number, currentCorrectionCost: number, - rawPredictions: IntermediateCompositedPrediction[] + rawPredictions: IntermediateTokenizedPrediction[] ) { if(currentCorrectionCost >= bestCorrectionCost + CORRECTION_SEARCH_THRESHOLDS.MAX_SEARCH_THRESHOLD) { return true; @@ -764,101 +806,109 @@ export function shouldStopSearchingEarly( */ export function predictFromCorrectionSequence( lexicalModel: LexicalModel, - corrections: ProbabilityMass[], - rootContext: Context, - transitionId: number -): IntermediateCompositedPrediction[] { - let predictionPrefixSequence: ProbabilityMass[] = []; - let tailPredictions: ProbabilityMass[]; - - let currentContext = rootContext; + predictionPrep: PredictionParameters +): IntermediateTokenizedPrediction[] { let successfulPredictions = 0; - for(let i = 0; i < corrections.length; i++) { - const correction = corrections[i].sample; + const correctionTokens = predictionPrep.tokens; + const context = predictionPrep.rootContext; + let currentContext = context; - // Step 2: predict based on the final token. - const predictions = lexicalModel.predict(correction, currentContext); + let prefixProb = 1; + + const predictionComponents = correctionTokens.map((correctionToken, i) => { + const correctionTransform = correctionToken.correction.sample; + const predictions = lexicalModel.predict(correctionTransform, currentContext); + const transitionId = correctionTransform.id; // Failsafe: if there are no matching predictions, create a fake prediction // matching the original text. if(predictions.length != 0) { successfulPredictions++; } else { - predictions.push({ + const failbackSuggestion = { sample: { - transform: correction, - displayAs: correction.insert + transform: correctionTransform, + displayAs: correctionTransform.insert }, // It's not found in the lexicon, so we'll take a low probability for it. // // Edit penalties will be applied via the correction component separately later on. p: Math.exp(-EDIT_DISTANCE_COST_SCALE) - }); + }; + + predictions.push(failbackSuggestion); } - if(i == corrections.length - 1) { - tailPredictions = predictions; - } else { - let bestMatch = predictions.find((p) => KMWString.length(p.sample.transform.insert) == KMWString.length(correction.insert)); - if(!bestMatch) { - bestMatch = predictions[0]; + // Regardless of origin, overwrite the transform's deleteLeft value with what it should actually hold. + predictions.forEach((entry) => { + // Remove the `p` field from the Dummy model's mocked suggestions; these should not be emitted. + delete (entry.sample as Outcome).p; + + entry.sample.transform.deleteLeft = correctionTransform.deleteLeft; + if(transitionId !== undefined) { + entry.sample.transformId = transitionId; + entry.sample.transform.id = transitionId; } + }); + + // Use traversals if possible - extract the most likely entry that is on the traversal, + // rather than predicting (and possibly extending) tokens not adjacent to the caret. + // + // Also, fall back to the actual correction string should prediction not be valid here. + const isLastToken = i == correctionTokens.length - 1; + const predictionsToReturn = isLastToken ? predictions : [predictions[0]]; - predictionPrefixSequence = predictionPrefixSequence.concat(bestMatch); + if(!isLastToken) { + prefixProb *= predictions[0].p; } - // Or maybe per prediction, in some manner? - currentContext = models.applyTransform(correction, currentContext); - } + return predictionsToReturn.map((prediction) => { + return { + prediction: prediction.sample, + correction: correctionTransform.insert, + casingRoot: correctionToken.casingRoot, + // This is tagged on as an addition because we need each final + // token-prediction's probability to be available in the next loop + // below. + predictionProb: prediction.p, + autoSelectable: correctionToken.autoSelectable + }; + }); + }); - if(!successfulPredictions) { + if(successfulPredictions == 0) { return []; } - const predictions: IntermediateCompositedPrediction[] = tailPredictions.map((p) => { - // Concat corrections + predictions for their components. - const predictionSequence = [...predictionPrefixSequence, p]; - const fullPrediction: ProbabilityMass = predictionSequence.reduce((prev, curr) => { - return { - sample: { - transform: models.buildMergedTransform(prev.sample.transform, curr.sample.transform), - displayAs: prev.sample.displayAs + curr.sample.displayAs - }, - p: prev.p * curr.p - }; - }, {sample: {transform: {insert: '', deleteLeft: 0}, displayAs: ''}, p: 1}); + // Constructs a common prefix for all but the final token's component. + const correctionCost = correctionTokens.reduce((accum, curr) => accum * curr.correction.p, 1); + const predictionPrefix = predictionComponents + .slice(0, predictionComponents.length-1) + .map((p) => p[0]); - const fullCorrection: ProbabilityMass = corrections.reduce((prev, curr) => { - return { - sample: prev.sample + curr.sample.insert, - p: prev.p * curr.p - } - }, {sample: '', p: 1}) - - if(transitionId !== undefined) { - fullPrediction.sample.transform.id = transitionId; - fullPrediction.sample.transformId = transitionId; - } + const completePredictionTuples: IntermediateTokenizedPrediction[] = predictionComponents[predictionComponents.length-1].map((tuple) => { + const predictionCost = tuple.predictionProb * prefixProb; - return { - components: { - prediction: fullPrediction.sample, - correction: fullCorrection.sample - }, + const returnVal: IntermediateTokenizedPrediction = { + components: [...predictionPrefix, tuple], metadata: { probabilities: { - prediction: fullPrediction.p, - correction: fullCorrection.p, - total: fullPrediction.p * fullCorrection.p + prediction: predictionCost, + correction: correctionCost, + total: predictionCost * correctionCost }, - autoSelectable: correctionValidForAutoSelect(fullCorrection.sample), + autoSelectable: tuple.autoSelectable, matchLevel: SuggestionSimilarity.none } - }; + } + + returnVal.components[0].prediction.transform.deleteLeft = predictionPrep.deleteLeft; + + return returnVal; }); - return predictions; + return completePredictionTuples; } /** @@ -869,20 +919,60 @@ export function predictFromCorrectionSequence( * @param lexicalModel * @param casingForm */ -export function applySuggestionCasing(suggestion: Suggestion, baseWord: string, lexicalModel: LexicalModel, casingForm: CasingForm) { - // Step 1: does the suggestion replace the whole word? If not, we should extend the suggestion to do so. - let unchangedLength = KMWString.length(baseWord) - suggestion.transform.deleteLeft; +export function applySuggestionCasing(predictionToken: TokenizedPredictionData, lexicalModel: LexicalModel) { + const suggestion = predictionToken.prediction; + + // Step 0: our pattern for generating predictions and corrections already + // enforces them to encompass the whole word. + + // Step 1: detect the original token's casing + let casingForm: CasingForm; - if(unchangedLength > 0) { - suggestion.transform.deleteLeft += unchangedLength; - suggestion.transform.insert = KMWString.substr(baseWord, 0, unchangedLength) + suggestion.transform.insert; + // If we are using the context-tracking engine (when traversals are enabled), + // we just leverage the context token's exampleInput to determine casing. + // + // If it's not available, the correction entry reflects a word-broken piece of + // the original context, with its original casing - so we use that instead. + let casingRoot = predictionToken.casingRoot ? predictionToken.casingRoot : predictionToken.correction; + if(!casingRoot) { + // There's no text in place to verify casing expectations; just leave it + // unchanged. + return; } + casingForm = detectCurrentCasing(lexicalModel, { + left: casingRoot, + startOfBuffer: true, + endOfBuffer: true + }); + // Step 2: Now that the transform affects the whole word, we may safely apply casing rules. suggestion.transform.insert = lexicalModel.applyCasing(casingForm, suggestion.transform.insert); suggestion.displayAs = lexicalModel.applyCasing(casingForm, suggestion.displayAs); } +export function compositeIntermediatePredictions(predictions: IntermediateTokenizedPrediction[]): IntermediateCompositedPrediction[] { + return predictions.map((predictionData) => { + const components = predictionData.components; + + return { + components: components.reduce((total, current) => { + const mergedTransform = models.buildMergedTransform(total.prediction.transform, current.prediction.transform); + const mergedDisplayAs = total.prediction.displayAs + current.prediction.displayAs + + return { + prediction: {...total.prediction, transform: mergedTransform, displayAs: mergedDisplayAs}, + correction: total.correction + current.correction + } + }, { + prediction: {...components[0].prediction, transform: { insert: '', deleteLeft: 0 }, displayAs: ''}, + correction: '' + }), + metadata: predictionData.metadata + }; + }); +} + /** * Given an array of suggestions output from the correction and model-lookup processes, * this function checks for any duplicate suggestions and merges them. From 6b27134eb8ea088e7c3dc194f0edc8265a65b658 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 11 May 2026 16:24:59 -0500 Subject: [PATCH 48/65] fix(web): adjusts existing unit tests to match new intermediate-prediction-data format --- .../early-correction-search-stopping.tests.ts | 14 +- ...ine-tokenized-correction-sequence.tests.ts | 113 +++- ...raversalless-correction-sequences.tests.ts | 157 +++-- .../predict-from-correction-sequence.tests.ts | 624 +++++++++++------- .../worker-thread/suggestion-casing.tests.ts | 208 +++--- 5 files changed, 689 insertions(+), 427 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts index 430d9c6c7e0..9595f15527a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts @@ -1,15 +1,15 @@ import { assert } from 'chai'; -import { CORRECTION_SEARCH_THRESHOLDS, IntermediateCompositedPrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index"; +import { CORRECTION_SEARCH_THRESHOLDS, IntermediateTokenizedPrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index"; -function mockIntermediatePrediction(value: number) { +function mockTokenizedPrediction(value: number) { return { metadata: { probabilities: { total: value } } - } as IntermediateCompositedPrediction + } as IntermediateTokenizedPrediction } describe('correction-search: shouldStopSearchingEarly', () => { @@ -22,7 +22,7 @@ describe('correction-search: shouldStopSearchingEarly', () => { assert.equal(predictionProbs.length, ModelCompositor.MAX_SUGGESTIONS, "test setup no longer valid"); // The only part for each entry we actually care about here: .totalProb. - const predictions = predictionProbs.map((entry) => mockIntermediatePrediction(entry)); + const predictions = predictionProbs.map((entry) => mockTokenizedPrediction(entry)); // Thresholding is performed in log-space. // 0.0501 and 0.0499 are offset on each side of 0.05, the last value in the array defined above. @@ -38,8 +38,8 @@ describe('correction-search: shouldStopSearchingEarly', () => { // // Can technically run the method with an empty array, but the actual scenario would have // at least one prediction present in the "found predictions" array. - assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, [mockIntermediatePrediction(Math.exp(-1))])); - assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, [mockIntermediatePrediction(Math.exp(-1))])); + assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, [mockTokenizedPrediction(Math.exp(-1))])); + assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, [mockTokenizedPrediction(Math.exp(-1))])); }); it('stops checking corrections earlier when enough predictions have been found', () => { @@ -48,7 +48,7 @@ describe('correction-search: shouldStopSearchingEarly', () => { // The only part for each entry we actually care about here: .totalProb. /** @type {import('#./predict-helpers.js').CorrectionPredictionTuple[]} */ - const predictions = predictionProbs.map((entry) => mockIntermediatePrediction(entry)); + const predictions = predictionProbs.map((entry) => mockTokenizedPrediction(entry)); const baseCost = 1; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts index 5cd7b67072b..818a908d1bb 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -21,7 +21,7 @@ import { ContextState, ContextToken, ContextTokenization, - IntermediateCompositedPrediction, + IntermediateTokenizedPrediction, ModelCompositor, TokenizationResultMapping } from "@keymanapp/lm-worker/test-index"; @@ -78,13 +78,17 @@ describe('determineTokenizedCorrectionSequence', () => { endOfBuffer: true }); - assert.deepEqual(results.tokenizedCorrection, [ + assert.deepEqual(results.tokens, [ { - sample: { - insert: 'fo', - deleteLeft: 0 + correction: { + sample: { + insert: 'fo', + deleteLeft: 0 + }, + p: trueInput.p }, - p: trueInput.p + casingRoot: 'fo', + autoSelectable: true } ]); }); @@ -129,12 +133,20 @@ describe('determineTokenizedCorrectionSequence', () => { endOfBuffer: true }); - assert.equal(results.tokenizedCorrection.length, 1); - assert.deepEqual(results.tokenizedCorrection[0].sample, { - insert: ' ', - deleteLeft: 0 - }); - assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + assert.equal(results.tokens.length, 1); + assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: results.tokens[0].correction.p + }, + casingRoot: ' ', + autoSelectable: false + }]); }); it(`properly analyzes common-case word-start - beginning a new token`, () => { @@ -178,12 +190,20 @@ describe('determineTokenizedCorrectionSequence', () => { }); - assert.equal(results.tokenizedCorrection.length, 1); - assert.deepEqual(results.tokenizedCorrection[0].sample, { - insert: 'f', - deleteLeft: 0 - }); - assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + assert.equal(results.tokens.length, 1); + assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: results.tokens[0].correction.p + }, + casingRoot: 'f', + autoSelectable: true + }]); }); it(`properly analyzes post-merge case`, () => { @@ -232,15 +252,17 @@ describe('determineTokenizedCorrectionSequence', () => { endOfBuffer: true }); - assert.deepEqual(results.tokenizedCorrection, [ - { + assert.deepEqual(results.tokens, [{ + correction: { sample: { insert: 'can\'t', deleteLeft: 0 }, p: trueInput.p - } - ]); + }, + casingRoot: 'can\'t', + autoSelectable: true + }]); }); // Will be handled far better after resolving multi-tokenization handling. @@ -285,12 +307,20 @@ describe('determineTokenizedCorrectionSequence', () => { }); - assert.equal(results.tokenizedCorrection.length, 1); - assert.deepEqual(results.tokenizedCorrection[0].sample, { - insert: ' ', - deleteLeft: 0 - }); - assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + assert.equal(results.tokens.length, 1); + assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: results.tokens[0].correction.p + }, + casingRoot: ' ', + autoSelectable: false + }]); }); it(`properly analyzes complex transition - multi-token replacement`, () => { @@ -334,21 +364,34 @@ describe('determineTokenizedCorrectionSequence', () => { // Coming up next - actually providing ALL correction elements, not just the final one. // We're not _quite_ ready for that yet, though. - assert.equal(results.tokenizedCorrection.length, 1); - assert.deepEqual(results.tokenizedCorrection[0].sample, { + assert.equal(results.tokens.length, 1); + assert.deepEqual(results.tokens[0].correction.sample, { insert: 'd', deleteLeft: 0 }); - assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: 'd', + deleteLeft: 0 + }, + p: results.tokens[0].correction.p + }, + casingRoot: 'd', + autoSelectable: true + }]); - const dummiedTuple: IntermediateCompositedPrediction = { - components: { + const dummiedTuple: IntermediateTokenizedPrediction = { + components: [{ prediction: { transform: { insert: 'dog', deleteLeft: 0 }, displayAs: 'dog' }, - correction: 'd' - }, + correction: 'd', + casingRoot: 'd' + }], metadata: { probabilities: { prediction: .25, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts index a1872641d3c..e90fb1845d9 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -13,7 +13,7 @@ import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { KMWString } from 'keyman/common/web-utils'; -import { IntermediateCompositedPrediction, ModelCompositor, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; +import { determineTraversallessCorrectionSequences, IntermediateTokenizedPrediction, ModelCompositor, models } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import DummyModel = models.DummyModel; @@ -80,12 +80,16 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.deepEqual(entry.tokenizedCorrection, [{ - sample: { - insert: 'appl', - deleteLeft: 0 + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: 'appl', + deleteLeft: 0 + }, + p: trueInput.p }, - p: trueInput.p + casingRoot: 'appl', + autoSelectable: true }]); }); @@ -122,13 +126,19 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.deepEqual(entry.tokenizedCorrection, [{ - sample: { - insert: 'iPhone', - deleteLeft: 0 - }, - p: trueInput.p - }]); + assert.deepEqual(predictionRootEntries[0].tokens, [ + { + correction: { + sample: { + insert: 'iPhone', + deleteLeft: 0 + }, + p: trueInput.p + }, + casingRoot: 'iPhone', + autoSelectable: true + } + ]); }); it(`properly analyzes common-case token-extension - adding a letter to an existing word`, () => { @@ -163,13 +173,19 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.deepEqual(entry.tokenizedCorrection, [{ - sample: { - insert: 'fo', - deleteLeft: 0 - }, - p: trueInput.p - }]); + assert.deepEqual(entry.tokens, [ + { + correction: { + sample: { + insert: 'fo', + deleteLeft: 0 + }, + p: trueInput.p + }, + casingRoot: 'fo', + autoSelectable: true + } + ]); }); it(`properly analyzes common-case whitespace - ending a token and adding a new one`, () => { @@ -204,12 +220,20 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.equal(entry.tokenizedCorrection.length, 1); - assert.deepEqual(entry.tokenizedCorrection[0].sample, { - insert: '', - deleteLeft: 0 - }); - assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + assert.equal(entry.tokens.length, 1); + assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: '', + deleteLeft: 0 + }, + p: entry.tokens[0].correction.p + }, + casingRoot: '', + autoSelectable: false + }]); }); @@ -245,12 +269,20 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.equal(entry.tokenizedCorrection.length, 1); - assert.deepEqual(entry.tokenizedCorrection[0].sample, { - insert: 'f', - deleteLeft: 0 - }); - assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + assert.equal(entry.tokens.length, 1); + assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: entry.tokens[0].correction.p + }, + casingRoot: 'f', + autoSelectable: true + }]); }); it(`properly analyzes post-merge case`, () => { @@ -285,12 +317,16 @@ describe('determineTraversallessCorrectionSequences', () => { } ); - assert.deepEqual(entry.tokenizedCorrection, [{ - sample: { - insert: 'can\'t', - deleteLeft: 0 + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: 'can\'t', + deleteLeft: 0 + }, + p: trueInput.p }, - p: trueInput.p + casingRoot: 'can\'t', + autoSelectable: true }]); }); @@ -328,12 +364,20 @@ describe('determineTraversallessCorrectionSequences', () => { // } // ); - assert.equal(entry.tokenizedCorrection.length, 1); - assert.deepEqual(entry.tokenizedCorrection[0].sample, { - insert: '', - deleteLeft: 0 - }); - assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + assert.equal(entry.tokens.length, 1); + assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: '', + deleteLeft: 0 + }, + p: entry.tokens[0].correction.p + }, + casingRoot: '', + autoSelectable: false + }]); }); it(`properly analyzes complex transition - multi-token replacement`, () => { @@ -370,21 +414,34 @@ describe('determineTraversallessCorrectionSequences', () => { // Coming up next - actually providing ALL correction elements, not just the final one. // We're not _quite_ ready for that yet, though. - assert.equal(entry.tokenizedCorrection.length, 1); - assert.deepEqual(entry.tokenizedCorrection[0].sample, { + assert.equal(entry.tokens.length, 1); + assert.deepEqual(entry.tokens[0].correction.sample, { insert: 'd', deleteLeft: 0 }); - assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); - const dummiedTuple: IntermediateCompositedPrediction = { - components: { + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: 'd', + deleteLeft: 0 + }, + p: entry.tokens[0].correction.p + }, + casingRoot: 'd', + autoSelectable: true + }]); + + const dummiedTuple: IntermediateTokenizedPrediction = { + components: [{ prediction: { transform: { insert: 'dog', deleteLeft: 0 }, displayAs: 'dog' }, - correction: 'd' - }, + correction: 'd', + casingRoot: 'd' + }], metadata: { probabilities: { prediction: .25, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts index f236baaa5d2..a31371ac01f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -4,16 +4,12 @@ import { assert } from 'chai'; import { deepCopy } from "keyman/common/web-utils"; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { EDIT_DISTANCE_COST_SCALE, models, predictFromCorrectionSequence, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; +import { EDIT_DISTANCE_COST_SCALE, PredictionParameters, models, predictFromCorrectionSequence, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; import CasingFunction = LexicalModelTypes.CasingFunction; -import Context = LexicalModelTypes.Context; -import Distribution = LexicalModelTypes.Distribution; import DummyModel = models.DummyModel; import Outcome = LexicalModelTypes.Outcome; -import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Suggestion = LexicalModelTypes.Suggestion; -import Transform = LexicalModelTypes.Transform; // See: developer/src/kmc-model/model-defaults.ts, defaultApplyCasing const applyCasing: CasingFunction = (casing, text) => { @@ -75,21 +71,32 @@ const DUMMY_MODEL_CONFIG = { describe('predictFromCorrectionSequence', () => { describe('on a single correction', () => { it('constructs suggestions matching multiple lexical entries directly - no transform ID', () => { - const context: Context = { - left: '', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; + const transitionID = 12345; - const correctionDistribution: Distribution = [{ - sample: { - insert: 'Its', - deleteLeft: 0 - }, - p: 0.6 - } - ]; + const parameters: PredictionParameters = { + rootContext: { + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'Its', + deleteLeft: 0, + id: transitionID + }, + p: 0.6 + }, + casingRoot: '', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; const dummied_suggestions: Outcome[] = [ { @@ -114,13 +121,12 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: [ dummied_suggestions ] }); - const transitionID = 12345; - const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); - predictions.forEach((entry) => assert.equal(entry.components.correction, 'Its')); + const predictions = predictFromCorrectionSequence(model, parameters); + predictions.forEach((entry) => assert.equal(entry.components[0].correction, 'Its')); predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 0.6)); predictions.sort(tupleDisplayOrderSort); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((s) => { + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((s) => { delete s.p; s.transformId = transitionID; s.transform.id = transitionID; @@ -132,23 +138,32 @@ describe('predictFromCorrectionSequence', () => { }); it('constructs suggestions matching multiple lexical entries directly - with transform ID', () => { - const context: Context = { - left: '', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - const transitionID = 314159; - const correctionDistribution: Distribution = [{ - sample: { - insert: 'Its', - deleteLeft: 0, - id: transitionID - }, - p: 0.6 - } - ]; + + const parameters: PredictionParameters = { + rootContext: { + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'Its', + deleteLeft: 0, + id: transitionID + }, + p: 0.6 + }, + casingRoot: '', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; const dummied_suggestions: Outcome[] = [ { @@ -173,42 +188,52 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: [ dummied_suggestions ] }); - const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); - predictions.forEach((entry) => assert.equal(entry.components.correction, 'Its')); + const predictions = predictFromCorrectionSequence(model, parameters); + predictions.forEach((entry) => assert.equal(entry.components[0].correction, 'Its')); predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 0.6)); predictions.sort(tupleDisplayOrderSort); - assert.sameOrderedMembers(predictions.map((entry) => entry.components.prediction.displayAs), ["it's", "its"]); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((entry) => { + assert.sameOrderedMembers(predictions.map((entry) => entry.components[0].prediction.displayAs), ["it's", "its"]); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((entry) => { entry = deepCopy(entry); entry.transformId = transitionID; entry.transform.id = transitionID; - delete entry.p; return entry; })); assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001); assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001); - predictions.forEach((prediction) => assert.equal(prediction.components.prediction.transformId, transitionID)); + predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transformId, transitionID)); }); it('constructs suggestions without input (as if after a context reset)', () => { - const context: Context = { - left: 'appl', - right: '', - startOfBuffer: true, - endOfBuffer: true + const transitionID = 271828; + + const parameters: PredictionParameters = { + rootContext: { + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'appl', + deleteLeft: 0, + id: transitionID + }, + p: 1 + }, + casingRoot: 'appl', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 }; - const correctionDistribution: Distribution = [{ - sample: { - insert: 'appl', - deleteLeft: 4 - }, - p: 1 - } - ]; - const dummied_suggestions: Outcome[] = [ { transform: { @@ -225,60 +250,79 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: [ dummied_suggestions ] }); - const transitionID = 12345; - const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID); - predictions.forEach((entry) => assert.equal(entry.components.correction, 'appl')); + const predictions = predictFromCorrectionSequence(model, parameters); + predictions.forEach((entry) => assert.deepEqual(entry.components.map((c => c.correction)), ['appl'])); predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 1)); predictions.sort(tupleDisplayOrderSort); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((s) => { + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), [dummied_suggestions.map((s) => { delete s.p; s.transformId = transitionID; s.transform.id = transitionID; return s; - })); + })]); }); }); describe('on a sequence of corrections', () => { it('returns results even if some correction tokens lack predictions', () => { - const context: Context = { - left: 'i want to eat a ', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; + const transitionID = 101; - const correctionSequence: Distribution = [ - { - sample: { - insert: 'golden', - deleteLeft: 0 - }, - p: 0.1 - }, { - sample: { - insert: ' ', - deleteLeft: 0 - }, - p: 0.2 - }, { - sample: { - insert: 'app', - deleteLeft: 0 - }, - p: 0.2 - } - ]; + const parameters: PredictionParameters = { + rootContext: { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'g', + deleteLeft: 0, + id: transitionID + }, + p: 0.1 + }, + casingRoot: 'g', + autoSelectable: true + }, { + correction: { + sample: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: ' ', + autoSelectable: true + }, { + correction: { + sample: { + insert: 'apple', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: 'apple', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; const dummied_suggestion_sequences: Outcome[][] = [ [ { transform: { - insert: "golden", + insert: "g", deleteLeft: 0 }, - displayAs: "golden", + displayAs: "g", p: 0.1 } ], @@ -295,70 +339,105 @@ describe('predictFromCorrectionSequence', () => { ] ]; - const transitionID = 101; - const expected_prediction: ProbabilityMass = { - sample: { + const expected_predictions: Suggestion[] = [ + { transform: { - insert: 'golden apple', + insert: 'g', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'g', + transformId: transitionID, + }, { + transform: { + insert: ' ', deleteLeft: 0, id: transitionID }, - displayAs: 'golden apple', + displayAs: ' ', transformId: transitionID - }, p: dummied_suggestion_sequences.map((dist) => { + }, { + transform: { + insert: 'apple', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'apple', + transformId: transitionID + } + ]; + + const expected_prediction_p = dummied_suggestion_sequences.map((dist) => { return dist[0] }).reduce((accum, curr) => { return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) - }, 1) - } + }, 1); const model = new DummyModel({ ...DUMMY_MODEL_CONFIG, futureSuggestions: dummied_suggestion_sequences }); - const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID); - predictions.forEach((entry) => assert.equal(entry.components.correction, 'golden app')); - predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + const predictions = predictFromCorrectionSequence(model, parameters); + predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['g', ' ', 'apple'])); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1))); predictions.sort(tupleDisplayOrderSort); - assert.equal(predictions[0].components.prediction.transform.insert, 'golden apple'); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), [expected_prediction.sample]); + assert.sameDeepOrderedMembers(predictions[0].components.map((c) => c.prediction), expected_predictions); - assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction.p, 0.00001); - assert.equal(predictions[0].components.prediction.transformId, transitionID); + assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction_p, 0.00001); }); it('returns no results if all correction tokens lack predictions', () => { - const context: Context = { - left: 'i want to eat a ', - right: '', - startOfBuffer: true, - endOfBuffer: true + const transitionID = 3; + + const parameters: PredictionParameters = { + rootContext: { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'golden', + deleteLeft: 0, + id: transitionID + }, + p: 0.1 + }, + casingRoot: 'golden', + autoSelectable: true + }, { + correction: { + sample: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: ' ', + autoSelectable: true + }, { + correction: { + sample: { + insert: 'app', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: 'app', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 }; - const correctionSequence: Distribution = [ - { - sample: { - insert: 'golden', - deleteLeft: 0 - }, - p: 0.1 - }, { - sample: { - insert: ' ', - deleteLeft: 0 - }, - p: 0.2 - }, { - sample: { - insert: 'app', - deleteLeft: 0 - }, - p: 0.2 - } - ]; - const dummied_suggestion_sequences: Outcome[][] = [ [], [], @@ -370,39 +449,59 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: dummied_suggestion_sequences }); - const predictions = predictFromCorrectionSequence(model, correctionSequence, context, 3); + const predictions = predictFromCorrectionSequence(model, parameters); assert.deepEqual(predictions, []); }); it('uses only the best suggestion for non-final corrected tokens', () => { - const context: Context = { - left: 'i want to eat a ', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; + const transitionID = 42; - const correctionSequence: Distribution = [ - { - sample: { - insert: 'g', - deleteLeft: 0 - }, - p: 0.1 - }, { - sample: { - insert: ' ', - deleteLeft: 0 - }, - p: 0.2 - }, { - sample: { - insert: 'app', - deleteLeft: 0 - }, - p: 0.2 - } - ]; + const parameters: PredictionParameters = { + rootContext: { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'g', + deleteLeft: 0, + id: transitionID + }, + p: 0.1 + }, + casingRoot: 'g', + autoSelectable: true + }, { + correction: { + sample: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: ' ', + autoSelectable: true + }, { + correction: { + sample: { + insert: 'app', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: 'app', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; const dummied_suggestion_sequences: Outcome[][] = [ [ @@ -442,72 +541,109 @@ describe('predictFromCorrectionSequence', () => { ] ]; - const transitionID = 42; - const expected_prediction: ProbabilityMass = { - sample: { + const expected_prediction_p = dummied_suggestion_sequences + .map((dist) => { + return dist[0] + }).reduce((accum, curr) => { + return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) + }, 1); + + const expected_predictions: Suggestion[] = [ + { transform: { - insert: 'golden apple', + insert: 'golden', deleteLeft: 0, id: transitionID }, - displayAs: 'golden apple', - transformId: 42 - }, p: dummied_suggestion_sequences.map((dist) => { - return dist[0] - }).reduce((accum, curr) => { - return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) - }, 1) - } + displayAs: 'golden', + transformId: transitionID + }, { + transform: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + displayAs: ' ', + transformId: transitionID + }, { + transform: { + insert: 'apple', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'apple', + transformId: transitionID + } + ]; const model = new DummyModel({ ...DUMMY_MODEL_CONFIG, futureSuggestions: dummied_suggestion_sequences }); - const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID); + const predictions = predictFromCorrectionSequence(model, parameters); // There should be no variations with 'green' or 'gray' apples. assert.equal(predictions.length, 1); - predictions.forEach((entry) => assert.equal(entry.components.correction, 'g app')); - predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['g', ' ', 'app'])); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1))); predictions.sort(tupleDisplayOrderSort); - assert.equal(predictions[0].components.prediction.transform.insert, 'golden apple'); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), [expected_prediction.sample]); + assert.deepEqual(predictions[0].components.map((c) => c.prediction.transform.insert), ['golden', ' ', 'apple']); + assert.sameDeepOrderedMembers(predictions[0].components.map((entry) => entry.prediction), expected_predictions); - assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction.p, 0.00001); - assert.equal(predictions[0].components.prediction.transformId, transitionID); + assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction_p, 0.00001); }); it('uses all suggestions generated from context-final correction-tokens', () => { - const context: Context = { - left: 'i want to eat a ', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; + const transitionID = 13; - const correctionSequence: Distribution = [ - { - sample: { - insert: 'golden', - deleteLeft: 0 - }, - p: 0.1 - }, { - sample: { - insert: ' ', - deleteLeft: 0 - }, - p: 0.2 - }, { - sample: { - insert: 'app', - deleteLeft: 0 - }, - p: 0.2 - } - ]; + const parameters: PredictionParameters = { + rootContext: { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'golden', + deleteLeft: 0, + id: transitionID + }, + p: 0.1 + }, + casingRoot: 'golden', + autoSelectable: true + }, { + correction: { + sample: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: ' ', + autoSelectable: true + }, { + correction: { + sample: { + insert: 'app', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: 'app', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; const dummied_suggestion_sequences: Outcome[][] = [ [ @@ -549,29 +685,40 @@ describe('predictFromCorrectionSequence', () => { const tailIndex = dummied_suggestion_sequences.length - 1; - const transitionID = 13; - const expected_predictions: ProbabilityMass[] = dummied_suggestion_sequences[tailIndex].map((p) => { - const expectedText = `golden ${p.transform.insert}`; + const expected_prediction_prefix_p = dummied_suggestion_sequences + .slice(0, dummied_suggestion_sequences.length - 1) + .map((dist) => { + return dist[0] + }).reduce((accum, curr) => { + return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) + }, 1); - return { - sample: { - transform: { - insert: expectedText, - deleteLeft: 0, - id: transitionID - }, - displayAs: expectedText, - transformId: transitionID - }, p: dummied_suggestion_sequences.map((dist) => { - return dist[0] - }).reduce((accum, curr, index) => { - if(tailIndex == index) { - return accum * p.p; - } else { - return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) - } - }, 1) - }; + const expected_prediction_prefix: Suggestion[] = [ + { + transform: { + insert: 'golden', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'golden', + transformId: transitionID + }, { + transform: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + displayAs: ' ', + transformId: transitionID + } + ]; + + const expected_prediction_sequences: Suggestion[][] = dummied_suggestion_sequences[tailIndex].map((p) => { + return [...expected_prediction_prefix, p]; + }); + + const expected_prediction_seq_probs: number[] = dummied_suggestion_sequences[tailIndex].map((p) => { + return p.p * expected_prediction_prefix_p; }); const model = new DummyModel({ @@ -579,22 +726,17 @@ describe('predictFromCorrectionSequence', () => { futureSuggestions: dummied_suggestion_sequences }); - const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID); + const predictions = predictFromCorrectionSequence(model, parameters); assert.equal(predictions.length, dummied_suggestion_sequences[dummied_suggestion_sequences.length - 1].length); - predictions.forEach((entry) => assert.equal(entry.components.correction, 'golden app')); - predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1))); + predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['golden', ' ', 'app'])); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1))); predictions.sort(tupleDisplayOrderSort); - assert.sameOrderedMembers( - predictions.map((t) => t.components.prediction.transform.insert), - ['golden apple', 'golden application', 'golden appetizer'] - ); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), expected_predictions.map((p => p.sample))); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), expected_prediction_sequences); for(let i = 0; i < predictions.length; i++) { - assert.approximately(predictions[i].metadata.probabilities.prediction, expected_predictions[i].p, 0.00001, `Expected probabilty mismatch at index ${i}`); - assert.equal(predictions[i].components.prediction.transformId, transitionID); + assert.approximately(predictions[i].metadata.probabilities.prediction, expected_prediction_seq_probs[i], 0.00001, `Expected probabilty mismatch at index ${i}`); } }); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/suggestion-casing.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/suggestion-casing.tests.ts index dd586eab646..b54b688614a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/suggestion-casing.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/suggestion-casing.tests.ts @@ -13,7 +13,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { applySuggestionCasing, models } from '@keymanapp/lm-worker/test-index'; +import { TokenizedPredictionData, applySuggestionCasing, models } from '@keymanapp/lm-worker/test-index'; import CasingFunction = LexicalModelTypes.CasingFunction; import TrieModel = models.TrieModel; @@ -45,117 +45,137 @@ describe('applySuggestionCasing', function() { ); it('properly cases suggestions with no suggestion root', function() { - let suggestion = { - transform: { - insert: 'the', - deleteLeft: 0 + let suggestion: TokenizedPredictionData[] = [{ + prediction: { + transform: { + insert: 'the', + deleteLeft: 0 + }, + displayAs: 'the' }, - displayAs: 'the' - }; - - applySuggestionCasing(suggestion, '', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'The'); - assert.equal(suggestion.transform.insert, 'The'); - - suggestion = { - transform: { - insert: 'thE', - deleteLeft: 0 - }, - displayAs: 'thE' - }; - - applySuggestionCasing(suggestion, '', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'ThE'); - assert.equal(suggestion.transform.insert, 'ThE'); - - suggestion = { - transform: { - insert: 'the', - deleteLeft: 0 + correction: '', + casingRoot: 'th' + }]; + + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'the'); + assert.equal(suggestion[0].prediction.transform.insert, 'the'); + + suggestion = [{ + prediction: { + transform: { + insert: 'ThE', + deleteLeft: 0 + }, + displayAs: 'ThE' }, - displayAs: 'the' - }; + correction: '', + casingRoot: 'Th' + }]; - applySuggestionCasing(suggestion, '', plainCasedModel, 'upper'); - assert.equal(suggestion.displayAs, 'THE'); - assert.equal(suggestion.transform.insert, 'THE'); + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'ThE'); + assert.equal(suggestion[0].prediction.transform.insert, 'ThE'); }); it('properly cases suggestions that fully replace the suggestion root', function() { - let suggestion = { - transform: { - insert: 'therefore', - deleteLeft: 3 + let suggestion: TokenizedPredictionData[] = [{ + prediction: { + transform: { + insert: 'therefore', + deleteLeft: 3 + }, + displayAs: 'therefore' }, - displayAs: 'therefore' - }; - - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'Therefore'); - assert.equal(suggestion.transform.insert, 'Therefore'); - - suggestion = { - transform: { - insert: 'thereFore', - deleteLeft: 3 + correction: 'The', + casingRoot: 'Th' + }]; + + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'Therefore'); + assert.equal(suggestion[0].prediction.transform.insert, 'Therefore'); + + suggestion = [{ + prediction: { + transform: { + insert: 'thereFore', + deleteLeft: 3 + }, + displayAs: 'thereFore' }, - displayAs: 'thereFore' - }; - - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'ThereFore'); - assert.equal(suggestion.transform.insert, 'ThereFore'); - - suggestion = { - transform: { - insert: 'therefore', - deleteLeft: 3 + correction: 'The', + casingRoot: 'Th' + }]; + + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'ThereFore'); + assert.equal(suggestion[0].prediction.transform.insert, 'ThereFore'); + + suggestion = [{ + prediction: { + transform: { + insert: 'therefore', + deleteLeft: 3 + }, + displayAs: 'therefore' }, - displayAs: 'therefore' - }; + correction: 'THE', + casingRoot: 'TH' + }]; - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'upper'); - assert.equal(suggestion.displayAs, 'THEREFORE'); - assert.equal(suggestion.transform.insert, 'THEREFORE'); + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'THEREFORE'); + assert.equal(suggestion[0].prediction.transform.insert, 'THEREFORE'); }); it('properly cases suggestions that do not fully replace the suggestion root', function() { - let suggestion = { - transform: { - insert: 'erefore', - deleteLeft: 1 + let suggestion: TokenizedPredictionData[] = [{ + prediction: { + transform: { + insert: 'therefore', + deleteLeft: 3 + }, + displayAs: 'therefore' }, - displayAs: 'therefore' - }; + correction: 'The', + casingRoot: 'Th' + }]; // When integrated, the 'the' string comes from a wordbreak operation on the current context. - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'Therefore'); - assert.equal(suggestion.transform.insert, 'Therefore'); - - suggestion = { - transform: { - insert: 'ereFore', - deleteLeft: 1 + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'Therefore'); + assert.equal(suggestion[0].prediction.transform.insert, 'Therefore'); + + suggestion = [{ + prediction: { + transform: { + insert: 'ThereFore', + deleteLeft: 3 + }, + displayAs: 'thereFore' }, - displayAs: 'thereFore' - }; - - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'ThereFore'); - assert.equal(suggestion.transform.insert, 'ThereFore'); - - suggestion = { - transform: { - insert: 'erefore', - deleteLeft: 1 + correction: 'The', + casingRoot: 'Th' + }]; + + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'ThereFore'); + assert.equal(suggestion[0].prediction.transform.insert, 'ThereFore'); + + suggestion = [{ + prediction: { + transform: { + insert: 'therefore', + deleteLeft: 3 + }, + displayAs: 'therefore' }, - displayAs: 'therefore' - }; + correction: 'THE', + casingRoot: 'TH' + }]; - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'upper'); - assert.equal(suggestion.displayAs, 'THEREFORE'); - assert.equal(suggestion.transform.insert, 'THEREFORE'); + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'THEREFORE'); + assert.equal(suggestion[0].prediction.transform.insert, 'THEREFORE'); }); }); \ No newline at end of file From fb0883eee00428e4526e96bd31b920cb40f4f8ff Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 13 May 2026 14:18:17 -0500 Subject: [PATCH 49/65] fix(web): apply original casing-application logic on a per-token basis --- .../worker-thread/src/main/predict-helpers.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 03523c3c1c2..a6785a0daf7 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -947,8 +947,10 @@ export function applySuggestionCasing(predictionToken: TokenizedPredictionData, }); // Step 2: Now that the transform affects the whole word, we may safely apply casing rules. - suggestion.transform.insert = lexicalModel.applyCasing(casingForm, suggestion.transform.insert); - suggestion.displayAs = lexicalModel.applyCasing(casingForm, suggestion.displayAs); + if(casingForm && casingForm != 'lower') { + suggestion.transform.insert = lexicalModel.applyCasing(casingForm, suggestion.transform.insert); + suggestion.displayAs = lexicalModel.applyCasing(casingForm, suggestion.displayAs); + } } export function compositeIntermediatePredictions(predictions: IntermediateTokenizedPrediction[]): IntermediateCompositedPrediction[] { From dbd060acb3194f337866a5a330932f35ac157eae Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 2 Jun 2026 15:14:26 -0500 Subject: [PATCH 50/65] change(web): clean up application of transitionIDs and applyInPost funcs --- .../worker-thread/src/main/predict-helpers.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index a6785a0daf7..aa44c43e82d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -308,6 +308,10 @@ export function determineTraversallessCorrectionSequences( // But, for now, only actually use the last one. const suggestionParams = buildCorrectionSequence(transitionEffects, context, new TokenizationResultMapping([correctionRoots[correctionRoots.length - 1]], null)); + if(transformId !== undefined) { + suggestionParams.tokens.forEach((token) => token.correction.sample.id = transformId); + } + const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample); const tokenizedCorrection = tokenizationMapping.tokenizedTransform; const tokenizedCorrectionEntries = [...tokenizedCorrection.values()]; @@ -322,9 +326,6 @@ export function determineTraversallessCorrectionSequences( ...suggestionParams, applyInPost: (p) => { p.metadata.preservationTransform = preservationTransform; - if(transformId !== undefined) { - p.components.forEach((entry) => entry.prediction.transformId = transformId); - } } }) } @@ -675,7 +676,6 @@ export async function correctAndEnumerate( return { rawPredictions: predictionData.flatMap((entry) => { const predictions = predictFromCorrectionSequence(lexicalModel, entry); - predictions.forEach((p) => entry.applyInPost(p)); return predictions; }) }; @@ -736,7 +736,6 @@ export async function correctAndEnumerate( const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, new TokenizationResultMapping([match], corrector)); const predictions = predictFromCorrectionSequence(lexicalModel, predictionPrep); - predictions.forEach((p) => predictionPrep.applyInPost(p)); // Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions. if(predictions.length > 0 && (bestCorrectionCost === undefined || bestCorrectionCost > match.totalCost)) { @@ -908,6 +907,8 @@ export function predictFromCorrectionSequence( return returnVal; }); + completePredictionTuples.forEach((pt) => predictionPrep.applyInPost(pt)); + return completePredictionTuples; } @@ -1147,6 +1148,7 @@ export function createDefaultKeep( let keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep'); if(inputTransform.id !== undefined) { keepOption.transformId = inputTransform.id; + keepOption.transform.id = inputTransform.id; } keepOption.matchesModel = false; From acec50b2c0b382ad26007d007115686fe11b9cf2 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 2 Jun 2026 16:28:23 -0500 Subject: [PATCH 51/65] fix(web): unit test patchup, dummy model updates for changed dummied-suggestion pattern --- .../src/main/model-compositor.ts | 6 +++ .../prediction/predictionContext.tests.ts | 25 +++++++----- .../create-default-keep.tests.ts | 40 +++++++++++++++++++ .../suggestion-similarity.tests.ts | 40 +++++++++++++++++++ 4 files changed, 100 insertions(+), 11 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index b4c9dd92c55..bffa6a73e92 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -203,6 +203,12 @@ export class ModelCompositor { } } + if(suggestions.filter((s) => s.tag == 'keep').length > 1) { + throw new Error(`Unexpected state: multiple keep suggestions exist: ${JSON.stringify(suggestions.filter((s) => s.tag == 'keep'))}`); + } else if(suggestions.filter((s) => s.tag == 'revert').length > 1) { + throw new Error(`Unexpected state: multiple revert suggestions exist! ${JSON.stringify(suggestions.filter((s) => s.tag == 'revert'))}`); + } + // Store the suggestions on the final token of the current context state (if it exists). // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root. if(postContextState) { diff --git a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.ts b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.ts index 15faf2189fd..530cce052f8 100644 --- a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.ts +++ b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.ts @@ -21,32 +21,32 @@ LMLayerWorker.loadModel(new models.DummyModel({ const appleDummySuggestionSets: Suggestion[][] = [[ // Set 1: { - transform: { insert: 'e', deleteLeft: 0}, + transform: { insert: 'apple', deleteLeft: 0}, displayAs: 'apple', }, { - transform: { insert: 'y', deleteLeft: 0}, + transform: { insert: 'apply', deleteLeft: 0}, displayAs: 'apply' }, { - transform: { insert: 'es', deleteLeft: 0}, + transform: { insert: 'apples', deleteLeft: 0}, displayAs: 'apples' } ], [ // Set 2: { - transform: { insert: 'e', deleteLeft: 0}, + transform: { insert: 'apple', deleteLeft: 0}, displayAs: 'apple', tag: 'keep' }, { - transform: { insert: 'y', deleteLeft: 0}, + transform: { insert: 'apply', deleteLeft: 0}, displayAs: 'apply' }, { - transform: { insert: 's', deleteLeft: 1}, + transform: { insert: 'apps', deleteLeft: 1}, displayAs: 'apps' } ], [ // Set 3: { - transform: { insert: 'ied', deleteLeft: 2}, + transform: { insert: 'applied', deleteLeft: 2}, displayAs: 'applied' } ], [ @@ -101,7 +101,7 @@ describe("PredictionContext", () => { suggestions = updateFake.secondCall.args[0]; assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apples']); assert.isNotOk(suggestions.find((obj) => obj.tag == 'keep')); - assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 0)); + assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 4)); textStore.insertTextBeforeCaret('e'); // appl| + e = apple let transcription = textStore.buildTranscriptionFrom(initialTextStore, null, true); @@ -113,7 +113,7 @@ describe("PredictionContext", () => { suggestions = updateFake.thirdCall.args[0]; assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apps']); assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, 'apple'); - assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'apps'); + assert.isOk(suggestions.find((obj) => obj.displayAs == 'apps')); }); it('ignores outdated predictions', async function () { @@ -140,14 +140,17 @@ describe("PredictionContext", () => { suggestions = updateFake.secondCall.args[0]; assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apples']); assert.isNotOk(suggestions.find((obj) => obj.tag == 'keep')); - assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 0)); + assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 4)); + textStore = SyntheticTextStore.from(initialTextStore); + textStore.insertTextBeforeCaret('e'); const baseTranscription = textStore.buildTranscriptionFrom(initialTextStore, null, true); // Mocking: corresponds to the second set of mocked predictions - round 2 of // 'apple', 'apply', 'apples'. const skippedPromise = langProcessor.predict(baseTranscription, dummiedGetLayer()); + textStore = SyntheticTextStore.from(initialTextStore); textStore.insertTextBeforeCaret('e'); // appl| + e = apple const finalTranscription = textStore.buildTranscriptionFrom(initialTextStore, null, true); @@ -219,7 +222,7 @@ describe("PredictionContext", () => { suggestions = updateFake.firstCall.args[0]; assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apps']); assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, 'apple'); - assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'apps'); + assert.isOk(suggestions.find((obj) => obj.displayAs == 'apps')); // Now for the real test. previousTextState = SyntheticTextStore.from(textState); // snapshot it! diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts index afc0d4d232f..45267649945 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts @@ -92,6 +92,46 @@ const testModelWithCasing = new DummyModel({ }); describe('createDefaultKeep', () => { + it(`creates an 'exact'-match suggestion based on context when no change occurs and no match is found`, () => { + const transformId = 314159; + + const context: Context = { + left: 'appl', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const expectedKeep: IntermediateCompositedPrediction = { + components: { + prediction: { + transform: { + insert: 'appl', + deleteLeft: 4, + id: transformId + }, + transformId, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + correction: 'appl' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, { sample: { insert: '', deleteLeft: 0, id: transformId }, p: 1}); + assert.deepEqual(tuple, expectedKeep); + }); + it(`creates an 'exact'-match suggestion based on simple primary input`, () => { const context: Context = { left: 'iphon', diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts index fe3a4816135..19864ee33e5 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts @@ -277,6 +277,46 @@ describe('processSimilarity', () => { assert.deepEqual(it_is.components.prediction, keep_it_is); }); + it('operates properly when no transition in context occurs', () => { + const transformId = 314159; + + const context: Context = { + left: 'appl', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const distribution: IntermediateCompositedPrediction[] = [ + { + components: { + prediction: { + transform: { + insert: 'apple', + deleteLeft: 4, + id: transformId + }, + transformId, + displayAs: 'apple' + }, + correction: 'appl' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 + }, + autoSelectable: true + } + } + ]; + + const result = processSimilarity(testModelWithCasing, distribution, context, context); + assert.isFalse(result); + assert.equal(distribution[0].metadata.matchLevel, SuggestionSimilarity.none); + }); + describe('with casing', () => { // If we ever add a mode that can force lowercase for certain words even // when the context is title-cased or upper-cased, this scenario would be From 15080a6c332650b8b9476df7dc859f1cea352594 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 12 Jun 2026 09:50:18 -0500 Subject: [PATCH 52/65] fix(web): address issues caught by AI review --- .../worker-thread/src/main/predict-helpers.ts | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index aa44c43e82d..6ef54116737 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -613,7 +613,7 @@ export function determineTokenizedCorrectionSequence( // The correction should always be based on the most recent external // transform/transcription ID. if(transition.transitionId !== undefined) { - suggestionParams.tokens.map((t) => t.correction.sample.id = transition.transitionId); + suggestionParams.tokens.forEach((t) => t.correction.sample.id = transition.transitionId); } const { deleteLeft } = transitionParams; @@ -825,9 +825,9 @@ export function predictFromCorrectionSequence( if(predictions.length != 0) { successfulPredictions++; } else { - const failbackSuggestion = { + const fallbackSuggestion = { sample: { - transform: correctionTransform, + transform: {...correctionTransform}, displayAs: correctionTransform.insert }, // It's not found in the lexicon, so we'll take a low probability for it. @@ -836,7 +836,7 @@ export function predictFromCorrectionSequence( p: Math.exp(-EDIT_DISTANCE_COST_SCALE) }; - predictions.push(failbackSuggestion); + predictions.push(fallbackSuggestion); } // Regardless of origin, overwrite the transform's deleteLeft value with what it should actually hold. @@ -862,6 +862,12 @@ export function predictFromCorrectionSequence( prefixProb *= predictions[0].p; } + // In case future models can use bigrams or similar strategies to adjust predictions based + // on prior tokens, we update the root context for following iterations with the current + // correction. (This could be improved by trying variations of valid predictions, but that's + // currently out-of-scope.) + currentContext = models.applyTransform(correctionToken.correction.sample, currentContext); + return predictionsToReturn.map((prediction) => { return { prediction: prediction.sample, @@ -958,6 +964,15 @@ export function compositeIntermediatePredictions(predictions: IntermediateTokeni return predictions.map((predictionData) => { const components = predictionData.components; + const reduceBaseTransform: Transform = { + insert: '', + deleteLeft: 0 + } + const transformId = predictionData.components[0].prediction.transformId; + if(transformId !== undefined) { + reduceBaseTransform.id = transformId; + } + return { components: components.reduce((total, current) => { const mergedTransform = models.buildMergedTransform(total.prediction.transform, current.prediction.transform); @@ -968,7 +983,7 @@ export function compositeIntermediatePredictions(predictions: IntermediateTokeni correction: total.correction + current.correction } }, { - prediction: {...components[0].prediction, transform: { insert: '', deleteLeft: 0 }, displayAs: ''}, + prediction: {...components[0].prediction, transform: reduceBaseTransform, displayAs: ''}, correction: '' }), metadata: predictionData.metadata From 245f43f52c22515d9d967acac949d770d008f7c3 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 13 May 2026 15:13:42 -0500 Subject: [PATCH 53/65] change(web): adjust TokenizationCorrector spec Build-bot: skip build:web Test-bot: skip --- .../main/correction/tokenization-corrector.ts | 75 ++++++++++++++----- .../correction/tokenization-result-mapping.ts | 58 ++++++++------ .../worker-thread/src/main/predict-helpers.ts | 15 +++- .../tokenization-corrector.tests.ts | 52 ++++--------- .../predict-from-correction-sequence.tests.ts | 11 +-- 5 files changed, 124 insertions(+), 87 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 835f9eef388..9ac64ef4e49 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -14,7 +14,7 @@ import { ContextToken } from "./context-token.js"; import { CorrectionSearchable, PathResult } from "./correction-searchable.js"; import { ContextTokenization } from "./context-tokenization.js"; import { QuotientNodeFinalizer } from "./quotient-node-finalizer.js"; -import { TokenizationResultMapping } from "./tokenization-result-mapping.js"; +import { TokenizationResult, TokenizationResultMapping } from "./tokenization-result-mapping.js"; import { EDIT_DISTANCE_COST_SCALE } from "./distance-modeler.js"; import { MAX_EDIT_THRESHOLD_FACTOR } from "./search-quotient-spur.js"; @@ -46,7 +46,7 @@ export type TokenResult = { * all correctable tokens, generating corrections for the full represented * range. */ -export class TokenizationCorrector implements CorrectionSearchable, TokenizationResultMapping> { +export class TokenizationCorrector implements CorrectionSearchable { public readonly tokenization: ContextTokenization; private readonly tailCorrectionLength: number; @@ -56,6 +56,8 @@ export class TokenizationCorrector implements CorrectionSearchable; private _previousResults: TokenizationResultMapping[] = []; + private _correctableCodepoints: number = 0; + private _correctablesMatched = 0; // fully private public readonly modelsCorrectables: boolean; @@ -65,6 +67,7 @@ export class TokenizationCorrector implements CorrectionSearchable this.tokenLookupMap.get(c.spaceId)); } + get correctableCodepoints(): number { + return this._correctableCodepoints; + } + /** * Returns the token, if it exists, that is considered "predictable". * @@ -142,6 +149,10 @@ export class TokenizationCorrector implements CorrectionSearchable boolean + filterClosure: (token: ContextToken, index?: number) => boolean ) { this.tokenization = tokenization; this.tailCorrectionLength = tailCorrectionLength; @@ -175,16 +186,23 @@ export class TokenizationCorrector implements CorrectionSearchable { // New issue: this mangles the space IDs! We almost certainly need some // sort of proper map to the source token. const searchModule = new QuotientNodeFinalizer(token.searchModule, index == orderedTokens.length - 1); this.tokenLookupMap.set(searchModule.spaceId, token); - const passesFilter = filterClosure(token); + // Index within the token subset being examined. + const passesFilter = filterClosure(token, index); modelsCorrectables ||= passesFilter; if(!passesFilter) { this._uncorrectables.push(searchModule); - } else if(index == tailCorrectionLength - 1) { + return; + } + + this.matchableTokenCount++; + this._correctableCodepoints += searchModule.codepointLength; + if(index == tailCorrectionLength - 1) { // The sole assignment case for this field. It may only be assigned for // the final token, and only if its text is of a form considered // correctable by the filter. @@ -270,13 +288,19 @@ export class TokenizationCorrector implements CorrectionSearchable 0) { + const results = this.collateResults(); + this._previousResults.push(results); + return { + 'type': 'complete', + cost: this.lastTotalCost, + mapping: results + }; + } else { + return { type: 'none' }; + } } } @@ -314,6 +338,8 @@ export class TokenizationCorrector implements CorrectionSearchable correction-string map with the obtained result. @@ -363,8 +391,8 @@ export class TokenizationCorrector implements CorrectionSearchable c == undefined) != -1) { + // If any token lacks a matching lookup value, abort. + if([...this.tokenLookupMap.keys()].find((k) => !this._generatedTokenResults.has(k))) { return { type: 'intermediate', cost: tokenizationCost @@ -376,12 +404,19 @@ export class TokenizationCorrector implements CorrectionSearchable 0) { + const correctionResults = this.collateResults(); + this._previousResults.push(correctionResults); + return { + type: 'complete', + cost: tokenizationCost, + mapping: correctionResults + }; + } else { + return { + type: 'none' + } + } } else { return { type: 'none' diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts index c5588424afe..da8f4c42a02 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts @@ -1,37 +1,51 @@ import { CorrectionResultMapping } from "./correction-result-mapping.js"; import { TokenizationCorrector, TokenResult } from './tokenization-corrector.js'; -export class TokenizationResultMapping implements CorrectionResultMapping> { +export interface TokenizationResult { + tokenCorrections: ReadonlyArray, + totalEditCount: number, + totalEditableCodepoints: number +} + +export class TokenizationResultMapping implements CorrectionResultMapping { readonly matchingSpace: TokenizationCorrector; - readonly matchedResult: ReadonlyArray; + readonly matchedResult: TokenizationResult; constructor(tokenization: TokenResult[], corrector?: TokenizationCorrector) { this.matchingSpace = corrector; - this.matchedResult = tokenization; + + this.matchedResult = { + tokenCorrections: tokenization, + totalEditCount: tokenization.reduce((accum, curr) => accum + curr.knownCost, 0), + // If based on a legacy/custom model not using traversals, we don't + // support edit operations (for correction) beyond the direct results of + // the most recent input distribution. + totalEditableCodepoints: corrector?.correctableCodepoints ?? 0 + } } get spaceId(): number { return this.matchingSpace?.tokenization.spaceId; } - // /** - // * Gets the number of Damerau-Levenshtein edits needed to reach the node's - // * matchString from the output induced by the input sequence used to reach it. - // * - // * (This is scaled by `SearchSpace.EDIT_DISTANCE_COST_SCALE` when included in - // * `totalCost`.) - // */ - // get knownCost(): number { - // return this.node.editCount; - // } - - // /** - // * Gets the "input sampling cost" of the edge, which should be considered as the - // * negative log-likelihood of the input path taken to reach the node. - // */ - // get inputSamplingCost(): number { - // return this.node.inputSamplingCost; - // } + /** + * Gets the number of Damerau-Levenshtein edits needed to reach the node's + * matchString from the output induced by the input sequence used to reach it. + * + * (This is scaled by `SearchSpace.EDIT_DISTANCE_COST_SCALE` when included in + * `totalCost`.) + */ + get knownCost(): number { + return this.matchedResult.totalEditCount; + } + + /** + * Gets the "input sampling cost" of the edge, which should be considered as the + * negative log-likelihood of the input path taken to reach the node. + */ + get inputSamplingCost(): number { + return this.matchedResult.tokenCorrections.reduce((accum, curr) => accum + curr.inputSamplingCost, 0); + } /** * Gets the "total cost" of the edge, which should be considered as the @@ -40,6 +54,6 @@ export class TokenizationResultMapping implements CorrectionResultMapping total + curr.totalCost, 0); + return this.matchedResult.tokenCorrections.reduce((total, curr) => total + curr.totalCost, 0); } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 6ef54116737..aa0cd46db3b 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -544,8 +544,8 @@ export function buildCorrectionSequence( const orderedTokens = tokenizationCorrection.matchingSpace?.orderedTokens; const tokens: PredictionParameters['tokens'] = []; - for(let i = 0; i < tokenizationCorrection.matchedResult.length; i++) { - const correction = tokenizationCorrection.matchedResult[i]; + for(let i = 0; i < tokenizationCorrection.matchedResult.tokenCorrections.length; i++) { + const correction = tokenizationCorrection.matchedResult.tokenCorrections[i]; /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if * there are significantly more likely words. We only need this to allow very minor fat-finger @@ -817,9 +817,18 @@ export function predictFromCorrectionSequence( const predictionComponents = correctionTokens.map((correctionToken, i) => { const correctionTransform = correctionToken.correction.sample; - const predictions = lexicalModel.predict(correctionTransform, currentContext); + let predictions = lexicalModel.predict(correctionTransform, currentContext); const transitionId = correctionTransform.id; + // Ensure codepointLength == prediction codepoint length if i does not match the tail! + // Filter out cases that do not conform to this condition. + if(i != correctionTokens.length - 1) { + predictions = predictions.filter((p) => { + const codepointLength = KMWString.length(correctionToken.correction.sample.insert); + return KMWString.length(p.sample.transform.insert) == codepointLength; + }); + } + // Failsafe: if there are no matching predictions, create a fake prediction // matching the original text. if(predictions.length != 0) { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts index 4cf66137e74..8949eb7a166 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts @@ -29,7 +29,8 @@ import { SubstitutionQuotientSpur, TokenizationCorrector, TokenResult, - TokenizationResultMapping + TokenizationResultMapping, + TokenizationResult } from '@keymanapp/lm-worker/test-index'; import Distribution = LexicalModelTypes.Distribution; @@ -302,7 +303,7 @@ describe('TokenizationCorrector', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; + const tokenResults = mapping.matchedResult.tokenCorrections; assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); assert.equal(tokenResults.length, 1); @@ -327,7 +328,7 @@ describe('TokenizationCorrector', () => { assert.equal(searchResult.type, 'none'); }); - it('finds a default correction for a single correctable token without a model match', () => { + it('returns no result when a single correctable token lacks a model match', () => { const fixture = buildFixture_therefore(); const theref = fixture.theref.tail; @@ -371,23 +372,6 @@ describe('TokenizationCorrector', () => { searchResult = instance.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.equal(searchResult.type, 'complete'); - if(searchResult.type == 'complete') { - const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; - assert.isNotNaN(searchResult.cost); - assert.equal(searchResult.cost, searchResult.mapping.totalCost); - assert.equal(tokenResults.length, 1); - assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['therefxyz']); - - // Now that an entry has been found, verify the corrector's state. - assert.isNotOk(instance.predictableToken); // should become an uncorrectable. - assert.isTrue(instance.generatedTokenResults.has(therefxyz)); - assert.equal(instance.generatedTokenResults.get(therefxyz), tokenResults[0]); - } - - // There should be no further possible suggestions. - searchResult = instance.handleNextNode(); assert.equal(searchResult.type, 'none'); }); @@ -411,7 +395,7 @@ describe('TokenizationCorrector', () => { let firstResults: ReadonlyArray; if(searchResult.type == 'complete') { const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; + const tokenResults = mapping.matchedResult.tokenCorrections; firstResults = tokenResults; assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); @@ -434,7 +418,7 @@ describe('TokenizationCorrector', () => { searchResult = instance.handleNextNode(); if(searchResult.type == 'complete') { const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; + const tokenResults = mapping.matchedResult.tokenCorrections; // Verify that the first (bound) token is not altered further. // It should receive no further correction attempts. @@ -445,7 +429,7 @@ describe('TokenizationCorrector', () => { } while(searchResult.type != 'none'); }); - it('immediately returns a single result when the only represented token is uncorrectable', () => { + it('immediately returns with no result when the only represented token is uncorrectable', () => { const fixture = buildFixture_terminalWhitespace(); const tokenization = fixture.spaceOnly; @@ -457,13 +441,7 @@ describe('TokenizationCorrector', () => { ); const searchResult = instance.handleNextNode(); - assert.equal(searchResult.type, 'complete'); - if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.matchedResult[0].matchString, ' '); - } - - const nilResult = instance.handleNextNode(); - assert.equal(nilResult.type, 'none'); + assert.equal(searchResult.type, 'none'); }); it('returns a single result when the final token is uncorrectable', () => { @@ -484,8 +462,8 @@ describe('TokenizationCorrector', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.matchedResult[0].matchString, 'space'); - assert.equal(searchResult.mapping.matchedResult[1].matchString, ' '); + assert.equal(searchResult.mapping.matchedResult.tokenCorrections[0].matchString, 'space'); + assert.equal(searchResult.mapping.matchedResult.tokenCorrections[1].matchString, ' '); } const nilResult = instance.handleNextNode(); @@ -502,20 +480,20 @@ describe('TokenizationCorrector', () => { let haveSeenSingleTokenCorrection = false; let haveSeenThreeTokenCorrection = false; for await(let phraseMatch of getBestMatches< - ReadonlyArray, + TokenizationResult, TokenizationResultMapping, TokenizationCorrector >(correctors, buildTestTimer())) { - if(phraseMatch.matchedResult.length == 1) { + if(phraseMatch.matchedResult.tokenCorrections.length == 1) { if(!haveSeenSingleTokenCorrection) { - assert.sameOrderedMembers(phraseMatch.matchedResult.map((t) => t.matchString), ['theref' /* -ore */]); + assert.sameOrderedMembers(phraseMatch.matchedResult.tokenCorrections.map((t) => t.matchString), ['theref' /* -ore */]); } haveSeenSingleTokenCorrection = true; - } else if(phraseMatch.matchedResult.length == 3) { + } else if(phraseMatch.matchedResult.tokenCorrections.length == 3) { if(!haveSeenThreeTokenCorrection) { - assert.sameOrderedMembers(phraseMatch.matchedResult.map((t) => t.matchString), ['the', ' ', 'ef' /* -fort */]); + assert.sameOrderedMembers(phraseMatch.matchedResult.tokenCorrections.map((t) => t.matchString), ['the', ' ', 'ef' /* -fort */]); } haveSeenThreeTokenCorrection = true; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts index a31371ac01f..5f751f50d07 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -542,8 +542,9 @@ describe('predictFromCorrectionSequence', () => { ]; const expected_prediction_p = dummied_suggestion_sequences - .map((dist) => { - return dist[0] + .map((dist, i) => { + // There is no valid 'g' entry corresponding to token index 0. + return i == 0 ? null : dist[0] }).reduce((accum, curr) => { return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) }, 1); @@ -551,11 +552,11 @@ describe('predictFromCorrectionSequence', () => { const expected_predictions: Suggestion[] = [ { transform: { - insert: 'golden', + insert: 'g', deleteLeft: 0, id: transitionID }, - displayAs: 'golden', + displayAs: 'g', transformId: transitionID }, { transform: { @@ -589,7 +590,7 @@ describe('predictFromCorrectionSequence', () => { predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1))); predictions.sort(tupleDisplayOrderSort); - assert.deepEqual(predictions[0].components.map((c) => c.prediction.transform.insert), ['golden', ' ', 'apple']); + assert.deepEqual(predictions[0].components.map((c) => c.prediction.transform.insert), ['g', ' ', 'apple']); assert.sameDeepOrderedMembers(predictions[0].components.map((entry) => entry.prediction), expected_predictions); assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction_p, 0.00001); From c2816da1d94029511514f1502ae8f66fc3bffe6e Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 4 Jun 2026 11:06:49 -0500 Subject: [PATCH 54/65] change(web): remove unused transition-edit field Build-bot: skip build:web Test-bot: skip --- .../main/correction/context-tokenization.ts | 20 -------- .../context/context-state.tests.ts | 1 + .../context/context-tokenization.tests.ts | 51 +------------------ .../context/transition-helpers.tests.ts | 1 - ...ine-suggestion-context-transition.tests.ts | 1 - 5 files changed, 3 insertions(+), 71 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 175c0eb202e..cb5af5a4954 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -108,18 +108,6 @@ export class ContextTokenization { */ readonly tokens: ContextToken[]; - /** - * Denotes whether or not the transition to this tokenization added or deleted - * any tokens. - */ - readonly transitionEdits?: { - addedNewTokens: boolean, - removedOldTokens: boolean, - // NOTE: slated for removal in an upcoming PR. Exists in this form to - // facilitate factorization of the changes into smaller bodies of work. - editedTokenCount: number - }; - /** * The portion of edits from the true input keystroke that are not part of the * final entry in `token`. If `null`, all edits are considered part of the @@ -147,18 +135,10 @@ export class ContextTokenization { throw new Error("ContextTokenization requires at least one existing ContextToken"); } this.tokens = [].concat(tokens); - if(tokenizationPath) { - this.transitionEdits = { - addedNewTokens: tokenizationPath?.inputs[0].sample.has(1) ?? false, - removedOldTokens: (tokenizationPath?.alignment.removedTokenCount ?? 0) > 0, - editedTokenCount: tokenizationPath?.inputs[0].sample.size - } - } this.taillessTrueKeystroke = taillessTrueKeystroke; } else { const priorToClone = param1; this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry)); - this.transitionEdits = priorToClone.transitionEdits ? {...priorToClone.transitionEdits} : null; this.taillessTrueKeystroke = priorToClone.taillessTrueKeystroke; } } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts index 677b541181a..23750ea44a0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts @@ -248,6 +248,7 @@ describe('ContextState', () => { assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. + assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 44ad985578f..1353255e75f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -26,7 +26,6 @@ import { ExtendedEditOperation, generateSubsetId, models, - TransitionEdge, SearchQuotientSpur, traceInsertEdits, LegacyQuotientSpur @@ -100,7 +99,6 @@ describe('ContextTokenization', function() { let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text)))); assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); - assert.isNotOk(tokenization.transitionEdits); assert.equal(tokenization.tail.exampleInput, 'day'); assert.isFalse(tokenization.tail.isWhitespace); }); @@ -108,36 +106,11 @@ describe('ContextTokenization', function() { it("constructs from a token array + alignment data", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const tokens = rawTextTokens.map((text => toTransformToken(text))); - const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; - // We _could_ flesh this out a bit more... but it's not really needed for this test. - const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec); - let transitionEdits: TransitionEdge = { - alignment: { - merges: [], - splits: [], - unmappedEdits: [], - edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)}, - removedTokenCount: 0 - }, - inputs: [{sample: (() => { - const map = new Map(); - map.set(0, emptyTransform); - return map; - })(), p: 1}], - inputSubsetId: generateSubsetId() - }; - - let tokenization = new ContextTokenization(tokens, transitionEdits, null /* dummy val */); + let tokenization = new ContextTokenization(tokens, null, null /* dummy val */); assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); - assert.isOk(tokenization.transitionEdits); - assert.deepEqual(tokenization.transitionEdits, { - addedNewTokens: false, - removedOldTokens: false, - editedTokenCount: 1 - }); assert.equal(tokenization.tail.exampleInput, 'day'); assert.isFalse(tokenization.tail.isWhitespace); }); @@ -145,27 +118,7 @@ describe('ContextTokenization', function() { it('clones', () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const tokens = rawTextTokens.map((text => toTransformToken(text))); - const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; - - // We _could_ flesh this out a bit more... but it's not really needed for this test. - const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec); - let transitionEdits: TransitionEdge = { - alignment: { - merges: [], - splits: [], - unmappedEdits: [], - edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)}, - removedTokenCount: 0 - }, - inputs: [{sample: (() => { - const map = new Map(); - map.set(0, emptyTransform); - return map; - })(), p: 1}], - inputSubsetId: generateSubsetId() - }; - - let baseTokenization = new ContextTokenization(tokens, transitionEdits, null /* dummy val */); + let baseTokenization = new ContextTokenization(tokens, null, null /* dummy val */); let cloned = new ContextTokenization(baseTokenization); assert.sameOrderedMembers( diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts index 882fc5a9d40..7e5c0ef8350 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts @@ -354,7 +354,6 @@ function assertMatchingToken(actual: ContextToken, expected: ContextToken, msg: function assertMatchingTokenization(actual: ContextTokenization, expected: ContextTokenization, msg: string) { assert.equal(actual.tokens.length, expected.tokens.length, msg); assert.deepEqual(actual.exampleInput, expected.exampleInput, msg); - assert.deepEqual(actual.transitionEdits, expected.transitionEdits, msg); for(let j=0; j < actual.tokens.length; j++) { const nestedMsg = `${msg}, token ${j}`; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index d232d90b1a7..88479bc7475 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -104,7 +104,6 @@ describe('determineContextTransition', () => { assert.equal(transition, tracker.latest); assert.isFalse(warningEmitterSpy.called); assert.sameOrderedMembers(transition.final.displayTokenization.exampleInput, ['this', ' ', 'is', ' ', 'for', ' ', 'techn']); - assert.isOk(transition.final.displayTokenization.transitionEdits); assert.equal(transition.final.context.left, targetContext.left); assert.equal(transition.final.context.right ?? "", targetContext.right ?? ""); assert.sameDeepOrderedMembers(transition.inputDistribution, inputDistribution); From bbce66b02d01a80c3a2d61a1551a811a7beb199a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 4 Jun 2026 13:54:04 -0500 Subject: [PATCH 55/65] change(web): remove .preservationTransform, .taillessTrueKeystroke Now that much of the infrastructure for handling boundary-correction is in place, we have improved techniques for ensuring that text before the current token is maintained when applying suggestions. Furthermore, the old techniques being removed were based on the assumption of there being "one true tokenization" - an assumption that will soon be _very_ much invalid. These will get in the way if we maintain them any longer, and so it is a good time to remove them. Build-bot: skip build:web Test-bot: skip --- .../main/correction/context-tokenization.ts | 84 +------------------ .../src/main/correction/transition-helpers.ts | 2 +- .../worker-thread/src/main/predict-helpers.ts | 49 +---------- .../context/context-state.tests.ts | 12 --- .../context/context-tokenization.tests.ts | 4 +- .../context/transition-helpers.tests.ts | 6 +- .../tokenization-corrector.tests.ts | 2 +- ...ine-suggestion-context-transition.tests.ts | 1 - .../determine-suggestion-range.tests.ts | 28 ++----- ...ine-tokenized-correction-sequence.tests.ts | 6 -- ...raversalless-correction-sequences.tests.ts | 6 -- 11 files changed, 18 insertions(+), 182 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index cb5af5a4954..30ccc523c2f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -108,26 +108,11 @@ export class ContextTokenization { */ readonly tokens: ContextToken[]; - /** - * The portion of edits from the true input keystroke that are not part of the - * final entry in `token`. If `null`, all edits are considered part of the - * final token's contents. - * - * If the final token is new due to a newly-introduced wordboundary traversed - * by the keystroke, this will generally be set to an empty transform that - * 'finalizes' the previous tail token. - * - * (Refer to #12494 for an example case.) - */ - readonly taillessTrueKeystroke: Transform; - constructor(priorToClone: ContextTokenization); constructor(tokens: ContextToken[]); - constructor(tokens: ContextToken[], alignment: TransitionEdge, taillessTrueKeystroke: Transform); + constructor(tokens: ContextToken[], alignment: TransitionEdge); constructor( - param1: ContextToken[] | ContextTokenization, - tokenizationPath?: TransitionEdge, - taillessTrueKeystroke?: Transform + param1: ContextToken[] | ContextTokenization ) { if(!(param1 instanceof ContextTokenization)) { const tokens = param1; @@ -135,11 +120,9 @@ export class ContextTokenization { throw new Error("ContextTokenization requires at least one existing ContextToken"); } this.tokens = [].concat(tokens); - this.taillessTrueKeystroke = taillessTrueKeystroke; } else { const priorToClone = param1; this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry)); - this.taillessTrueKeystroke = priorToClone.taillessTrueKeystroke; } } @@ -391,7 +374,7 @@ export class ContextTokenization { tokenization.push(token); } - return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tokenization), null, this.taillessTrueKeystroke); + return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tokenization)); } /** @@ -486,11 +469,7 @@ export class ContextTokenization { affectedToken = null; } - return new ContextTokenization( - this.tokens.slice(0, sliceIndex).concat(tailTokenization), - null, - determineTaillessTrueKeystroke(transitionEdge) - ); + return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tailTokenization)); } } @@ -1175,59 +1154,4 @@ export function assembleTransforms(stackedInserts: string[], stackedDeletes: num } return transformMap; -} - -/** - * Used to construct and represent the part of the incoming transform that does - * not land as part of the final token in the resulting context. This component - * should be preserved by any suggestions that get applied. - * @param tokenizationAnalysis - * @returns - */ -export function determineTaillessTrueKeystroke(tokenizationAnalysis: TransitionEdge) { - // undefined by default; we haven't yet determined if we're still affecting - // the same token that was the tail in the previous tokenization state. - let taillessTrueKeystroke: Transform; - - // If tokens were inserted, emit an empty transform; this prevents - // suggestions from replacing the "current" token. - const bestTokenizedInput = tokenizationAnalysis.inputs[0].sample; - if(bestTokenizedInput.has(1)) { - // Sets a default transform that will be returned even if the main - // transform body lies entirely within a new token. - taillessTrueKeystroke = { insert: '', deleteLeft: 0 }; - - // While the .size() > 1 case could also land here, it is ALSO covered - // by the loop that follows, without fail. - } - - const transformKeys = [...tokenizationAnalysis.inputs[0].sample.keys()]; - transformKeys.pop(); - - for(let i of transformKeys) { - /* - * Thinking ahead to multitokenization: - * - * If what we have is not on the "true" tokenization, then... we need to - * do multitoken effects, right? We're basing new suggestions based on a - * state that does not currently exist! We'd need to enforce THAT state, - * *then* do the suggestion! - * - Which gets fun if we auto-apply such a case, as the new "true" tokenization - * no longer results directly from the true input. - * - * If we give tokens unique IDs on first creation, we could backtrace to - * find the most recent common ancestor. - * - simple cases (same 'token', but different input transform lengths/effects) - * will have the same prior token ID - */ - const primaryInput = tokenizationAnalysis.inputs[0].sample.get(i); - if(!taillessTrueKeystroke) { - taillessTrueKeystroke = {...primaryInput}; - } else { - taillessTrueKeystroke.insert += primaryInput.insert; - taillessTrueKeystroke.deleteLeft += primaryInput.deleteLeft; - } - } - - return taillessTrueKeystroke; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts index ad520fb0418..2f8cfa0f84a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts @@ -130,7 +130,7 @@ export function transitionTokenizations( // Following call: is actually designed to build SubstitutionQuotientSpurs. const transitionedTokenization = rootTokenization.evaluateTransition(precomp[1], trueInput.id, bestProb); - const remadeTokenization = new ContextTokenization(transitionedTokenization.tokens, subset.transitionEdges.get(rootTokenization), transitionedTokenization.taillessTrueKeystroke); + const remadeTokenization = new ContextTokenization(transitionedTokenization.tokens, subset.transitionEdges.get(rootTokenization)); // If the last token is empty and has no flag for a revertable transition, // attempt to copy the previous token's revertable transition flag. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index aa0cd46db3b..cbbf22d1cc7 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -6,7 +6,7 @@ import { searchForProperty, WordBreakProperty } from '@keymanapp/models-wordbrea import { TransformUtils } from './transformUtils.js'; import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTokenLike } from './correction/context-token.js'; -import { ContextTokenization, mapWhitespacedTokenization } from './correction/context-tokenization.js'; +import { ContextTokenization } from './correction/context-tokenization.js'; import { ContextTracker } from './correction/context-tracker.js'; import { ContextState, determineContextSlideTransform } from './correction/context-state.js'; import { ContextTransition } from './correction/context-transition.js'; @@ -181,12 +181,6 @@ export interface PredictionMetadata { * available upon initial construction of this type. */ matchLevel?: SuggestionSimilarity; - - /** - * Text from the triggering input that should _not_ be affected by the - * prediction. - */ - preservationTransform?: Transform; } export interface IntermediateTokenizedPrediction { @@ -312,21 +306,9 @@ export function determineTraversallessCorrectionSequences( suggestionParams.tokens.forEach((token) => token.correction.sample.id = transformId); } - const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample); - const tokenizedCorrection = tokenizationMapping.tokenizedTransform; - const tokenizedCorrectionEntries = [...tokenizedCorrection.values()]; - - // IF: array has multiple entries, then build the preservation-transform as below, including the deleteLeft. - // If not, don't make one! - const preservationTransform = tokenizedCorrectionEntries.slice(0, -1).reduce((accum, curr) => { - return { insert: accum.insert + curr.insert, deleteLeft: accum.deleteLeft + curr.deleteLeft }; - }, { insert: '', deleteLeft: 0, id: correction.sample.id}); - returnedPredictionData.push({ ...suggestionParams, - applyInPost: (p) => { - p.metadata.preservationTransform = preservationTransform; - } + applyInPost: (p) => {} }) } @@ -616,16 +598,9 @@ export function determineTokenizedCorrectionSequence( suggestionParams.tokens.forEach((t) => t.correction.sample.id = transition.transitionId); } - const { deleteLeft } = transitionParams; - return { ...suggestionParams, - applyInPost: (entry: IntermediateTokenizedPrediction) => { - entry.metadata.preservationTransform = tokenization.taillessTrueKeystroke; - // // Will need an extra lookup layer if the suggestion is generated from within a cluster. - // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization); - entry.components[0].prediction.transform.deleteLeft = deleteLeft; - } + applyInPost: (entry) => {} }; } @@ -1328,24 +1303,6 @@ export function finalizeSuggestions( const suggestions = deduplicatedSuggestionTuples.map((tuple) => { const prediction = tuple.components.prediction; - // If this is a suggestion after any form of wordbreak input, make sure we preserve any components - // from prior tokens! - // - // Note: may need adjustment if/when supporting phrase-level correction. - if(tuple.metadata.preservationTransform) { - const mergedTransform = { - ...models.buildMergedTransform(tuple.metadata.preservationTransform, {...prediction.transform, deleteLeft: 0}), - deleteLeft: prediction.transform.deleteLeft - }; - - // Temporarily and locally drops 'readonly' semantics so that we can reassign the transform. - // See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers - let mutableSuggestion = prediction as {-readonly [transform in keyof Suggestion]: Suggestion[transform]}; - - // Assignment via by-reference behavior, as suggestion is an object - mutableSuggestion.transform = mergedTransform; - } - // Is sometimes not set during unit tests. if(prediction.transformId !== undefined) { prediction.transform.id = prediction.transformId; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts index 23750ea44a0..9b777b5c1e2 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts @@ -247,9 +247,6 @@ describe('ContextState', () => { let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve the added whitespace when predicting a token that follows after it. - - assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch?.final; @@ -275,8 +272,6 @@ describe('ContextState', () => { let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve the added whitespace when predicting a token that follows after it. - assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch?.final; @@ -319,7 +314,6 @@ describe('ContextState', () => { let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: '', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch.final; @@ -345,8 +339,6 @@ describe('ContextState', () => { let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve the added whitespace when predicting a token that follows after it. - assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch.final; @@ -376,8 +368,6 @@ describe('ContextState', () => { let newContextMatch = baseState.analyzeTransition(existingContext, [{sample: transform, p: 1}]); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve all text preceding the new token when applying a suggestion. - assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: 'd ', deleteLeft: 0}); // The 'wordbreak' transform let state = newContextMatch.final; @@ -401,8 +391,6 @@ describe('ContextState', () => { let newContextMatch = baseState.analyzeTransition(existingContext, [{sample: transform, p: 1}]); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve all text preceding the new token when applying a suggestion. - assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: 'tor ', deleteLeft: 0 }); // The 'wordbreak' transform let state = newContextMatch.final; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 1353255e75f..af1d90aa3a7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -107,7 +107,7 @@ describe('ContextTokenization', function() { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const tokens = rawTextTokens.map((text => toTransformToken(text))); - let tokenization = new ContextTokenization(tokens, null, null /* dummy val */); + let tokenization = new ContextTokenization(tokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); @@ -118,7 +118,7 @@ describe('ContextTokenization', function() { it('clones', () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const tokens = rawTextTokens.map((text => toTransformToken(text))); - let baseTokenization = new ContextTokenization(tokens, null, null /* dummy val */); + let baseTokenization = new ContextTokenization(tokens); let cloned = new ContextTokenization(baseTokenization); assert.sameOrderedMembers( diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts index 7e5c0ef8350..0d161c6e84f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts @@ -175,11 +175,7 @@ function generateFixtureForTokenizationOutboundTransition ( // CURRENTLY NOT DONE: adding new or replacement tokens for text to be placed after 'quotientNodeToExtend'. - const transitionedTokenization = new ContextTokenization( - srcTokenization.tokens.slice(0, srcTokenization.tokens.length - 1 + relativeTailIndex).concat(token), - tokenizationEdge, - null - ); + const transitionedTokenization = new ContextTokenization(srcTokenization.tokens.slice(0, srcTokenization.tokens.length - 1 + relativeTailIndex).concat(token)); return { /** diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts index 8949eb7a166..4b7577662c0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts @@ -359,7 +359,7 @@ describe('TokenizationCorrector', () => { p: 1 } const therefxyz = new ContextToken(new SubstitutionQuotientSpur(therefxy, [zInput], zInput)); - const therefxyzTokenization = new ContextTokenization([therefxyz], null, null); + const therefxyzTokenization = new ContextTokenization([therefxyz]); const instance = new TokenizationCorrector( therefxyzTokenization, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index 88479bc7475..8cdbed20b24 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -107,7 +107,6 @@ describe('determineContextTransition', () => { assert.equal(transition.final.context.left, targetContext.left); assert.equal(transition.final.context.right ?? "", targetContext.right ?? ""); assert.sameDeepOrderedMembers(transition.inputDistribution, inputDistribution); - assert.isNotOk(transition.final.displayTokenization.taillessTrueKeystroke); assert.equal(transition.transitionId, 1); } finally { warningEmitterSpy.restore(); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts index e18174895ec..f13dd8c3c61 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts @@ -47,8 +47,6 @@ function buildQuickBrownFixture() { baseTokenization.tokens.slice(0, baseTokenCount-1).concat( new ContextToken(new LegacyQuotientSpur(baseTokenization.tail.searchModule, plainInsertDistrib, plainInsertDistrib[0])) ), - null, - null ); const newTokenInsertDistrib: Distribution = [ @@ -58,8 +56,6 @@ function buildQuickBrownFixture() { baseTokenization.tokens.slice(0, baseTokenCount).concat( new ContextToken(new LegacyQuotientSpur(new LegacyQuotientRoot(plainModel), newTokenInsertDistrib, newTokenInsertDistrib[0])) ), - null, - null ); const charReplaceDistrib: Distribution = [ @@ -69,9 +65,7 @@ function buildQuickBrownFixture() { const charReplaceTokenization = new ContextTokenization( baseTokenization.tokens.slice(0, baseTokenCount - 1).concat( new ContextToken(new LegacyQuotientSpur(baseTokenization.tail.searchModule, charReplaceDistrib, charReplaceDistrib[0])) - ), - null, - null + ) ); const eraseTokenDistrib: Distribution = [ @@ -80,9 +74,7 @@ function buildQuickBrownFixture() { const eraseTokenTokenization = new ContextTokenization( baseTokenization.tokens.slice(0, baseTokenCount - 1).concat( new ContextToken(new LegacyQuotientRoot(plainModel)) - ), - null, - null + ) ); const del5Insert5Distrib: Distribution = [ @@ -91,9 +83,7 @@ function buildQuickBrownFixture() { const del5Insert5Tokenization = new ContextTokenization( baseTokenization.tokens.slice(0, baseTokenCount - 3).concat( new ContextToken(new LegacyQuotientSpur(baseTokenization.tokens[baseTokenCount-2].searchModule, del5Insert5Distrib, del5Insert5Distrib[0])) - ), - null, - null + ) ); const deleteToBoundDistrib: Distribution = [ @@ -102,9 +92,7 @@ function buildQuickBrownFixture() { const deleteToBoundTokenization = new ContextTokenization( baseTokenization.tokens.slice(0, baseTokenCount - 3).concat( new ContextToken(new LegacyQuotientSpur(baseTokenization.tokens[baseTokenCount-2].searchModule, deleteToBoundDistrib, deleteToBoundDistrib[0])) - ), - null, - null + ) ); return { @@ -252,9 +240,7 @@ describe('determineSuggestionRange', () => { const tokensToAppend = rawText.map((t) => ContextToken.fromRawText(plainModel, t, false)); const foxVsAlligatorTokenization = new ContextTokenization( - originalQuickBrownTokenization.tokens.slice(0, transitionSliceIndex).concat(tokensToAppend), - null, - null + originalQuickBrownTokenization.tokens.slice(0, transitionSliceIndex).concat(tokensToAppend) ) const analysis = determineSuggestionRange(originalQuickBrownTokenization.tokens, foxVsAlligatorTokenization.tokens, tokenEquality); @@ -276,9 +262,7 @@ describe('determineSuggestionRange', () => { const tokensToAppend = rawText.map((t) => ContextToken.fromRawText(plainModel, t, false)); const dogsAndCatTokenization = new ContextTokenization( - originalQuickBrownTokenization.tokens.slice(0, originalTokenCount - 1).concat(tokensToAppend), - null, - null + originalQuickBrownTokenization.tokens.slice(0, originalTokenCount - 1).concat(tokensToAppend) ) const analysis = determineSuggestionRange(originalQuickBrownTokenization.tokens, dogsAndCatTokenization.tokens, tokenEquality); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts index 818a908d1bb..f52dd3b166a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -13,7 +13,6 @@ import { assert } from 'chai'; import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; -import { KMWString } from 'keyman/common/web-utils'; import { determineTokenizedCorrectionSequence, @@ -403,10 +402,5 @@ describe('determineTokenizedCorrectionSequence', () => { }; results.applyInPost(dummiedTuple); - - assert.deepEqual(dummiedTuple.metadata.preservationTransform, { - insert: trueInput.sample.insert.substring(0, KMWString.length(trueInput.sample.insert) - 1), // remove the 'd'. - deleteLeft: trueInput.sample.deleteLeft - 1 - }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts index e90fb1845d9..1c09cd928db 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -11,7 +11,6 @@ import { assert } from 'chai'; import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; -import { KMWString } from 'keyman/common/web-utils'; import { determineTraversallessCorrectionSequences, IntermediateTokenizedPrediction, ModelCompositor, models } from "@keymanapp/lm-worker/test-index"; @@ -453,10 +452,5 @@ describe('determineTraversallessCorrectionSequences', () => { }; entry.applyInPost(dummiedTuple); - - assert.deepEqual(dummiedTuple.metadata.preservationTransform, { - insert: trueInput.sample.insert.substring(0, KMWString.length(trueInput.sample.insert) - 1), // remove the 'd'. - deleteLeft: trueInput.sample.deleteLeft - 1 - }); }); }); \ No newline at end of file From 4b8c37efc9408e31109d0c7f02647fe212a92bb9 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 14 Apr 2026 14:42:12 -0500 Subject: [PATCH 56/65] feat(web): add prepareTokenizationSearch helper method This method is designed to determine the appropriate range of tokens, within each context variant, should be eligible for correction when generating predictions and corrections. Build-bot: skip build:web Test-bot: skip --- .../worker-thread/src/main/predict-helpers.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index cbbf22d1cc7..33c428d1acd 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -604,6 +604,39 @@ export function determineTokenizedCorrectionSequence( }; } +export function prepareTokenizationSearch( + transition: ContextTransition, + tokenizations: ContextTokenization[] +) { + // Goal - determine what parts of each tokenization are searchable & prep them for correcion-search. + const tokenizationAnalyses = tokenizations.map((tokenization) => { + return { + tokenization: tokenization, + analysis: determineSuggestionRange(transition.base.displayTokenization.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId) + }; + }); + + const biggestCommonRemoval = tokenizationAnalyses.reduce( + (biggest, current) => biggest.length > current.analysis.tokensToRemove.length ? biggest : current.analysis.tokensToRemove, + [] as ContextTokenLike[] + ); + + const tokenizationSetup = tokenizationAnalyses.map((tuple) => { + // These tokens are unaffected by the input whatsoever, though their + // probability may affect thresholding for the non-locked tokens. + const unaffectedTokenCount = biggestCommonRemoval.length - tuple.analysis.tokensToRemove.length; + + const mutatedLength = tuple.analysis.tokensToPredict.length; + return new TokenizationCorrector(tuple.tokenization, mutatedLength, (token, index) => { + return index >= unaffectedTokenCount // is a modified token + && index == mutatedLength - 1 // TEMP: adjacent to the caret (TO BE REMOVED) + && correctionValidForAutoSelect(token.exampleInput); // and is eligible text-correction + }); + }); + + return tokenizationSetup; +} + /** * This method performs the correction-search and model-lookup operations for * prediction generation by using the user's context state and potential From 2faebb47219f18f5dcaa7bd04f60f678dccd942a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 7 May 2026 11:19:44 -0500 Subject: [PATCH 57/65] change(web): convert primary correction-search loop for white-space correction Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/distance-modeler.ts | 12 ++++-- .../worker-thread/src/main/predict-helpers.ts | 42 +++++++++++-------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index 7019c2aaa30..77d11d42756 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -661,12 +661,16 @@ export async function *getBestMatches< let lowestCostSource = spaceQueue.dequeue(); const newResult = lowestCostSource.handleNextNode(); - spaceQueue.enqueue(lowestCostSource); - spaceQueue = new PriorityQueue(CORRECTION_QUEUE_COMPARATOR, spaceQueue.toArray()); if(newResult.type == 'none') { + // Do not re-add the source if its searchspace is exhausted. return null; - } else if(newResult.type == 'complete') { + } else { + spaceQueue.enqueue(lowestCostSource); + spaceQueue = new PriorityQueue(CORRECTION_QUEUE_COMPARATOR, spaceQueue.toArray()); + } + + if(newResult.type == 'complete') { const mapping = newResult.mapping; return filter(mapping) ? mapping : null; } @@ -683,7 +687,7 @@ export async function *getBestMatches< if(timer.timeSinceLastDefer > STANDARD_TIME_BETWEEN_DEFERS) { await timer.defer(); } - } while(!timer.elapsed && spaceQueue.peek().currentCost < Number.POSITIVE_INFINITY); + } while(!timer.elapsed && spaceQueue.count > 0 && spaceQueue.peek().currentCost < Number.POSITIVE_INFINITY); return null; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 33c428d1acd..acf56cde741 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -12,10 +12,9 @@ import { ContextState, determineContextSlideTransform } from './correction/conte import { ContextTransition } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; -import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js'; -import { TokenResult } from './correction/tokenization-corrector.js'; -import { TokenizationCorrector } from './correction/tokenization-corrector.js'; -import { TokenizationResultMapping } from './correction/tokenization-result-mapping.js'; +import { EDIT_DISTANCE_COST_SCALE, getBestMatches } from './correction/distance-modeler.js'; +import { TokenizationCorrector, TokenResult } from './correction/tokenization-corrector.js'; +import { TokenizationResult, TokenizationResultMapping } from './correction/tokenization-result-mapping.js'; import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; @@ -630,7 +629,7 @@ export function prepareTokenizationSearch( return new TokenizationCorrector(tuple.tokenization, mutatedLength, (token, index) => { return index >= unaffectedTokenCount // is a modified token && index == mutatedLength - 1 // TEMP: adjacent to the caret (TO BE REMOVED) - && correctionValidForAutoSelect(token.exampleInput); // and is eligible text-correction + && (token.codepointLength == 0 || correctionValidForAutoSelect(token.exampleInput)); // and is eligible text-correction }); }); @@ -717,31 +716,38 @@ export async function correctAndEnumerate( const tokenizations = transition.final.tokenizations; const searchModules = tokenizations.map(t => t.tail.searchModule); + const preppedTokenizationSearch = prepareTokenizationSearch(transition, tokenizations); + // Only run the correction search when corrections are enabled. let rawPredictions: IntermediateTokenizedPrediction[] = []; let bestCorrectionCost: number; - for await(const match of getBestTokenMatches(searchModules, timer)) { - // Corrections obtained: now to predict from them! - const tokenization = tokenizations.find(t => t.spaceId == match.spaceId); - - // If our 'match' results in fully deleting the new token, reject it and try again. - if(match.matchSequence.length == 0 && match.inputSequence.length != 0) { + for await(const match of getBestMatches(preppedTokenizationSearch, timer)) { + const { totalEditCount, totalEditableCodepoints } = match.matchedResult; + // If our 'match' fully replaces the tokens, reject it and try again. + // + // If the known edit count matches the total length of editable text, reject + // the suggestion source. Q: for any token, or across ALL tokens? If "for + // any", we need to distinguish between penalization where corrections + // couldn't be found and where there's actual edit cost. (2.5 edits does + // stand out a bit, but we should do something more robust.) + if(totalEditCount != 0 && totalEditableCodepoints == totalEditCount) { // TODO: double-check approach! continue; } - // If our 'match' fully replaces the token, reject it and try again. - if(match.matchSequence.length != 0 && match.matchSequence.length == match.knownCost) { + // Perhaps make a return object, add a cumulative 'editCount' property? + // Or, we could just sum it up here. + if(totalEditCount > 0 && !searchModules.find(s => s.correctionsEnabled)) { continue; } - if(match.editCount > 0 && !searchModules.find(s => s.correctionsEnabled)) { - continue; - } + // Worth considering: extend Traversal to allow direct prediction lookups? + // let traversal = match.finalTraversal; + const tokenization = match.matchingSpace.tokenization; const suggestionRange = determineSuggestionRange(transition.base.displayTokenization.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); suggestionRange.transitionId = transition.transitionId; - const corrector = new TokenizationCorrector(tokenization, suggestionRange.tokensToPredict.length, () => true); - const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, new TokenizationResultMapping([match], corrector)); + // const corrector = new TokenizationCorrector(tokenization, suggestionRange.tokensToPredict.length, () => true); + const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, match); const predictions = predictFromCorrectionSequence(lexicalModel, predictionPrep); From 095ee327ae625c3f305bcf760b2be886399138f8 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 22 Jun 2026 08:36:47 -0500 Subject: [PATCH 58/65] docs(web): adjust comments per review --- .../worker-thread/src/main/predict-helpers.ts | 4 +--- .../predict-from-correction-sequence.tests.ts | 11 ++++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index c61f6346ca1..a417dadba82 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -663,7 +663,7 @@ export function shouldStopSearchingEarly( * Given a generated set of corrections from the correction-search process, this * function searches the lexical model for valid predictions rooted on each. * - * While doing so, it also associates each prediction with metadata used for to + * While doing so, it also associates each prediction with metadata used to * "rank" and select the best predictions once the search is complete. This is * performed at later stages. * @param lexicalModel @@ -689,8 +689,6 @@ export function predictFromCorrectionSequence( for(let i = 0; i < corrections.length; i++) { const correction = corrections[i].sample; - - // Step 2: predict based on the final token. const predictions = lexicalModel.predict(correction, currentContext); // Failsafe: if there are no matching predictions, create a fake prediction diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts index 64cea5b2300..6446087009e 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -1,3 +1,12 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-04-13 + * + * This file unit tests against the `predictFromCorrectionSequence` + * prediction-helper function, validating construction of predictions based on + * their root correction sequences. + */ import { assert } from 'chai'; @@ -41,7 +50,7 @@ const applyCasing: CasingFunction = (casing, text) => { // Capitalizes the first code unit of the string, leaving the rest intact. return text.substring(0, headUnitLength).toUpperCase() // head - uppercased - .concat(text.substring(headUnitLength)); // tail - lowercased + .concat(text.substring(headUnitLength)); // tail - unchanged } }; From ed15bf89c6f5fe631c3defc0252439d497b2ee11 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 22 Jun 2026 10:14:14 -0500 Subject: [PATCH 59/65] docs(web): document TokenizationCorrector fields, rearrange into proper groups --- .../src/main/correction/tokenization-corrector.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 835f9eef388..c1e3173d52b 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -47,8 +47,16 @@ export type TokenResult = { * range. */ export class TokenizationCorrector implements CorrectionSearchable, TokenizationResultMapping> { + /** + * The root ContextTokenization all generated corrections are based upon. + */ public readonly tokenization: ContextTokenization; - private readonly tailCorrectionLength: number; + + /** + * Indicates whether or not the correction range of this TokenizationCorrector + * started with any tokens considered "correctable". + */ + public readonly modelsCorrectables: boolean; // public read-only via properties private readonly _uncorrectables: QuotientNodeFinalizer[]; @@ -58,13 +66,13 @@ export class TokenizationCorrector implements CorrectionSearchable; private tokenCostMap: Map; private tokenLookupMap: Map; private lastTotalCost: number; private handleHasBeenCalled: boolean = false; private predictableMatchFound: boolean = false; + private readonly tailCorrectionLength: number; get currentCost(): number { const correctable = this.selectionQueue.peek(); From ed422810be7165f10c0c2716e5d289679ff34516 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 22 Jun 2026 10:26:13 -0500 Subject: [PATCH 60/65] change(web): apply suggestions from PR review - A few let -> const - Adds and fixes doc-comments --- .../worker-thread/src/main/predict-helpers.ts | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 6ef54116737..17a6c7fa5a4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -17,7 +17,6 @@ import { TokenResult } from './correction/tokenization-corrector.js'; import { TokenizationCorrector } from './correction/tokenization-corrector.js'; import { TokenizationResultMapping } from './correction/tokenization-result-mapping.js'; -import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; import Keep = LexicalModelTypes.Keep; @@ -795,12 +794,7 @@ export function shouldStopSearchingEarly( * "rank" and select the best predictions once the search is complete. This is * performed at later stages. * @param lexicalModel - * @param corrections Each `correction` should insert a full token's text to be - * appended to the context resulting from all preceding corrections. - * @param rootContext This context should represent all portions of the - * post-context not represented by the entries of the `corrections` array. - * @param transitionId Indicates the unique ID of the transition that triggered - * prediction generation. + * @param predictionPrep * @returns */ export function predictFromCorrectionSequence( @@ -921,33 +915,28 @@ export function predictFromCorrectionSequence( /** * Applies the specified casing-form to generated suggestions, leveraging the model's * defined casing behaviors to do so. - * @param suggestion - * @param baseWord + * @param predictionToken A tuple representing a single context token's prediction and base correction * @param lexicalModel - * @param casingForm */ export function applySuggestionCasing(predictionToken: TokenizedPredictionData, lexicalModel: LexicalModel) { - const suggestion = predictionToken.prediction; - // Step 0: our pattern for generating predictions and corrections already - // enforces them to encompass the whole word. - - // Step 1: detect the original token's casing - let casingForm: CasingForm; + // enforces that they encompass the whole word. + const suggestion = predictionToken.prediction; // If we are using the context-tracking engine (when traversals are enabled), // we just leverage the context token's exampleInput to determine casing. // // If it's not available, the correction entry reflects a word-broken piece of // the original context, with its original casing - so we use that instead. - let casingRoot = predictionToken.casingRoot ? predictionToken.casingRoot : predictionToken.correction; + const casingRoot = predictionToken.casingRoot ? predictionToken.casingRoot : predictionToken.correction; if(!casingRoot) { // There's no text in place to verify casing expectations; just leave it // unchanged. return; } - casingForm = detectCurrentCasing(lexicalModel, { + // Step 1: detect the original token's casing + const casingForm = detectCurrentCasing(lexicalModel, { left: casingRoot, startOfBuffer: true, endOfBuffer: true @@ -960,6 +949,13 @@ export function applySuggestionCasing(predictionToken: TokenizedPredictionData, } } +/** + * Composites a set of `IntermediateTokenizedPrediction`s, merging the tokenized + * data into corresponding `IntermediateCompositedPrediction`s representing the + * full range of affected context. + * @param predictions + * @returns + */ export function compositeIntermediatePredictions(predictions: IntermediateTokenizedPrediction[]): IntermediateCompositedPrediction[] { return predictions.map((predictionData) => { const components = predictionData.components; From bdc824a9c10633a52098954c41d8c8f20755fafc Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 22 Jun 2026 10:48:26 -0500 Subject: [PATCH 61/65] change(web): rename compositeIntermdiatePredictions to composeIntermediatePredictions --- .../worker-thread/src/main/model-compositor.ts | 4 ++-- .../predictive-text/worker-thread/src/main/predict-helpers.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index bffa6a73e92..25863a5937c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -1,7 +1,7 @@ import * as models from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { applySuggestionCasing, compositeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; +import { applySuggestionCasing, composeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTracker } from './correction/context-tracker.js'; @@ -151,7 +151,7 @@ export class ModelCompositor { // We want to dedupe before trimming the list so that we can present a full set // of viable distinct suggestions if available. - const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, compositeIntermediatePredictions(rawPredictions), context); + const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, composeIntermediatePredictions(rawPredictions), context); // Needs "casing" to be applied first. const postContext = postContextState?.context ?? models.applyTransform(inputTransform, context); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 17a6c7fa5a4..f6fdb5b730d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -950,13 +950,13 @@ export function applySuggestionCasing(predictionToken: TokenizedPredictionData, } /** - * Composites a set of `IntermediateTokenizedPrediction`s, merging the tokenized + * Composes a set of `IntermediateTokenizedPrediction`s, merging the tokenized * data into corresponding `IntermediateCompositedPrediction`s representing the * full range of affected context. * @param predictions * @returns */ -export function compositeIntermediatePredictions(predictions: IntermediateTokenizedPrediction[]): IntermediateCompositedPrediction[] { +export function composeIntermediatePredictions(predictions: IntermediateTokenizedPrediction[]): IntermediateCompositedPrediction[] { return predictions.map((predictionData) => { const components = predictionData.components; From 03d20f1d1c343b3c07c908f876f8e48d0db1a166 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 29 Jun 2026 12:17:51 -0500 Subject: [PATCH 62/65] change(web): rework type nomenclature --- .../worker-thread/src/main/predict-helpers.ts | 32 ++++++------- .../early-correction-search-stopping.tests.ts | 4 +- .../prediction-helpers/auto-correct.tests.ts | 48 +++++++++---------- .../create-default-keep.tests.ts | 14 +++--- ...ine-tokenized-correction-sequence.tests.ts | 4 +- ...raversalless-correction-sequences.tests.ts | 4 +- .../suggestion-deduplication.tests.ts | 10 ++-- .../suggestion-finalization.tests.ts | 10 ++-- .../suggestion-similarity.tests.ts | 18 +++---- 9 files changed, 72 insertions(+), 72 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 914f31cb95c..3a7bcdf6efa 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -174,7 +174,7 @@ export interface PredictionMetadata { preservationTransform?: Transform; } -export interface IntermediateCompositedPrediction { +export interface CompositedIntermediatePrediction { /** * Contains the fully composited predictive-text Suggestion and its underlying correction string. */ @@ -185,7 +185,7 @@ export interface IntermediateCompositedPrediction { metadata: PredictionMetadata; } -type IntermediatePrediction = IntermediateCompositedPrediction; +type IntermediatePrediction = CompositedIntermediatePrediction; /** * An enum to be used when categorizing the level of similarity between @@ -494,7 +494,7 @@ export interface PredictionParameters { * @param entry * @returns */ - applyInPost: (entry: IntermediateCompositedPrediction) => void + applyInPost: (entry: CompositedIntermediatePrediction) => void } export function buildCorrectionSequence( @@ -577,7 +577,7 @@ export function determineTokenizedCorrectionSequence( return { ...suggestionParams, - applyInPost: (entry: IntermediateCompositedPrediction) => { + applyInPost: (entry: CompositedIntermediatePrediction) => { entry.metadata.preservationTransform = tokenization.taillessTrueKeystroke; // // Will need an extra lookup layer if the suggestion is generated from within a cluster. // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization); @@ -612,7 +612,7 @@ export async function correctAndEnumerate( /** * The suggestions generated based on the user's input state. */ - rawPredictions: IntermediateCompositedPrediction[]; + rawPredictions: CompositedIntermediatePrediction[]; /** * The id of a prior ContextTransition event that triggered a Suggestion found @@ -668,7 +668,7 @@ export async function correctAndEnumerate( const searchModules = tokenizations.map(t => t.tail.searchModule); // Only run the correction search when corrections are enabled. - let rawPredictions: IntermediateCompositedPrediction[] = []; + let rawPredictions: CompositedIntermediatePrediction[] = []; let bestCorrectionCost: number; for await(const match of getBestTokenMatches(searchModules, timer)) { // Corrections obtained: now to predict from them! @@ -721,7 +721,7 @@ export async function correctAndEnumerate( export function shouldStopSearchingEarly( bestCorrectionCost: number, currentCorrectionCost: number, - rawPredictions: IntermediateCompositedPrediction[] + rawPredictions: CompositedIntermediatePrediction[] ) { if(currentCorrectionCost >= bestCorrectionCost + CORRECTION_SEARCH_THRESHOLDS.MAX_SEARCH_THRESHOLD) { return true; @@ -767,7 +767,7 @@ export function predictFromCorrectionSequence( corrections: ProbabilityMass[], rootContext: Context, transitionId: number -): IntermediateCompositedPrediction[] { +): CompositedIntermediatePrediction[] { let predictionPrefixSequence: ProbabilityMass[] = []; let tailPredictions: ProbabilityMass[]; @@ -816,7 +816,7 @@ export function predictFromCorrectionSequence( return []; } - const predictions: IntermediateCompositedPrediction[] = tailPredictions.map((p) => { + const predictions: CompositedIntermediatePrediction[] = tailPredictions.map((p) => { // Concat corrections + predictions for their components. const predictionSequence = [...predictionPrefixSequence, p]; const fullPrediction: ProbabilityMass = predictionSequence.reduce((prev, curr) => { @@ -896,13 +896,13 @@ export function applySuggestionCasing(suggestion: Suggestion, baseWord: string, */ export function dedupeSuggestions( lexicalModel: LexicalModel, - rawPredictions: IntermediateCompositedPrediction[], + rawPredictions: CompositedIntermediatePrediction[], context: Context ) { const wordbreak = determineModelWordbreaker(lexicalModel); - let suggestionDistribMap: {[key: string]: IntermediateCompositedPrediction} = {}; - let suggestionDistribution: IntermediateCompositedPrediction[] = []; + let suggestionDistribMap: {[key: string]: CompositedIntermediatePrediction} = {}; + let suggestionDistribution: CompositedIntermediatePrediction[] = []; // Deduplicator + annotator of 'keep' suggestions. for(let tuple of rawPredictions) { @@ -953,7 +953,7 @@ export function dedupeSuggestions( */ export function processSimilarity( lexicalModel: LexicalModel, - suggestionDistribution: IntermediateCompositedPrediction[], + suggestionDistribution: CompositedIntermediatePrediction[], context: Context, trueInput: ProbabilityMass ): boolean { @@ -1030,7 +1030,7 @@ export function createDefaultKeep( lexicalModel: LexicalModel, context: Context, trueInput: ProbabilityMass -): IntermediateCompositedPrediction { +): CompositedIntermediatePrediction { const { sample: inputTransform, p: inputTransformProb } = trueInput; const wordbreak = determineModelWordbreaker(lexicalModel); const tokenizer = determineModelTokenizer(lexicalModel); @@ -1111,7 +1111,7 @@ export function correctionValidForAutoSelect(correction: string) { return false; } -export function predictionAutoSelect(suggestionDistribution: IntermediateCompositedPrediction[]) { +export function predictionAutoSelect(suggestionDistribution: CompositedIntermediatePrediction[]) { if(suggestionDistribution.length == 0) { return; } @@ -1205,7 +1205,7 @@ export function predictionAutoSelect(suggestionDistribution: IntermediateComposi */ export function finalizeSuggestions( lexicalModel: LexicalModel, - deduplicatedSuggestionTuples: IntermediateCompositedPrediction[], + deduplicatedSuggestionTuples: CompositedIntermediatePrediction[], context: Context, inputTransform: Transform, verbose?: boolean diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts index 430d9c6c7e0..bed2c0f2255 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts @@ -1,6 +1,6 @@ import { assert } from 'chai'; -import { CORRECTION_SEARCH_THRESHOLDS, IntermediateCompositedPrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index"; +import { CORRECTION_SEARCH_THRESHOLDS, CompositedIntermediatePrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index"; function mockIntermediatePrediction(value: number) { return { @@ -9,7 +9,7 @@ function mockIntermediatePrediction(value: number) { total: value } } - } as IntermediateCompositedPrediction + } as CompositedIntermediatePrediction } describe('correction-search: shouldStopSearchingEarly', () => { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts index b55886bb42f..ecb4b09357e 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts @@ -1,6 +1,6 @@ import { assert } from 'chai'; -import { AUTOSELECT_PROPORTION_THRESHOLD, IntermediateCompositedPrediction, predictionAutoSelect, SuggestionSimilarity, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; +import { AUTOSELECT_PROPORTION_THRESHOLD, CompositedIntermediatePrediction, predictionAutoSelect, SuggestionSimilarity, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; /* * Preconditions: * - there should always be a 'keep' option. Now, whether or not that option @@ -9,7 +9,7 @@ import { AUTOSELECT_PROPORTION_THRESHOLD, IntermediateCompositedPrediction, pred */ describe('predictionAutoSelect', () => { it(`does not throw when no suggestions are available`, () => { - const predictions: IntermediateCompositedPrediction[] = []; + const predictions: CompositedIntermediatePrediction[] = []; const originalPredictions = [].concat(predictions); assert.doesNotThrow(() => predictionAutoSelect(predictions)); @@ -17,7 +17,7 @@ describe('predictionAutoSelect', () => { }); it(`selects solitary 'keep' suggestion that does match the model`, () => { - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ { components: { prediction: { @@ -51,7 +51,7 @@ describe('predictionAutoSelect', () => { }); it(`does not select suggestions if the root correction has no letters`, () => { - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ { components: { prediction: { @@ -106,7 +106,7 @@ describe('predictionAutoSelect', () => { }); it(`does not select solitary 'keep' suggestion that doesn't match the model`, () => { - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ { components: { prediction: { @@ -140,7 +140,7 @@ describe('predictionAutoSelect', () => { }); it(`selects 'keep' suggestion that does match the model over any alternatives`, () => { - const keepSuggestion: IntermediateCompositedPrediction = { + const keepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { tag: 'keep', @@ -163,7 +163,7 @@ describe('predictionAutoSelect', () => { } } - const highestNonKeepSuggestion: IntermediateCompositedPrediction = { + const highestNonKeepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { transform: { // can be null / "mocked out" @@ -184,7 +184,7 @@ describe('predictionAutoSelect', () => { } }; - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { @@ -238,7 +238,7 @@ describe('predictionAutoSelect', () => { }); it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => { - const keepSuggestion: IntermediateCompositedPrediction = { + const keepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { tag: 'keep', @@ -265,7 +265,7 @@ describe('predictionAutoSelect', () => { // This threshold may be subject to change. // // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts. - const onlyNonKeepSuggestion: IntermediateCompositedPrediction = { + const onlyNonKeepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { transform: { // can be null / "mocked out" @@ -286,7 +286,7 @@ describe('predictionAutoSelect', () => { } }; - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, onlyNonKeepSuggestion ]; @@ -305,7 +305,7 @@ describe('predictionAutoSelect', () => { }); it(`does not select non-'keep' without sufficient winning probability`, () => { - const keepSuggestion: IntermediateCompositedPrediction = { + const keepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { tag: 'keep', @@ -332,7 +332,7 @@ describe('predictionAutoSelect', () => { // This threshold may be subject to change. // // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts. - const highestNonKeepSuggestion: IntermediateCompositedPrediction = { + const highestNonKeepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { transform: { // can be null / "mocked out" @@ -353,7 +353,7 @@ describe('predictionAutoSelect', () => { } }; - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { @@ -412,7 +412,7 @@ describe('predictionAutoSelect', () => { }); it(`does select non-'keep' with sufficient winning probability`, () => { - const keepSuggestion: IntermediateCompositedPrediction = { + const keepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { tag: 'keep', @@ -435,7 +435,7 @@ describe('predictionAutoSelect', () => { } } - const highestNonKeepSuggestion: IntermediateCompositedPrediction = { + const highestNonKeepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { transform: { // can be null / "mocked out" @@ -456,7 +456,7 @@ describe('predictionAutoSelect', () => { } }; - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { @@ -513,7 +513,7 @@ describe('predictionAutoSelect', () => { }); it('ignores non key-matched suggestions when key-matched suggestions exist', () => { - const keepSuggestion: IntermediateCompositedPrediction = { + const keepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { tag: 'keep', @@ -537,7 +537,7 @@ describe('predictionAutoSelect', () => { } } - const expectedSuggestion: IntermediateCompositedPrediction = { + const expectedSuggestion: CompositedIntermediatePrediction = { components: { prediction: { transform: { // can be null / "mocked out" @@ -559,7 +559,7 @@ describe('predictionAutoSelect', () => { } }; - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, expectedSuggestion, { @@ -597,7 +597,7 @@ describe('predictionAutoSelect', () => { // The idea: avoid "over-correcting" when a potential correction has a // super-high-frequency word. it('does not auto-select suggestion if its root correction is not most likely', () => { - const keepSuggestion: IntermediateCompositedPrediction = { + const keepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { tag: 'keep', @@ -620,7 +620,7 @@ describe('predictionAutoSelect', () => { } } - const highestCorrectionSuggestion: IntermediateCompositedPrediction = { + const highestCorrectionSuggestion: CompositedIntermediatePrediction = { components: { prediction: { transform: { // can be null / "mocked out" @@ -641,7 +641,7 @@ describe('predictionAutoSelect', () => { } }; - const highestNonKeepSuggestion: IntermediateCompositedPrediction = { + const highestNonKeepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { transform: { // can be null / "mocked out" @@ -662,7 +662,7 @@ describe('predictionAutoSelect', () => { } }; - const predictions: IntermediateCompositedPrediction[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, highestNonKeepSuggestion, highestCorrectionSuggestion diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts index afc0d4d232f..8943b85da5f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts @@ -12,7 +12,7 @@ import { assert } from 'chai'; import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; -import { IntermediateCompositedPrediction, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index"; import CasingFunction = LexicalModelTypes.CasingFunction; import Context = LexicalModelTypes.Context; @@ -108,7 +108,7 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: IntermediateCompositedPrediction = { + const expectedKeep: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -152,7 +152,7 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: IntermediateCompositedPrediction = { + const expectedKeep: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -196,7 +196,7 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: IntermediateCompositedPrediction = { + const expectedKeep: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -240,7 +240,7 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: IntermediateCompositedPrediction = { + const expectedKeep: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -284,7 +284,7 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: IntermediateCompositedPrediction = { + const expectedKeep: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -328,7 +328,7 @@ describe('createDefaultKeep', () => { p: 1 }; - const expectedKeep: IntermediateCompositedPrediction = { + const expectedKeep: CompositedIntermediatePrediction = { components: { prediction: { transform: { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts index 5cd7b67072b..6f5729a7c87 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -21,7 +21,7 @@ import { ContextState, ContextToken, ContextTokenization, - IntermediateCompositedPrediction, + CompositedIntermediatePrediction, ModelCompositor, TokenizationResultMapping } from "@keymanapp/lm-worker/test-index"; @@ -341,7 +341,7 @@ describe('determineTokenizedCorrectionSequence', () => { }); assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); - const dummiedTuple: IntermediateCompositedPrediction = { + const dummiedTuple: CompositedIntermediatePrediction = { components: { prediction: { transform: { insert: 'dog', deleteLeft: 0 }, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts index a1872641d3c..112c61d2a85 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -13,7 +13,7 @@ import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { KMWString } from 'keyman/common/web-utils'; -import { IntermediateCompositedPrediction, ModelCompositor, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, ModelCompositor, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import DummyModel = models.DummyModel; @@ -377,7 +377,7 @@ describe('determineTraversallessCorrectionSequences', () => { }); assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); - const dummiedTuple: IntermediateCompositedPrediction = { + const dummiedTuple: CompositedIntermediatePrediction = { components: { prediction: { transform: { insert: 'dog', deleteLeft: 0 }, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts index 4a2afcf60dc..d28d7a6343b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts @@ -4,7 +4,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { deepCopy } from 'keyman/common/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { IntermediateCompositedPrediction, dedupeSuggestions, models } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, dedupeSuggestions, models } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import DummyModel = models.DummyModel; @@ -24,7 +24,7 @@ const testModel = new DummyModel({ * @returns */ const build_its_is_set = () => { - const its: IntermediateCompositedPrediction = { + const its: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -46,7 +46,7 @@ const build_its_is_set = () => { } }; - const it_is: IntermediateCompositedPrediction = { + const it_is: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -67,7 +67,7 @@ const build_its_is_set = () => { } }; - const is: IntermediateCompositedPrediction = { + const is: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -88,7 +88,7 @@ const build_its_is_set = () => { } }; - const is_not: IntermediateCompositedPrediction = { + const is_not: CompositedIntermediatePrediction = { components: { prediction: { transform: { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts index 9ba1faec742..2bf9a6f96a4 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts @@ -5,7 +5,7 @@ import { deepCopy } from 'keyman/common/web-utils'; import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { IntermediateCompositedPrediction, finalizeSuggestions, models } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, finalizeSuggestions, models } from "@keymanapp/lm-worker/test-index"; import DummyModel = models.DummyModel; import Outcome = LexicalModelTypes.Outcome; @@ -48,7 +48,7 @@ const testModelWithoutSpacing = new DummyModel({ */ const build_its_is_set = (verbose?: string) => { const verboseFlag = (verbose == 'verbose' ? true : false); - const its: IntermediateCompositedPrediction = { + const its: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -70,7 +70,7 @@ const build_its_is_set = (verbose?: string) => { } }; - const it_is: IntermediateCompositedPrediction = { + const it_is: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -91,7 +91,7 @@ const build_its_is_set = (verbose?: string) => { } }; - const is: IntermediateCompositedPrediction = { + const is: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -112,7 +112,7 @@ const build_its_is_set = (verbose?: string) => { } }; - const is_not: IntermediateCompositedPrediction = { + const is_not: CompositedIntermediatePrediction = { components: { prediction: { transform: { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts index 6a97039d245..76a605e869c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts @@ -5,7 +5,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { deepCopy } from 'keyman/common/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { IntermediateCompositedPrediction, models, processSimilarity, SuggestionSimilarity, toAnnotatedSuggestion } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, models, processSimilarity, SuggestionSimilarity, toAnnotatedSuggestion } from "@keymanapp/lm-worker/test-index"; import CasingFunction = LexicalModelTypes.CasingFunction; import Context = LexicalModelTypes.Context; @@ -109,7 +109,7 @@ const testModelWithCasing = new DummyModel({ * @returns */ const build_its_is_set = () => { - const its: IntermediateCompositedPrediction = { + const its: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -131,7 +131,7 @@ const build_its_is_set = () => { } }; - const it_is: IntermediateCompositedPrediction = { + const it_is: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -152,7 +152,7 @@ const build_its_is_set = () => { } }; - const is: IntermediateCompositedPrediction = { + const is: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -173,7 +173,7 @@ const build_its_is_set = () => { } }; - const is_not: IntermediateCompositedPrediction = { + const is_not: CompositedIntermediatePrediction = { components: { prediction: { transform: { @@ -222,7 +222,7 @@ describe('processSimilarity', () => { const testSet = build_its_is_set(); const distribution = [...Object.values(testSet)]; - const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; + const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)]; expectation[0].metadata.matchLevel = SuggestionSimilarity.exact; // its expectation[1].metadata.matchLevel = SuggestionSimilarity.sameKey; // it_is expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is @@ -259,7 +259,7 @@ describe('processSimilarity', () => { const testSet = build_its_is_set(); const distribution = [...Object.values(testSet)]; - const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; + const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)]; expectation[0].metadata.matchLevel = SuggestionSimilarity.sameKey; // its expectation[1].metadata.matchLevel = SuggestionSimilarity.exact; // it_is expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is @@ -313,7 +313,7 @@ describe('processSimilarity', () => { const distribution = [...Object.values(testSet)]; - const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; + const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)]; expectation[0].metadata.matchLevel = SuggestionSimilarity.sameKey; // its expectation[1].metadata.matchLevel = SuggestionSimilarity.sameText; // it_is expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is @@ -355,7 +355,7 @@ describe('processSimilarity', () => { const distribution = [...Object.values(testSet)]; - const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)]; + const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)]; expectation.forEach((entry) => entry.metadata.matchLevel = SuggestionSimilarity.none); processSimilarity(testModelWithoutCasing, distribution, context, trueInput); From b9d0201d92c1fc4b8d948e2c805527ddb2ebc735 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 14 Apr 2026 14:42:12 -0500 Subject: [PATCH 63/65] feat(web): add prepareTokenizationSearch helper method This method is designed to determine the appropriate range of tokens, within each context variant, should be eligible for correction when generating predictions and corrections. Build-bot: skip build:web Test-bot: skip --- .../worker-thread/src/main/predict-helpers.ts | 76 +++- .../prepare-tokenization-search.tests.ts | 329 ++++++++++++++++++ 2 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 41bc99acf5e..c784b5f41de 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -5,7 +5,7 @@ import { searchForProperty, WordBreakProperty } from '@keymanapp/models-wordbrea import { TransformUtils } from './transformUtils.js'; import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; -import { ContextTokenLike } from './correction/context-token.js'; +import { ContextToken, ContextTokenLike } from './correction/context-token.js'; import { ContextTokenization } from './correction/context-tokenization.js'; import { ContextTracker } from './correction/context-tracker.js'; import { ContextState, determineContextSlideTransform } from './correction/context-state.js'; @@ -603,6 +603,80 @@ export function determineTokenizedCorrectionSequence( }; } +/** + * Given the base 'display' tokenization and an array of target tokenizations + * for a context transition, this method determines the range needed for + * correction-search processes and builds the appropriate + * `TokenizationCorrector` instances for the search. + * @param transition + * @param tokenizations + * @param configuration Allows custom configuration for selecting correctable + * tokens within each TokenizationCorrector. Intended for use with unit + * testing. + * @returns + */ +export function prepareTokenizationSearch( + transition: ContextTransition, + tokenizations: ContextTokenization[], + configuration?: { + /** + * Should return true if the input index is within the appropriate range for + * correction. When called, the index of the first token in correction + * range will be provided to `rangeStart`. + * @param index + * @param rangeStart + */ + rangeValidator?: (index: number, rangeStart: number) => boolean, + /** + * Should return true if the token represents text valid for text correction + * processes, regardless of position. + * @param token + * @returns + */ + correctableValidator?: (token: ContextToken) => boolean + } +) { + // Goal - determine what parts of each tokenization are searchable & prep them for correcion-search. + const tokenizationAnalyses = tokenizations.map((tokenization) => { + return { + tokenization: tokenization, + analysis: determineSuggestionRange(transition.base.displayTokenization.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId) + }; + }); + + const biggestCommonRemoval = tokenizationAnalyses.reduce( + (biggest, current) => biggest.length > current.analysis.tokensToRemove.length ? biggest : current.analysis.tokensToRemove, + [] as ContextTokenLike[] + ); + + configuration ??= {}; + // If a custom range validator is set, just use that one. + const rangeValidator = configuration.rangeValidator; + + configuration.correctableValidator ??= (token) => (token.codepointLength == 0 || correctionValidForAutoSelect(token.exampleInput)); + const tokenizationSetup = tokenizationAnalyses.map((tuple) => { + // These tokens are unaffected by the input whatsoever, though their + // probability may affect thresholding for the non-locked tokens. + const unaffectedTokenCount = biggestCommonRemoval.length - tuple.analysis.tokensToRemove.length; + // Unaffected tokens should still be part of the correction range; they'll just + // be marked noncorrectable. + const mutatedLength = tuple.analysis.tokensToPredict.length + unaffectedTokenCount; + + configuration.rangeValidator = rangeValidator ?? ((index, rangeStart) => { + return index >= rangeStart // is a modified token + && index == mutatedLength - 1 // TEMP: adjacent to the caret (TO BE REMOVED) + }); + return new TokenizationCorrector(tuple.tokenization, mutatedLength, (token, index) => { + // is within range for correction + return configuration.rangeValidator(index, unaffectedTokenCount) + // and is eligible for text-correction + && configuration.correctableValidator(token); + }); + }); + + return tokenizationSetup; +} + /** * This method performs the correction-search and model-lookup operations for * prediction generation by using the user's context state and potential diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts new file mode 100644 index 00000000000..67ea7cf5bd8 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts @@ -0,0 +1,329 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-07-21 + * + * This file unit tests against the `prepareTokenizationSearch` + * prediction-helper function, which uses results from + * `determineSuggestionRange` to build phrase-level correctors for multi-token + * correction. + */ + +import { assert } from 'chai'; + +import { LexicalModelTypes } from '@keymanapp/common-types'; +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { ContextState, ContextTokenization, ContextTransition, models, prepareTokenizationSearch, ContextToken, LegacyQuotientRoot, LegacyQuotientSpur } from "@keymanapp/lm-worker/test-index"; + +import Context = LexicalModelTypes.Context; +import Distribution = LexicalModelTypes.Distribution; +import Transform = LexicalModelTypes.Transform; +import TrieModel = models.TrieModel; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + +describe.only('prepareTokenizationSearch', () => { + it('handles simple-case, single tokenization transitions well', () => { + const baseContext: Context = { + left: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'a', + startOfBuffer: true, + endOfBuffer: true + }; + + const targetTokenization = new ContextTokenization([ContextToken.fromRawText(testModel, 'a')]); + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization]); + + transition.finalize(nextState, [ + { sample: { insert: 'a', deleteLeft: 0, id: 1}, p: 1 } + ]); + + const correctorsForSearch = prepareTokenizationSearch(transition, [targetTokenization]); + assert.equal(correctorsForSearch.length, 1); + + const corrector = correctorsForSearch[0]; + assert.deepEqual(corrector.orderedTokens, targetTokenization.tokens); + assert.deepEqual(corrector.correctableTokens, []); + assert.deepEqual(corrector.uncorrectableTokens, []); + assert.deepEqual(corrector.predictableToken, targetTokenization.tail); + + assert.equal(corrector.correctableCodepoints, 1); + assert.isTrue(corrector.modelsCorrectables) + assert.equal(corrector.tokenization, targetTokenization); + }); + + it('handles single-tokenization transitions from whitespace well', () => { + const baseContext: Context = { + left: 'space', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'space ', + startOfBuffer: true, + endOfBuffer: true + }; + + const targetTokenization = new ContextTokenization([ + transition.base.displayTokenization.tail, + ContextToken.fromRawText(testModel, ' '), + ContextToken.fromRawText(testModel, '', true) + ]); + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization]); + + transition.finalize(nextState, [ + { sample: { insert: ' ', deleteLeft: 0, id: 1}, p: 1 } + ]); + + const correctorsForSearch = prepareTokenizationSearch(transition, [targetTokenization]); + assert.equal(correctorsForSearch.length, 1); + + const corrector = correctorsForSearch[0]; + assert.deepEqual(corrector.orderedTokens, targetTokenization.tokens.slice(1)); + assert.deepEqual(corrector.correctableTokens, []); + assert.deepEqual(corrector.uncorrectableTokens, [targetTokenization.tokens[1]]); + assert.deepEqual(corrector.predictableToken, targetTokenization.tail); + + assert.equal(corrector.correctableCodepoints, 0); + assert.isTrue(corrector.modelsCorrectables) + assert.equal(corrector.tokenization, targetTokenization); + }); + + it('handles divergent transitions with possible wordbreaks', () => { + const baseContext: Context = { + left: 'space', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'spaced', + startOfBuffer: true, + endOfBuffer: true + }; + + const distribution: Distribution = [ + { sample: { insert: 'd', deleteLeft: 0, id: 1}, p: .8 }, + { sample: { insert: ' ', deleteLeft: 0, id: 1}, p: .2 } + ]; + + + const targetTokenization1 = new ContextTokenization([ + new ContextToken(new LegacyQuotientSpur(transition.base.displayTokenization.tail.searchModule, [distribution[0]], distribution[0])) + ]); + + const targetTokenization2 = new ContextTokenization([ + transition.base.displayTokenization.tail, + new ContextToken(new LegacyQuotientSpur(new LegacyQuotientRoot(testModel), [distribution[1]], distribution[0])), + ContextToken.fromRawText(testModel, '', true) + ]); + + const tokenizations = [targetTokenization1, targetTokenization2]; + + const nextState = new ContextState(targetContext, testModel, targetTokenization1, tokenizations); + + transition.finalize(nextState, distribution); + + const correctorsForSearch = prepareTokenizationSearch(transition, tokenizations); + assert.equal(correctorsForSearch.length, 2); + + const corrector1 = correctorsForSearch[0]; + assert.deepEqual(corrector1.orderedTokens, targetTokenization1.tokens); + assert.deepEqual(corrector1.correctableTokens, []); + assert.deepEqual(corrector1.uncorrectableTokens, []); + assert.deepEqual(corrector1.predictableToken, targetTokenization1.tail); + + assert.equal(corrector1.correctableCodepoints, 6); + assert.isTrue(corrector1.modelsCorrectables) + assert.equal(corrector1.tokenization, targetTokenization1); + + const corrector2 = correctorsForSearch[1]; + assert.deepEqual(corrector2.orderedTokens, targetTokenization2.tokens); + assert.deepEqual(corrector2.correctableTokens, []); + assert.deepEqual(corrector2.uncorrectableTokens, targetTokenization2.tokens.slice(0, 2)); + assert.deepEqual(corrector2.predictableToken, targetTokenization2.tail); + + assert.equal(corrector2.correctableCodepoints, 0); + assert.isTrue(corrector2.modelsCorrectables) + assert.equal(corrector2.tokenization, targetTokenization2); + }); + + it('handles simple-case, backspace tokenization transitions well', () => { + const baseContext: Context = { + left: 'apples', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'apple', + startOfBuffer: true, + endOfBuffer: true + }; + + const targetTokenization = new ContextTokenization([ContextToken.fromRawText(testModel, 'apple')]); + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization]); + + transition.finalize(nextState, [ + { sample: { insert: '', deleteLeft: 1, id: 1}, p: 1 } + ]); + + const correctorsForSearch = prepareTokenizationSearch(transition, [targetTokenization]); + assert.equal(correctorsForSearch.length, 1); + + const corrector = correctorsForSearch[0]; + assert.deepEqual(corrector.orderedTokens, targetTokenization.tokens); + assert.deepEqual(corrector.correctableTokens, []); + assert.deepEqual(corrector.uncorrectableTokens, []); + assert.deepEqual(corrector.predictableToken, targetTokenization.tail); + + assert.equal(corrector.correctableCodepoints, 5); + assert.isTrue(corrector.modelsCorrectables) + assert.equal(corrector.tokenization, targetTokenization); + }); + + it('handles whitespace-token deletion transitions well', () => { + const baseContext: Context = { + left: 'apples ', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'apples', + startOfBuffer: true, + endOfBuffer: true + }; + + const targetTokenization = new ContextTokenization([transition.base.displayTokenization.tokens[0]]); + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization]); + + transition.finalize(nextState, [ + { sample: { insert: '', deleteLeft: 1, id: 1}, p: 1 } + ]); + + const correctorsForSearch = prepareTokenizationSearch(transition, [targetTokenization]); + assert.equal(correctorsForSearch.length, 1); + + const corrector = correctorsForSearch[0]; + assert.deepEqual(corrector.orderedTokens, targetTokenization.tokens); + assert.deepEqual(corrector.correctableTokens, []); + assert.deepEqual(corrector.uncorrectableTokens, []); + assert.deepEqual(corrector.predictableToken, targetTokenization.tail); + + assert.equal(corrector.correctableCodepoints, 6); + assert.isTrue(corrector.modelsCorrectables) + assert.equal(corrector.tokenization, targetTokenization); + }); + + // TODO: a more complex transition set. + it('handles complicated dictionary-style wordbreaking transitions', () => { + const baseContext: Context = { + left: 'myapplesandsour', + startOfBuffer: true, + endOfBuffer: true + }; + + // To be safe, make sure the spaces are constructed in proper left-to-right order, + // no matter the variation. + const my = ContextToken.fromRawText(testModel, 'my') + const apples = ContextToken.fromRawText(testModel, 'apples'); + const apple = ContextToken.fromRawText(testModel, 'apple'); + + const and = ContextToken.fromRawText(testModel, 'and'); + const sand = ContextToken.fromRawText(testModel, 'sand'); + const sands = ContextToken.fromRawText(testModel, 'sands'); + + const sour = ContextToken.fromRawText(testModel, 'sour'); + const our = ContextToken.fromRawText(testModel, 'our'); + + const baseTokenization = new ContextTokenization(([my, apples, and, sour])); + + const baseTokenizationVariants: ContextTokenization[] = [ + new ContextTokenization(([my, apple, sand, sour])), + new ContextTokenization(([my, apple, sands, our])) + ] + const transition = new ContextTransition(new ContextState(baseContext, testModel, baseTokenization, baseTokenizationVariants), 0); + + // ---- + const targetContext: Context = { + left: 'myapplesandsourg', + startOfBuffer: true, + endOfBuffer: true + }; + + const distribution: Distribution = [ + { sample: { insert: 'g', deleteLeft: 0, id: 1}, p: 1 } + ]; + + const g = () => new ContextToken(new LegacyQuotientSpur(new LegacyQuotientRoot(testModel), distribution, distribution[0])); + const ourg = () => new ContextToken(new LegacyQuotientSpur(our.searchModule, distribution, distribution[0])); + const sourg = () => new ContextToken(new LegacyQuotientSpur(sour.searchModule, distribution, distribution[0])); + + const targetTokenization = new ContextTokenization([my, apples, and, sour, g()]); + + const targetTokenizationVariants: ContextTokenization[] = [ + new ContextTokenization([my, apples, and, sourg()]), + new ContextTokenization([my, apple, sand, sour, g()]), + new ContextTokenization([my, apple, sand, sourg()]), + new ContextTokenization([my, apple, sands, our, g()]), + new ContextTokenization([my, apple, sands, ourg()]) + ]; + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization, ...targetTokenizationVariants]); + transition.finalize(nextState, distribution); + + const variationStartIndex: Map = new Map(); + for(let variant of nextState.tokenizations) { + let i = 0; + while(variant.tokens[i]?.exampleInput == baseTokenization.tokens[i]?.exampleInput) { + i++; + } + variationStartIndex.set(variant, i); + } + + const correctorsForSearch = prepareTokenizationSearch(transition, nextState.tokenizations, { + rangeValidator: (index, rs) => index >= rs + }); + assert.equal(correctorsForSearch.length, nextState.tokenizations.length); + + correctorsForSearch.forEach((corrector, index) => { + const tokenization = nextState.tokenizations.find((t) => corrector.tokenization == t); + assert.isOk(tokenization); + + assert.deepEqual(corrector.orderedTokens, tokenization.tokens.slice(1), `Error for variant at index ${index}`); + const correctables = tokenization.tokens.slice(variationStartIndex.get(tokenization), -1) + assert.deepEqual(corrector.correctableTokens, correctables, `Error for variant at index ${index}`); + assert.deepEqual(corrector.uncorrectableTokens, tokenization.tokens.slice(1, variationStartIndex.get(tokenization)), `Error for variant at index ${index}`); + assert.deepEqual(corrector.predictableToken, tokenization.tail, `Error for variant at index ${index}`); + + assert.equal( + corrector.correctableCodepoints, + correctables.reduce((accum, curr) => accum + curr.codepointLength, 0) + tokenization.tail.codepointLength, + `Error for variant at index ${index}` + ); + assert.isTrue(corrector.modelsCorrectables, `Error for variant at index ${index}`); + }); + }); +}); \ No newline at end of file From d2b5ac583264de321364010ec2eaacaf778b1479 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 31 Jul 2026 16:29:58 -0500 Subject: [PATCH 64/65] fix(web): fix broken unit test --- web/src/engine/predictive-text/templates/src/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index d141f1f9ae1..77d48261744 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -60,7 +60,7 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans deleteLeft: first.deleteLeft + mergedSecondDelete } - if(first.id && first.id == second.id) { + if(first.id !== undefined && first.id == second.id) { returnedObj.id = first.id; } From 487ce1c0ded8203ce3cc6c6593332d1539967635 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 5 Aug 2026 14:50:19 -0500 Subject: [PATCH 65/65] change(web): address PR review comments and corrections --- .../worker-thread/src/main/predict-helpers.ts | 29 ++++++++++++++++--- .../prepare-tokenization-search.tests.ts | 14 ++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index bff2607b373..c34d8debff7 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -636,6 +636,10 @@ export function prepareTokenizationSearch( correctableValidator?: (token: ContextToken) => boolean } ) { + // Create duplicate of config parameter in order to prevent unwanted + // side-effects across multiple calls. + configuration = {...configuration}; + // Goal - determine what parts of each tokenization are searchable & prep them for correcion-search. const tokenizationAnalyses = tokenizations.map((tokenization) => { return { @@ -644,6 +648,15 @@ export function prepareTokenizationSearch( }; }); + // The token "removal" from each analysis is always based upon the + // base-state's tokenization - the same set of tokens. + // + // It is a consistent base from which all target tokenization variants are + // derived, first by removing the specified token count, then by adding on the + // edits + new additions. + + // Thus, the first step: what's the largest count removed by ANY transition + // variant? const biggestCommonRemoval = tokenizationAnalyses.reduce( (biggest, current) => biggest.length > current.analysis.tokensToRemove.length ? biggest : current.analysis.tokensToRemove, [] as ContextTokenLike[] @@ -655,13 +668,21 @@ export function prepareTokenizationSearch( configuration.correctableValidator ??= (token) => (token.codepointLength == 0 || correctionValidForAutoSelect(token.exampleInput)); const tokenizationSetup = tokenizationAnalyses.map((tuple) => { - // These tokens are unaffected by the input whatsoever, though their - // probability may affect thresholding for the non-locked tokens. + // While removed by at least one variant, this number of tokens was left in + // place, untouched, during the transition to the variant under + // consideration. These tokens are thus unaffected by the input whatsoever, + // though their probability may affect thresholding for the non-locked + // tokens. const unaffectedTokenCount = biggestCommonRemoval.length - tuple.analysis.tokensToRemove.length; - // Unaffected tokens should still be part of the correction range; they'll just - // be marked noncorrectable. + // Unaffected tokens should still be part of the correction range; they'll + // just be marked noncorrectable. The edited + appended tokens (for actual + // correction and prediction), of course, are also part of that range. const mutatedLength = tuple.analysis.tokensToPredict.length + unaffectedTokenCount; + // Redefined here to capture this loop's value for `mutatedLength`. + + // If the token falls past the unaffected range... + // and, *for now*, is actually the tail token, then it may be corrected. configuration.rangeValidator = rangeValidator ?? ((index, rangeStart) => { return index >= rangeStart // is a modified token && index == mutatedLength - 1 // TEMP: adjacent to the caret (TO BE REMOVED) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts index 67ea7cf5bd8..945785ada00 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts @@ -23,7 +23,7 @@ import TrieModel = models.TrieModel; const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); -describe.only('prepareTokenizationSearch', () => { +describe('prepareTokenizationSearch', () => { it('handles simple-case, single tokenization transitions well', () => { const baseContext: Context = { left: '', @@ -312,18 +312,18 @@ describe.only('prepareTokenizationSearch', () => { const tokenization = nextState.tokenizations.find((t) => corrector.tokenization == t); assert.isOk(tokenization); - assert.deepEqual(corrector.orderedTokens, tokenization.tokens.slice(1), `Error for variant at index ${index}`); + assert.deepEqual(corrector.orderedTokens, tokenization.tokens.slice(1), `Error for variant's ordered tokens at index ${index}`); const correctables = tokenization.tokens.slice(variationStartIndex.get(tokenization), -1) - assert.deepEqual(corrector.correctableTokens, correctables, `Error for variant at index ${index}`); - assert.deepEqual(corrector.uncorrectableTokens, tokenization.tokens.slice(1, variationStartIndex.get(tokenization)), `Error for variant at index ${index}`); - assert.deepEqual(corrector.predictableToken, tokenization.tail, `Error for variant at index ${index}`); + assert.deepEqual(corrector.correctableTokens, correctables, `Error for variant's correctable tokens at index ${index}`); + assert.deepEqual(corrector.uncorrectableTokens, tokenization.tokens.slice(1, variationStartIndex.get(tokenization)), `Error for variant's uncorrectable tokens at index ${index}`); + assert.deepEqual(corrector.predictableToken, tokenization.tail, `Error for variant's predictable token at index ${index}`); assert.equal( corrector.correctableCodepoints, correctables.reduce((accum, curr) => accum + curr.codepointLength, 0) + tokenization.tail.codepointLength, - `Error for variant at index ${index}` + `Error for variant's correctable-codepoint count at index ${index}` ); - assert.isTrue(corrector.modelsCorrectables, `Error for variant at index ${index}`); + assert.isTrue(corrector.modelsCorrectables, `Error for variant's 'models correctables' flag at index ${index}`); }); }); }); \ No newline at end of file