diff --git a/docs/epics.md b/docs/epics.md new file mode 100644 index 00000000000..77166f3cd1e --- /dev/null +++ b/docs/epics.md @@ -0,0 +1,18 @@ +# Epics + +This document lists epics in the Keyman project. + + Name | Lead | Target release | Spec +-----------------------|-----------|----------------|-------------------- + epic/embed-osk-in-kmx | @mcdurdin | 19.0 | [embed-osk-in-kmx] + epic/autocorrect | @jahorton | 19.0 | + epic/boundary-correction | @jahorton | 20.0 | + + +Note: when establishing a new epic branch, add the details of the epic to this +file, as a clean way to start an epic without unreviewed code changes in the +base epic PR. + +--- + +[embed-osk-in-kmx]: https://docs.google.com/document/d/15EjtSH7NAsGrdapfB3E7SlS6CqI-2zPy3PNyfUiAv54/edit?tab=t.0 \ No newline at end of file diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index d141f1f9ae1..77d48261744 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -60,7 +60,7 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans deleteLeft: first.deleteLeft + mergedSecondDelete } - if(first.id && first.id == second.id) { + if(first.id !== undefined && first.id == second.id) { returnedObj.id = first.id; } diff --git a/web/src/engine/predictive-text/templates/src/tokenization.ts b/web/src/engine/predictive-text/templates/src/tokenization.ts index fd8ed28d5ca..47ef927fa5b 100644 --- a/web/src/engine/predictive-text/templates/src/tokenization.ts +++ b/web/src/engine/predictive-text/templates/src/tokenization.ts @@ -95,6 +95,10 @@ export function tokenize( currentIndex = nextIndex; } + if(tokenization.left.length == 0) { + tokenization.left.push({text: '', isWhitespace: false}); + } + // New step 2: handle any rejoins needed. // Handle any desired special handling for directly-pre-caret scenarios - where for this diff --git a/web/src/engine/predictive-text/worker-thread/build.sh b/web/src/engine/predictive-text/worker-thread/build.sh index 2e7fd9d90e5..2d4171b599e 100755 --- a/web/src/engine/predictive-text/worker-thread/build.sh +++ b/web/src/engine/predictive-text/worker-thread/build.sh @@ -21,6 +21,7 @@ SRCMAP_CLEANER="${KEYMAN_ROOT}/web/build/tools/building/sourcemap-root/index.js" ################################ Main script ################################ SUBPROJECT_NAME=engine/predictive-text/worker-thread +SUBPROJECT_HELPERS=engine/predictive-text/helpers builder_describe \ "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \ @@ -100,6 +101,7 @@ function do_test() { WTR_INSPECT=" --manual" fi + test-headless-typescript $SUBPROJECT_HELPERS test-headless-typescript $SUBPROJECT_NAME web-test-runner --config ./src/tests/test-runner/web-test-runner${WTR_CONFIG}.config.mjs ${WTR_INSPECT} diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index bc9b412bf30..19ae3544776 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -46,7 +46,13 @@ export class ContextState { /** * Denotes the possible tokenization(s) for the represented Context. */ - tokenization: ContextTokenization; + _tokenizations: ContextTokenization[]; + + /** + * Tracks the tokenization pattern best matching the word boundary + * patterns in the actual context. + */ + _displayTokenization: ContextTokenization; /** * Denotes the keystroke-sourced Transform that was last applied to a @@ -97,7 +103,14 @@ export class ContextState { * is visible to the user. */ get displayTokenization(): ContextTokenization { - return this.tokenization; + return this._displayTokenization; + } + + /** + * Denotes the possible tokenization(s) for the represented Context. + */ + get tokenizations(): ContextTokenization[] { + return this._tokenizations; } /** @@ -127,13 +140,16 @@ export class ContextState { * @param tokenization Precomputed tokenization for the context, leveraging previous * correction-search progress and results */ - constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization); - constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization) { + constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization, tokenizations?: ContextTokenization[]); + constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization, tokenizations?: ContextTokenization[]) { if(!(param1 instanceof ContextState)) { this.context = param1; this.model = model; if(tokenization) { - this.tokenization = tokenization; + this._tokenizations = tokenizations ? tokenizations : [tokenization]; + this._displayTokenization = tokenization; + + this.inputTransforms = new Map(); } else { this.initFromReset(); } @@ -142,7 +158,8 @@ export class ContextState { Object.assign(this, stateToClone); this.inputTransforms = new Map(stateToClone.inputTransforms); - this.tokenization = new ContextTokenization(stateToClone.tokenization); + this._displayTokenization = new ContextTokenization(stateToClone._displayTokenization); + this._tokenizations = stateToClone.tokenizations.map((t) => new ContextTokenization(t)); // A shallow copy of the array is fine, but we'd be best off // not aliasing the array itself. @@ -173,10 +190,35 @@ export class ContextState { if(baseTokens.length == 0) { baseTokens.push(ContextToken.fromRawText(this.model, '')); } - this.tokenization = new ContextTokenization(baseTokens); + this._displayTokenization = new ContextTokenization(baseTokens); + this._tokenizations = [this._displayTokenization]; this.inputTransforms = new Map(); } + /** + * Builds a variant of the ContextState with no input metadata, reiterating the resulting + * state of a recent transition but without any associated input mutation data. + * @param stateToClone + * @param context + * @returns + */ + transitionContextWindow(context: Context) { + // The context may have slid since the last ContextState observation. We check for slide effects + // and apply them here as well. + const slideUpdateTransform = determineContextSlideTransform(this.context, context); + + const model = this.model; + + // Should a context-reset or similar occur, there is no input to replace, nor is there + // pre-tail token transform data to preserve. It's important that part, in particular, + // be cleared so that Suggestions in this state are not adversely affected. + const displayTokenization = this._displayTokenization.applyContextSlide(model, slideUpdateTransform); + const tokenizations = this.tokenizations.map((t) => t.applyContextSlide(model, slideUpdateTransform)); + + const state = new ContextState(context, this.model, displayTokenization, tokenizations); + return state; + } + /** * As written, this method attempts to determine the context state and tokenization(s) * that result from applying an incoming transform distribution to the incoming context @@ -209,8 +251,7 @@ export class ContextState { const slideUpdateTransform = determineContextSlideTransform(this.context, context); - // Goal: allow multiple base tokenizations. - const startTokenizations = [this.tokenization].map((t) => { + const startTokenizations = this.tokenizations.map((t) => { return t.applyContextSlide(lexicalModel, slideUpdateTransform); }); @@ -222,14 +263,15 @@ export class ContextState { // If the tokenizations match, clone the ContextState; we want to preserve a post-application // context separately from pre-application contexts for predictions based on empty roots. const state = new ContextState(this); - state.tokenization = [...startTokenizations.values()][0]; + state._tokenizations = [...startTokenizations.values()]; + state._displayTokenization = this._displayTokenization.applyContextSlide(lexicalModel, slideUpdateTransform); transition.finalize(state, transformDistribution); return transition; } const { subsets, keyMatchingUserContext: trueInputSubsetKey } = precomputeTransitions(startTokenizations, transformDistribution); - const resultTokenization = transitionTokenizations(subsets, transformDistribution).get(trueInputSubsetKey); - + const possibleTokenizations = transitionTokenizations(subsets, transformDistribution); + const resultTokenization = possibleTokenizations.get(trueInputSubsetKey); // ------------ // So, if we have a suggestion transition ID at the end and didn't just apply... @@ -240,10 +282,11 @@ export class ContextState { // 'any'.) const state = new ContextState(applyTransform(trueInput, context), lexicalModel); - state.tokenization = resultTokenization; + state._tokenizations = [resultTokenization]; // TODO: [...possibleTokenizations.values()]; + state._displayTokenization = resultTokenization; state.appliedInput = transformDistribution?.[0].sample; transition.finalize(state, transformDistribution); - transition.revertableTransitionId = state.tokenization.tail.appliedTransitionId; + transition.revertableTransitionId = state._displayTokenization.tail.appliedTransitionId; return transition; } } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index e1265b4b634..ed1598c159c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -15,7 +15,6 @@ import { LegacyQuotientSpur } from "./legacy-quotient-spur.js"; import { LegacyQuotientRoot } from "./legacy-quotient-root.js"; import { generateSubsetId } from './tokenization-subsets.js'; -import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; @@ -37,11 +36,42 @@ function textToCharTransforms(text: string, transitionId?: number): Transform[] [...text].map(insert => ({insert, deleteLeft: 0})); } + +/** + * Defines an interface compatible with ContextToken that is useful for handling + * cases that should not be considered correctable. + */ +export interface ContextTokenLike { + /** + * Generates text corresponding to the net effects of the most likely inputs + * received that can correspond to the represented token. + */ + exampleInput: string; + + /** + * Reports the length in codepoints of corrected text represented by the + * current token. + */ + codepointLength: number; + + /** + * Whether or not the token is likely still being edited by the user (due to + * adjacency of the caret) + */ + isPartial?: boolean; + + /** + * Gets a compact string-based representation of `inputRange` that + * maps compatible token source ranges to each other. + */ + sourceRangeKey?: string; +} + /** * Represents cached data about one token (either a word or a unit of whitespace) * in the context and associated correction-search progress and results. */ -export class ContextToken { +export class ContextToken implements ContextTokenLike { /** * Indicates whether or not the token is considered whitespace. */ @@ -56,6 +86,10 @@ export class ContextToken { } private _searchModule: SearchQuotientNode; + /** + * Whether or not the token is likely still being edited by the user (due to + * adjacency of the caret) + */ isPartial: boolean; /** @@ -125,11 +159,11 @@ export class ContextToken { } /** - * Call this to record the original keystroke Transforms for the context range - * corresponding to this token. + * Reports the length in codepoints of corrected text represented by the + * current token. */ - addInput(inputSource: PathInputProperties, distribution: Distribution) { - this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource); + get codepointLength() { + return this._searchModule.codepointLength; } get inputCount() { @@ -169,7 +203,7 @@ export class ContextToken { /** * Generates text corresponding to the net effects of the most likely inputs - * received that can correspond to the current instance. + * received that can correspond to the represented token. */ get exampleInput(): string { return this.searchModule.bestExample.text; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 7e198eee9b8..ae2eac85843 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -10,10 +10,11 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { KMWString } from 'keyman/common/web-utils'; -import { ContextToken } from './context-token.js'; +import { ContextToken, ContextTokenLike } from './context-token.js'; import { TransformUtils } from '../transformUtils.js'; import { computeDistance, EditOperation, EditTuple } from './classical-calculation.js'; import { LegacyQuotientRoot } from './legacy-quotient-root.js'; +import { LegacyQuotientSpur } from './legacy-quotient-spur.js'; import { determineModelTokenizer } from '../model-helpers.js'; import { ExtendedEditOperation, SegmentableDistanceCalculation } from './segmentable-calculation.js'; import { PathInputProperties } from './search-quotient-node.js'; @@ -171,38 +172,11 @@ export class ContextTokenization { */ readonly tokens: ContextToken[]; - /** - * Denotes whether or not the transition to this tokenization added or deleted - * any tokens. - */ - readonly transitionEdits?: { - addedNewTokens: boolean, - removedOldTokens: boolean, - // NOTE: slated for removal in an upcoming PR. Exists in this form to - // facilitate factorization of the changes into smaller bodies of work. - editedTokenCount: number - }; - - /** - * The portion of edits from the true input keystroke that are not part of the - * final entry in `token`. If `null`, all edits are considered part of the - * final token's contents. - * - * If the final token is new due to a newly-introduced wordboundary traversed - * by the keystroke, this will generally be set to an empty transform that - * 'finalizes' the previous tail token. - * - * (Refer to #12494 for an example case.) - */ - readonly taillessTrueKeystroke: Transform; - constructor(priorToClone: ContextTokenization); constructor(tokens: ContextToken[]); - constructor(tokens: ContextToken[], alignment: TransitionEdge, taillessTrueKeystroke: Transform); + constructor(tokens: ContextToken[], alignment: TransitionEdge); constructor( - param1: ContextToken[] | ContextTokenization, - tokenizationPath?: TransitionEdge, - taillessTrueKeystroke?: Transform + param1: ContextToken[] | ContextTokenization ) { if(!(param1 instanceof ContextTokenization)) { const tokens = param1; @@ -210,19 +184,9 @@ export class ContextTokenization { throw new Error("ContextTokenization requires at least one existing ContextToken"); } this.tokens = [].concat(tokens); - if(tokenizationPath) { - this.transitionEdits = { - addedNewTokens: tokenizationPath?.inputs[0].sample.has(1) ?? false, - removedOldTokens: (tokenizationPath?.alignment.removedTokenCount ?? 0) > 0, - editedTokenCount: tokenizationPath?.inputs[0].sample.size - } - } - this.taillessTrueKeystroke = taillessTrueKeystroke; } else { const priorToClone = param1; this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry)); - this.transitionEdits = priorToClone.transitionEdits ? {...priorToClone.transitionEdits} : null; - this.taillessTrueKeystroke = priorToClone.taillessTrueKeystroke; } } @@ -273,9 +237,14 @@ export class ContextTokenization { // Assumption: deleteLeft is always empty. (There's nothing to the left; // we apply on that side.) + + // This can occur during a context-reset or similar operation that would + // repeat the same prediction state. We build clean tokenizations here, + // bereft of `preservationTransform` data and other data about the original + // state's input. if(TransformUtils.isEmpty(transform)) { // No edits needed? Why retokenize? - return new ContextTokenization(this); + return new ContextTokenization(this.tokens); } // Step 1: build a window for window-start retokenization in case the context-window slid. @@ -397,7 +366,7 @@ export class ContextTokenization { } /** - * Given the existing tokenization and an incoming input `Transform`, this + * Given this existing tokenization and an incoming input `Transform`, this * method precomputes how both the current, pre-application tokenization will * be altered and how the incoming Transform will be tokenized. * @@ -414,165 +383,7 @@ export class ContextTokenization { transform: Transform, edgeOptions?: EdgeWindowOptions ): TokenizationTransitionEdits { - // Step 4: now that our window's been properly updated, determine what the - // input's effects on the context is. - // - // Context does not slide within this function. - // - // Assumption: this alignment cannot fail; we KNOW there's a solid - // before-and-after relationship here, and we can base it on the results of - // a prior syncToSourceWindow call. - // - // We don't wish to do the full tokenization here - we only want to check - // over the last few tokens that might reasonably shift. We also want to - // batch effects. - - // Do not mutate the original transform; it can cause unexpected assertion - // effects in unit tests. - const edgeTransform = {...transform, deleteRight: transform.deleteRight || 0}; - const edgeWindow = buildEdgeWindow(this.tokens, edgeTransform, false, edgeOptions); - const { - retokenizationText, - editBoundary, - sliceIndex: edgeSliceIndex - } = edgeWindow; - // Prevent mutation of the original return property. - const stackedDeletes = edgeWindow.deleteLengths.slice(); - - const tokenize = determineModelTokenizer(lexicalModel); - const postTokenization = tokenize({left: retokenizationText + transform.insert, startOfBuffer: true, endOfBuffer: true}).left.map(t => t.text); - if(postTokenization.length == 0) { - postTokenization.push(''); - } - const { stackedInserts, firstInsertPostIndex } = traceInsertEdits(postTokenization, transform); - - // What does the edge's retokenization look like when we remove the inserted portions? - const retokenizedEdge = postTokenization.slice(0, firstInsertPostIndex); - const insertBoundaryToken = postTokenization[firstInsertPostIndex]; - - // Note: requires that helpers have not mutated `stackedInserts`. - const uninsertedBoundaryToken = KMWString.substring(insertBoundaryToken, 0, KMWString.lastIndexOf(insertBoundaryToken, stackedInserts[0])); - - // Do not preserve empty tokens here, even if tokenization normally would produce one. - // It's redundant and replaceable for tokenization batching efforts. - if(uninsertedBoundaryToken != '') { - retokenizedEdge.push(uninsertedBoundaryToken); - } - - // We've found the root token within the root context state to which deletes (and inserts) - // may be applied. - // We've also found the last post-application token to which transform changes contributed. - // How do these indices line up - we need to properly construct and index our transforms, - // but 'merge' and 'split' edits can mess up that indexing. - - const currentTokens = this.tokens; - const preTokenization = currentTokens - .slice(edgeSliceIndex, editBoundary.tokenIndex+1) - .map(t => t.exampleInput); - - // Determine the effects of splits & merges as applied to the original - // cached context state. - const { mergeOffset, splitOffset, editPath, merges, splits } = analyzePathMergesAndSplits( - preTokenization, - postTokenization.slice(0, firstInsertPostIndex+1) - ); - - /* - * Final steps: We can now safely index the transforms. Let's do it! - * 1. Determine the first index a Transform may align to - * 2. Build the transforms - * - * Notes: - * - text applied to the end of a 'merged' token at the tail: should have - * index 0, not -1. - * - pretokenization index will mismatch by -1: -SUM(merge size - 1) - * - Ex: can + ' + t => can't - * -1 0 0 - * - text applied to the end of a 'split' token at the tail: should also - * have index 0, not 1. - * - posttokenization index will mismatch by +1: SUM(split size - 1) - * - new token after 'split': index 1 - * - Ex: can' + ? => can + ' + ? - * 0 -1 0 1 - * - * The first transform applies at the end of the retokenized zone and its - * associated index. The question: were there deletes that occurred? - */ - - const lastEditedPreTokenIndex = editBoundary.tokenIndex - edgeSliceIndex; - let shiftDeletes = false; - // first popped entry == 0 - a delete no-op. - if(stackedDeletes[stackedDeletes.length - 1] == 0) { - // the boundary indices found by both methods above differ - if(lastEditedPreTokenIndex + mergeOffset != firstInsertPostIndex + splitOffset) { - shiftDeletes = true; - } - - // there are no inserts, so we don't affect the boundary token we landed on. - if(stackedDeletes.length > 1 && transform.insert == '') { - shiftDeletes = true; - } - } - - if(shiftDeletes) { - // Do not add a zero-length delete if we're not actually altering the - // corresponding token at all. - stackedDeletes.pop(); - } - - // The first delete always applies to index 0. If the built edge window - // omits a context-final empty-string, adjust the tokenization indices - // accordingly. - const tailIndex = 0 - (stackedDeletes.length - 1) + (editBoundary.omitsEmptyToken ? -1 : 0); - // Mutates stackedInserts, stackedDeletes. - const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length); - const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex); - if(transform.id !== undefined) { - transformMap.forEach((v) => v.id = transform.id); - } - - // If there's an empty transform in the final token's position and we - // already know we're dropping tokens - and only deleting - we're dropping - // an otherwise-untracked empty token - make sure it's included! - const droppedFinalTransform = baseRemovedTokenCount > 0 - && transform.insert == '' - && TransformUtils.isEmpty(transformMap.get(0)) - && shiftDeletes; - // Past that, if we have more delete entries than insert entries for our transforms, we - // dropped some tokens outright. - const removedTokenCount = baseRemovedTokenCount + (droppedFinalTransform ? 1 : 0); - - // Final step: check for any unexpected boundary shifts not mappable to 'merge' / 'split' - // and not caused by transforms. All transforms always apply in sequence at the end. - const unmappedEdits: EditTuple[] = []; - for(let i = 0; i < editPath.length - transformMap.size; i++) { - const op = editPath[i].op; - switch(op) { - case 'merge': - case 'split': - // already calculated - // can fall through to the `continue;` line. - case 'match': - continue; - default: - // Should only be substitutions here. - // We may wish to add extra analysis in the future when supporting - // prediction from multiple competing tokenizations. - unmappedEdits.push(editPath[i] as EditTuple); - } - } - - return { - alignment: { - edgeWindow: {...edgeWindow, retokenization: retokenizedEdge}, - merges, - splits, - unmappedEdits, - removedTokenCount - }, - tokenizedTransform: transformMap, - isBksp: TransformUtils.isBackspace(transform) - }; + return mapWhitespacedTokenization(this.tokens, lexicalModel, transform, edgeOptions); } /** @@ -627,7 +438,7 @@ export class ContextTokenization { tokenization.push(token); } - return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tokenization), null, this.taillessTrueKeystroke); + return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tokenization)); } /** @@ -705,8 +516,10 @@ export class ContextTokenization { inputSource.segment.end = appliedLength; } - affectedToken = new ContextToken(affectedToken); - affectedToken.addInput(inputSource, distribution); + affectedToken = new ContextToken( + new LegacyQuotientSpur(affectedToken.searchModule, distribution, inputSource), + affectedToken.isPartial + ); // Do not adjust the original token, as it may be used by other transitions. // Only adjust the new, extended token. @@ -735,8 +548,7 @@ export class ContextTokenization { return new ContextTokenization( tokenSequence, - null, - determineTaillessTrueKeystroke(transitionEdge.inputs[0].sample) + null ); } } @@ -844,6 +656,192 @@ interface RetokenizedEdgeWindow extends EdgeWindow { retokenization: string[]; } +/** + * Given an existing tokenization and an incoming input `Transform`, this + * method precomputes how both the current, pre-application tokenization will + * be altered and how the incoming Transform will be tokenized. + * + * This function is able to operate with a reduced interface, not requiring + * the full ContextToken/ContextState/etc subsystem and its related + * SearchQuotientNode requirements. + * + * Note that this method is designed for use with languages that employ + * classical space-based wordbreaking. Do not use it for languages that need + * dictionary-based wordbreaking support! + * @param tokens + * @param lexicalModel + * @param transform + * @param edgeOptions + * @returns + */ +export function mapWhitespacedTokenization( + tokens: ContextTokenLike[], + lexicalModel: LexicalModel, + transform: Transform, + edgeOptions?: EdgeWindowOptions +): TokenizationTransitionEdits { + // Step 4: now that our window's been properly updated, determine what the + // input's effects on the context is. + // + // Context does not slide within this function. + // + // Assumption: this alignment cannot fail; we KNOW there's a solid + // before-and-after relationship here, and we can base it on the results of + // a prior syncToSourceWindow call. + // + // We don't wish to do the full tokenization here - we only want to check + // over the last few tokens that might reasonably shift. We also want to + // batch effects. + + // Do not mutate the original transform; it can cause unexpected assertion + // effects in unit tests. + const edgeTransform = {...transform, deleteRight: transform.deleteRight || 0}; + const edgeWindow = buildEdgeWindow(tokens, edgeTransform, false, edgeOptions); + const { + retokenizationText, + editBoundary, + sliceIndex: edgeSliceIndex + } = edgeWindow; + // Prevent mutation of the original return property. + const stackedDeletes = edgeWindow.deleteLengths.slice(); + + const tokenize = determineModelTokenizer(lexicalModel); + const postTokenization = tokenize({left: retokenizationText + transform.insert, startOfBuffer: true, endOfBuffer: true}).left.map(t => t.text); + if(postTokenization.length == 0) { + postTokenization.push(''); + } + const { stackedInserts, firstInsertPostIndex } = traceInsertEdits(postTokenization, transform); + + // What does the edge's retokenization look like when we remove the inserted portions? + const retokenizedEdge = postTokenization.slice(0, firstInsertPostIndex); + const insertBoundaryToken = postTokenization[firstInsertPostIndex]; + + // Note: requires that helpers have not mutated `stackedInserts`. + const uninsertedBoundaryToken = KMWString.substring(insertBoundaryToken, 0, KMWString.lastIndexOf(insertBoundaryToken, stackedInserts[0])); + + // Do not preserve empty tokens here, even if tokenization normally would produce one. + // It's redundant and replaceable for tokenization batching efforts. + if(uninsertedBoundaryToken != '') { + retokenizedEdge.push(uninsertedBoundaryToken); + } + + // We've found the root token within the root context state to which deletes (and inserts) + // may be applied. + // We've also found the last post-application token to which transform changes contributed. + // How do these indices line up - we need to properly construct and index our transforms, + // but 'merge' and 'split' edits can mess up that indexing. + + const currentTokens = tokens; + const preTokenization = currentTokens + .slice(edgeSliceIndex, editBoundary.tokenIndex+1) + .map(t => t.exampleInput); + + // Determine the effects of splits & merges as applied to the original + // cached context state. + const { mergeOffset, splitOffset, editPath, merges, splits } = analyzePathMergesAndSplits( + preTokenization, + postTokenization.slice(0, firstInsertPostIndex+1) + ); + + /* + * Final steps: We can now safely index the transforms. Let's do it! + * 1. Determine the first index a Transform may align to + * 2. Build the transforms + * + * Notes: + * - text applied to the end of a 'merged' token at the tail: should have + * index 0, not -1. + * - pretokenization index will mismatch by -1: -SUM(merge size - 1) + * - Ex: can + ' + t => can't + * -1 0 0 + * - text applied to the end of a 'split' token at the tail: should also + * have index 0, not 1. + * - posttokenization index will mismatch by +1: SUM(split size - 1) + * - new token after 'split': index 1 + * - Ex: can' + ? => can + ' + ? + * 0 -1 0 1 + * + * The first transform applies at the end of the retokenized zone and its + * associated index. The question: were there deletes that occurred? + */ + + const lastEditedPreTokenIndex = editBoundary.tokenIndex - edgeSliceIndex; + let shiftDeletes = false; + // first popped entry == 0 - a delete no-op. + if(stackedDeletes[stackedDeletes.length - 1] == 0) { + // the boundary indices found by both methods above differ + if(lastEditedPreTokenIndex + mergeOffset != firstInsertPostIndex + splitOffset) { + shiftDeletes = true; + } + + // there are no inserts, so we don't affect the boundary token we landed on. + if(stackedDeletes.length > 1 && transform.insert == '') { + shiftDeletes = true; + } + } + + if(shiftDeletes) { + // Do not add a zero-length delete if we're not actually altering the + // corresponding token at all. + stackedDeletes.pop(); + } + + // The first delete always applies to index 0. If the built edge window + // omits a context-final empty-string, adjust the tokenization indices + // accordingly. + const tailIndex = 0 - (stackedDeletes.length - 1) + (editBoundary.omitsEmptyToken ? -1 : 0); + // Mutates stackedInserts, stackedDeletes. + const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length); + const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex); + if(transform.id !== undefined) { + transformMap.forEach((v) => v.id = transform.id); + } + + // If there's an empty transform in the 0 position and we already know we're + // dropping tokens - and only deleting - we're dropping an + // otherwise-untracked empty token - make sure it's included! + const droppedFinalTransform = baseRemovedTokenCount > 0 + && transform.insert == '' + && TransformUtils.isEmpty(transformMap.get(0)) + && shiftDeletes; + + // Past that, if we have more delete entries than insert entries for our transforms, we + // dropped some tokens outright. + const removedTokenCount = baseRemovedTokenCount + (droppedFinalTransform ? 1 : 0); + + // Final step: check for any unexpected boundary shifts not mappable to 'merge' / 'split' + // and not caused by transforms. All transforms always apply in sequence at the end. + const unmappedEdits: EditTuple[] = []; + for(let i = 0; i < editPath.length - transformMap.size; i++) { + const op = editPath[i].op; + switch(op) { + case 'merge': + case 'split': + // already calculated + // can fall through to the `continue;` line. + case 'match': + continue; + default: + // Should only be substitutions here. + // We may wish to add extra analysis in the future when supporting + // prediction from multiple competing tokenizations. + unmappedEdits.push(editPath[i] as EditTuple); + } + } + + return { + alignment: { + edgeWindow: {...edgeWindow, retokenization: retokenizedEdge}, + merges, + splits, + unmappedEdits, + removedTokenCount + }, + tokenizedTransform: transformMap, + isBksp: TransformUtils.isBackspace(transform) + }; +} + /** * Constructs a window on one side of the represented context that is aligned to * existing tokenization. @@ -858,7 +856,7 @@ interface RetokenizedEdgeWindow extends EdgeWindow { * @returns */ export function buildEdgeWindow( - currentTokens: ContextToken[], + currentTokens: ContextTokenLike[], // Requires deleteRight be explicitly set. transform: Transform & { deleteRight: number }, applyAtFront: boolean, @@ -1244,88 +1242,4 @@ export function assembleTransforms(stackedInserts: string[], stackedDeletes: num } return transformMap; -} - -/** - * Used to construct and represent the part of the incoming transform that does - * not land as part of the final token in the resulting context. This component - * should be preserved by any suggestions that get applied. - * @param tokenizedInputs The precomputed tokenization for incoming inputs - * involved in a pre-transition context tokenization to a post-transition - * context tokenization. - * @returns - */ -export function determineTaillessTrueKeystroke(tokenizedInput: Map) { - if(!tokenizedInput || tokenizedInput.size == 0) { - throw new Error(`tokenizedInput must not be nullish or empty; even an empty transform should have an entry`); - } - - // undefined by default; we haven't yet determined if we're still affecting - // the same token that was the tail in the previous tokenization state. - let taillessTrueKeystroke: Transform; - - // If tokens were inserted, emit an empty transform; this prevents - // suggestions from replacing the "current" token. - if(tokenizedInput.has(1)) { - // Sets a default transform that will be returned even if the main - // transform body lies entirely within a new token. - taillessTrueKeystroke = { insert: '', deleteLeft: 0 }; - - // While the .size() > 1 case could also land here, it is ALSO covered - // by the loop that follows, without fail. - } - - // We first wish to find the transform that affects the final post-transition - // token. Accordingly, skip past any transforms that deleted pre-transition - // tokens. - const transformKeys = [...tokenizedInput.keys()]; - do { - const tailKey = transformKeys[transformKeys.length - 1]; - const tailTransform = tokenizedInput.get(tailKey); - - // Do not treat pure-backspace transforms at the tail end of context as the - // transform applied to the suggestion if the input was tokenized; this - // scenario implies that a prior token is being edited instead. - if(TransformUtils.isBackspace(tailTransform) && transformKeys.length > 1) { - transformKeys.pop(); - continue; - } else if(transformKeys.length < 2) { - break; - } - - const penultimateKey = transformKeys[transformKeys.length - 2]; - const penultimateTransform = tokenizedInput.get(penultimateKey); - - if( - // Erasing a single-char whitespace requires deletion of two tokens, the - // last of which is empty. Check for this case and handle it accordingly - // as well. - TransformUtils.isEmpty(tailTransform) && TransformUtils.isBackspace(penultimateTransform) - ) { - transformKeys.pop(); - continue; - } else { - break; - } - } while(true); - - // Ignore the transform that applies to the suggestion-root token - it should - // contribute to the suggestion, rather than be a fixed, universally-applied - // constant. - transformKeys.pop(); - - // If no inputs remain, that's fine - that means of the remaining - // post-transition context tokens, only the final token is affected by the - // input. - for(let i of transformKeys) { - const primaryInput = tokenizedInput.get(i); - if(!taillessTrueKeystroke) { - taillessTrueKeystroke = {...primaryInput}; - } else { - taillessTrueKeystroke.insert += primaryInput.insert; - taillessTrueKeystroke.deleteLeft += primaryInput.deleteLeft; - } - } - - return taillessTrueKeystroke; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts index 32c59aa6eb0..791e32d35e6 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -195,9 +195,13 @@ export class ContextTransition { // body and after any appended whitespace. resultingTokenization.tail.appliedTransitionId = suggestion.transform.id; - const resultingState = new ContextState(applyTransform(transformToApply, baseState.context), lexicalModel); - resultingState.tokenization = resultingTokenization; // [resultingTokenization].concat(preservedVariations); - resultingState.appliedInput = baseState.appliedInput; + const resultingState = new ContextState( + applyTransform(transformToApply, baseState.context), + lexicalModel, + resultingTokenization, + [resultingTokenization] // [resultingTokenization].concat(preservedVariations); + ); + resultingState.appliedInput = transformToApply; resultingState.appliedSuggestionId = suggestion.id; resultingState.suggestions = this.final.suggestions; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/deletion-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/deletion-quotient-spur.ts new file mode 100644 index 00000000000..d592a757e31 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/deletion-quotient-spur.ts @@ -0,0 +1,55 @@ +/** + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-02-03 + * + * This file adds a SearchQuotientSpur variant modeling deletion of the corresponding + * keystroke. + */ + +import { LexicalModelTypes } from "@keymanapp/common-types"; + +import { SearchNode } from "./distance-modeler.js"; +import { PathInputProperties, SearchQuotientNode } from "./search-quotient-node.js"; +import { SearchQuotientSpur } from "./search-quotient-spur.js"; +import { TokenResultMapping } from "./token-result-mapping.js"; + +import Distribution = LexicalModelTypes.Distribution; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; + +export class DeletionQuotientSpur extends SearchQuotientSpur { + public readonly insertLength: number = 0; + public readonly leftDeleteLength: number = 0; + + constructor( + parentNode: SearchQuotientNode, + inputs: Distribution>, + inputSource: PathInputProperties | ProbabilityMass + ) { + super(parentNode, inputs, inputSource, parentNode.codepointLength); + } + + construct(parentNode: SearchQuotientNode, inputs: ProbabilityMass>[], inputSource: PathInputProperties): this { + return new DeletionQuotientSpur(parentNode, inputs, inputSource) as this; + } + + protected buildEdgesFromResults(baseResults: ReadonlyArray): SearchNode[] { + return baseResults + // If there are already at least 2 edits for a node, do not add new edits. + .filter((n) => n.editCount < 2) + .flatMap((n) => n.buildDeletionEdges(this.inputs, this.spaceId)); + } + + get edgeKey(): string { + const baseKey = super.edgeKey; + // Be sure to mark the distinction between this node; appending a suffix can + // stack for multiple sequential deletions. + return `${baseKey}DEL`; + } + + get bestExample() { + // As deletion spurs add no new input, we can just re-use the parent node's version. + return this.parentNode.bestExample; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index b69e6fd2d53..77d11d42756 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -38,7 +38,7 @@ enum TimedTaskTypes { CORRECTING = 2 } -enum PathEdge { +export enum PathEdge { ROOT = 'root', INSERTION = 'insertion', DELETION = 'deletion', @@ -162,7 +162,7 @@ export class SearchNode { * Notes the edit operation used for the most recent edge in the node's * represented search path. */ - private readonly lastEdgeType: PathEdge; + readonly lastEdgeType: PathEdge; constructor(rootTraversal: LexiconTraversal, spaceId: number, toKey?: (arg0: string) => string); constructor(node: SearchNode, spaceId?: number, edgeType?: PathEdge); @@ -276,7 +276,11 @@ export class SearchNode { * character not seen in the input, as if the user accidentally skipped typing * it. No new input will be expected, but the search will continue one * character deeper in the backing lexicon. - * @param spaceId + * @param spaceId A unique identifier associated with the SearchQuotientNode + * that calls this method and processes the resulting SearchNodes. + * + * If left empty, the nodes will be associated with the same SearchQuotientNode + * as this instance. * @returns An array of SearchNodes corresponding to lexical entries that are * prefixed with the lexicon entry represented by the current Node's * matchSequence text. @@ -657,12 +661,16 @@ export async function *getBestMatches< let lowestCostSource = spaceQueue.dequeue(); const newResult = lowestCostSource.handleNextNode(); - spaceQueue.enqueue(lowestCostSource); - spaceQueue = new PriorityQueue(CORRECTION_QUEUE_COMPARATOR, spaceQueue.toArray()); if(newResult.type == 'none') { + // Do not re-add the source if its searchspace is exhausted. return null; - } else if(newResult.type == 'complete') { + } else { + spaceQueue.enqueue(lowestCostSource); + spaceQueue = new PriorityQueue(CORRECTION_QUEUE_COMPARATOR, spaceQueue.toArray()); + } + + if(newResult.type == 'complete') { const mapping = newResult.mapping; return filter(mapping) ? mapping : null; } @@ -679,7 +687,7 @@ export async function *getBestMatches< if(timer.timeSinceLastDefer > STANDARD_TIME_BETWEEN_DEFERS) { await timer.defer(); } - } while(!timer.elapsed && spaceQueue.peek().currentCost < Number.POSITIVE_INFINITY); + } while(!timer.elapsed && spaceQueue.count > 0 && spaceQueue.peek().currentCost < Number.POSITIVE_INFINITY); return null; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts new file mode 100644 index 00000000000..26aa9e55b38 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/insertion-quotient-spur.ts @@ -0,0 +1,53 @@ +/** + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-02-03 + * + * This file adds a SearchQuotientSpur variant modeling insertion of + * lexical-entry prefix characters - an operation with no corresponding + * keystroke. + */ + +import { SENTINEL_CODE_UNIT } from "@keymanapp/models-templates"; +import { PathEdge, SearchNode } from "./distance-modeler.js"; +import { SearchQuotientNode } from "./search-quotient-node.js"; +import { SearchQuotientSpur } from "./search-quotient-spur.js"; +import { TokenResultMapping } from "./token-result-mapping.js"; + +export class InsertionQuotientSpur extends SearchQuotientSpur { + public readonly insertLength = 1; + public readonly leftDeleteLength = 0; + + constructor( + parentNode: SearchQuotientNode + ) { + super(parentNode, null, null, parentNode.codepointLength + 1); + } + + construct(parentNode: SearchQuotientNode): this { + return new InsertionQuotientSpur(parentNode) as this; + } + + protected buildEdgesFromResults(baseNodes: ReadonlyArray): SearchNode[] { + // Note that .buildInsertionEdges will not extend any nodes reached by empty-input + // or by deletions. + return baseNodes + // If there are already at least 2 edits for a node, do not add new edits. + // Also, do not permit insert edits to follow delete edits. + .filter((n) => n.lastEdgeType != PathEdge.DELETION && n.editCount < 2) + .flatMap((n) => n.buildInsertionEdges(this.spaceId)); + } + + get edgeKey(): string { + return `SR[${this.parentNode.sourceRangeKey}]L${this.codepointLength}INS`; + } + + get bestExample() { + const base = this.parentNode.bestExample; + // We use the SENTINEL char as an insertion place-holder, as there's no + // actual keystroke to source better characters from. + base.text += SENTINEL_CODE_UNIT; + + return base; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts index 89ce2e1b14f..4ffd8a25eca 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts @@ -15,6 +15,8 @@ import { PathResult } from './correction-searchable.js'; import { SearchNode } from './distance-modeler.js'; import { SearchQuotientNode, PathInputProperties } from './search-quotient-node.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; +import { SearchQuotientRoot } from './search-quotient-root.js'; +import { LegacyQuotientRoot } from './legacy-quotient-root.js'; import { TokenResultMapping } from './token-result-mapping.js'; import Distribution = LexicalModelTypes.Distribution; @@ -51,6 +53,10 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { return new LegacyQuotientSpur(parentNode, inputs, inputSource) as this; } + constructRoot(): SearchQuotientRoot { + return new LegacyQuotientRoot(this.model); + } + protected buildEdgesFromResults(priorResults: ReadonlyArray): SearchNode[] { // With a newly-available input, we can extend new input-dependent paths from // our previously-reached 'extractedResults' nodes. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts index 4a5ee0723f4..43192f40a91 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts @@ -12,8 +12,8 @@ import { PriorityQueue } from 'keyman/common/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; import { CORRECTION_QUEUE_COMPARATOR, PathResult } from './correction-searchable.js'; -import { LegacyQuotientRoot } from './legacy-quotient-root.js'; import { generateSpaceSeed, InputSegment, SearchQuotientNode } from './search-quotient-node.js'; +import { SearchQuotientRoot } from './search-quotient-root.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; import { TokenResultMapping } from './token-result-mapping.js'; @@ -70,6 +70,10 @@ export class SearchQuotientCluster extends SearchQuotientNode { throw new Error(`SearchQuotientNode does not share the same source identifiers as others in the cluster`); } + if(path instanceof SearchQuotientRoot) { + throw new Error(`SearchQuotientRoot instances may not be part of clusters`); + } + lowestPossibleSingleCost = Math.min(lowestPossibleSingleCost, path.lowestPossibleSingleCost); } @@ -173,16 +177,16 @@ export class SearchQuotientCluster extends SearchQuotientNode { // What if we're trying to merge something previously split? // That can only happen at the head of the incoming space, so we check for it early here. - if(space.inputCount == 1 && space instanceof SearchQuotientSpur) { + if(space.inputCount == 1 && space instanceof SearchQuotientSpur && space.parents[0] instanceof SearchQuotientRoot) { // In such a case... the 'leading edge' of the incoming space needs to be checked // against the trailing edge of `this` instance's entries. const thisTailInputSource = this.inputSegments[this.inputSegments.length - 1]; - const thisTailSpaceIds = this.parents.map((path) => (path as SearchQuotientSpur).inputSource.subsetId); + const thisTailSpaceIds = this.parents.map((path) => (path as SearchQuotientSpur).inputSource?.subsetId); const spaceHeadInputSource = space.inputSegments[0]; const isOnSplitInput = thisTailSpaceIds.some((entry) => entry == space.inputSource.subsetId) - && thisTailInputSource.end == spaceHeadInputSource.start; + && thisTailInputSource?.end == spaceHeadInputSource.start; // In this case, we only rebuild the single path; an outer stack frame will reconstitute // the split cluster from the individual paths built here. @@ -217,7 +221,9 @@ export class SearchQuotientCluster extends SearchQuotientNode { split(charIndex: number): [SearchQuotientNode, SearchQuotientNode][] { // Don't rebuild if this is already a perfect split point! if(this.codepointLength <= charIndex) { - return [[this, new LegacyQuotientRoot(this.model)]]; + // We'll assume that the search path is either using legacy nodes or is not; + // we shouldn't see mixed-use cases. + return [[this, (this.parents[0] as SearchQuotientSpur).constructRoot()]]; } const results = this.parents.flatMap((p) => p.split(charIndex)); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index dbc07422876..9cecde727d0 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -17,7 +17,6 @@ import { EDIT_DISTANCE_COST_SCALE, SearchNode } from './distance-modeler.js'; import { generateSpaceSeed, InputSegment, PathInputProperties, SearchQuotientNode } from './search-quotient-node.js'; import { generateSubsetId } from './tokenization-subsets.js'; import { SearchQuotientRoot } from './search-quotient-root.js'; -import { LegacyQuotientRoot } from './legacy-quotient-root.js'; import { TokenResultMapping } from './token-result-mapping.js'; import Distribution = LexicalModelTypes.Distribution; @@ -45,7 +44,7 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { readonly inputs?: Distribution; readonly inputSource?: PathInputProperties; - private parentNode: SearchQuotientNode; + protected readonly parentNode: SearchQuotientNode; readonly spaceId: number; readonly inputCount: number; @@ -177,6 +176,11 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { inputSource: PathInputProperties ): this; + // TODO: Remove once LegacyQuotientRoot + LegacyQuotientSpur are removed! + constructRoot(): SearchQuotientRoot { + return new SearchQuotientRoot(this.model); + } + // spaces are in sequence here. // `this` = head 'space'. public merge(space: SearchQuotientNode): SearchQuotientNode { @@ -267,7 +271,7 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { // // stopgap: maybe go ahead and check each input for any that are longer? // won't matter shortly, though. - return [[this, new LegacyQuotientRoot(this.model)]]; + return [[this, this.constructRoot()]]; } else { const firstSet: Distribution = this.inputs.map((input) => ({ // keep insert head @@ -304,7 +308,7 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { // construct two SearchPath instances based on the two sets! return [[ parent, - this.construct(new LegacyQuotientRoot(this.model), secondSet, { + this.construct(this.constructRoot(), secondSet, { ...this.inputSource, segment: { ...this.inputSource.segment, diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts new file mode 100644 index 00000000000..76067bccc36 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts @@ -0,0 +1,48 @@ +/** + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-02-03 + * + * This file adds a SearchQuotientSpur variant modeling match & substitute edit + * operations in regard to the corresponding keystroke. + */ + +import { LexicalModelTypes } from "@keymanapp/common-types"; +import { KMWString } from "keyman/common/web-utils"; + +import { SearchNode } from "./distance-modeler.js"; +import { PathInputProperties, SearchQuotientNode } from "./search-quotient-node.js"; +import { SearchQuotientSpur } from "./search-quotient-spur.js"; +import { TokenResultMapping } from "./token-result-mapping.js"; + +import Distribution = LexicalModelTypes.Distribution; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; + +export class SubstitutionQuotientSpur extends SearchQuotientSpur { + public readonly insertLength: number; + public readonly leftDeleteLength: number; + + constructor( + parentNode: SearchQuotientNode, + inputs: Distribution>, + inputSource: PathInputProperties | ProbabilityMass + ) { + // Compute this SearchPath's codepoint length & edge length. + const inputSample = inputs?.[0].sample ?? { insert: '', deleteLeft: 0 }; + const insertLength = KMWString.length(inputSample.insert); + super(parentNode, inputs, inputSource, parentNode.codepointLength + insertLength - inputSample.deleteLeft); + + // Compute this SearchPath's codepoint length & edge length. + this.insertLength = insertLength; + this.leftDeleteLength = inputSample.deleteLeft; + } + + construct(parentNode: SearchQuotientNode, inputs: ProbabilityMass>[], inputSource: PathInputProperties): this { + return new SubstitutionQuotientSpur(parentNode, inputs, inputSource) as this; + } + + protected buildEdgesFromResults(baseResults: ReadonlyArray): SearchNode[] { + return baseResults.flatMap((n) => n.buildSubstitutionEdges(this.inputs, this.spaceId)); + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts index aa64a603df0..862ac1f740f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts @@ -75,6 +75,10 @@ export class TokenResultMapping implements CorrectionResultMapping, return this.node; } + get inputCount(): number { + return this.matchingSpace.inputCount; + } + get inputSequence(): ProbabilityMass[] { return this.node.priorInput; } @@ -131,8 +135,8 @@ export class TokenResultMapping implements CorrectionResultMapping, return new SearchNode(this.node, spaceId); } - buildInsertionEdges(): SearchNode[] { - return this.node.buildInsertionEdges(); + buildInsertionEdges(spaceId?: number): SearchNode[] { + return this.node.buildInsertionEdges(spaceId); } buildDeletionEdges(dist: Distribution, edgeId: number): SearchNode[] { @@ -142,4 +146,8 @@ export class TokenResultMapping implements CorrectionResultMapping, buildSubstitutionEdges(dist: Distribution, edgeId: number): SearchNode[] { return this.node.buildSubstitutionEdges(dist, edgeId); } + + get lastEdgeType() { + return this.node.lastEdgeType; + } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 444ea9ee85b..a2d7709c644 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -14,7 +14,7 @@ import { ContextToken } from "./context-token.js"; import { CorrectionSearchable, PathResult } from "./correction-searchable.js"; import { ContextTokenization } from "./context-tokenization.js"; import { QuotientNodeFinalizer } from "./quotient-node-finalizer.js"; -import { TokenizationResultMapping } from "./tokenization-result-mapping.js"; +import { TokenizationResult, TokenizationResultMapping } from "./tokenization-result-mapping.js"; import { EDIT_DISTANCE_COST_SCALE } from "./distance-modeler.js"; import { MAX_EDIT_THRESHOLD_FACTOR } from "./search-quotient-spur.js"; @@ -34,6 +34,7 @@ import { MAX_EDIT_THRESHOLD_FACTOR } from "./search-quotient-spur.js"; export type TokenResult = { matchString: string, inputSamplingCost: number, + inputCount: number, knownCost: number, totalCost: number } @@ -45,9 +46,17 @@ export type TokenResult = { * all correctable tokens, generating corrections for the full represented * range. */ -export class TokenizationCorrector implements CorrectionSearchable, TokenizationResultMapping> { +export class TokenizationCorrector implements CorrectionSearchable { + /** + * The root ContextTokenization all generated corrections are based upon. + */ public readonly tokenization: ContextTokenization; - private readonly tailCorrectionLength: number; + + /** + * Indicates whether or not the correction range of this TokenizationCorrector + * started with any tokens considered "correctable". + */ + public readonly modelsCorrectables: boolean; // public read-only via properties private readonly _uncorrectables: QuotientNodeFinalizer[]; @@ -55,6 +64,8 @@ export class TokenizationCorrector implements CorrectionSearchable; private _previousResults: TokenizationResultMapping[] = []; + private _correctableCodepoints: number = 0; + private _correctablesMatched = 0; // fully private private selectionQueue: PriorityQueue; @@ -62,6 +73,9 @@ export class TokenizationCorrector implements CorrectionSearchable; private lastTotalCost: number; private handleHasBeenCalled: boolean = false; + private predictableMatchFound: boolean = false; + private matchableTokenCount = 0; + private readonly tailCorrectionLength: number; get currentCost(): number { const correctable = this.selectionQueue.peek(); @@ -103,6 +117,10 @@ export class TokenizationCorrector implements CorrectionSearchable this.tokenLookupMap.get(c.spaceId)); } + get correctableCodepoints(): number { + return this._correctableCodepoints; + } + /** * Returns the token, if it exists, that is considered "predictable". * @@ -139,6 +157,10 @@ export class TokenizationCorrector implements CorrectionSearchable boolean + filterClosure: (token: ContextToken, index?: number) => boolean ) { this.tokenization = tokenization; this.tailCorrectionLength = tailCorrectionLength; @@ -170,15 +192,25 @@ export class TokenizationCorrector implements CorrectionSearchable { // New issue: this mangles the space IDs! We almost certainly need some // sort of proper map to the source token. const searchModule = new QuotientNodeFinalizer(token.searchModule, index == orderedTokens.length - 1); this.tokenLookupMap.set(searchModule.spaceId, token); - if(!filterClosure(token)) { + // Index within the token subset being examined. + const passesFilter = filterClosure(token, index); + modelsCorrectables ||= passesFilter; + if(!passesFilter) { this._uncorrectables.push(searchModule); - } else if(index == tailCorrectionLength - 1) { + return; + } + + this.matchableTokenCount++; + this._correctableCodepoints += searchModule.codepointLength; + if(index == tailCorrectionLength - 1) { // The sole assignment case for this field. It may only be assigned for // the final token, and only if its text is of a form considered // correctable by the filter. @@ -187,6 +219,8 @@ export class TokenizationCorrector implements CorrectionSearchable 0) { + const results = this.collateResults(); + this._previousResults.push(results); + return { + 'type': 'complete', + cost: this.lastTotalCost, + mapping: results + }; + } else { + return { type: 'none' }; + } } } @@ -276,8 +317,9 @@ export class TokenizationCorrector implements CorrectionSearchable { - if(correctableToUpdate != this._predictable) { + if(!correctionIsThePredictable) { // Lock the 'correctable' token now that either a valid correction for // it has been found or all possible corrections are exhausted. We only // consider a single correction for most of a tokenization's tokens, @@ -289,18 +331,27 @@ export class TokenizationCorrector implements CorrectionSearchable correction-string map with the obtained result. this._generatedTokenResults.set(correctableToUpdate.spaceId, tokenResult.mapping); } @@ -342,8 +399,8 @@ export class TokenizationCorrector implements CorrectionSearchable c == undefined) != -1) { + // If any token lacks a matching lookup value, abort. + if([...this.tokenLookupMap.keys()].find((k) => !this._generatedTokenResults.has(k))) { return { type: 'intermediate', cost: tokenizationCost @@ -351,11 +408,27 @@ export class TokenizationCorrector implements CorrectionSearchable 0) { + const correctionResults = this.collateResults(); + this._previousResults.push(correctionResults); + return { + type: 'complete', + cost: tokenizationCost, + mapping: correctionResults + }; + } else { + return { + type: 'none' + } + } + } else { + return { + type: 'none' + }; + } } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts index 32e0fb48fce..da8f4c42a02 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts @@ -1,37 +1,51 @@ import { CorrectionResultMapping } from "./correction-result-mapping.js"; import { TokenizationCorrector, TokenResult } from './tokenization-corrector.js'; -export class TokenizationResultMapping implements CorrectionResultMapping> { +export interface TokenizationResult { + tokenCorrections: ReadonlyArray, + totalEditCount: number, + totalEditableCodepoints: number +} + +export class TokenizationResultMapping implements CorrectionResultMapping { readonly matchingSpace: TokenizationCorrector; - readonly matchedResult: ReadonlyArray; + readonly matchedResult: TokenizationResult; - constructor(tokenization: TokenResult[], corrector: TokenizationCorrector) { + constructor(tokenization: TokenResult[], corrector?: TokenizationCorrector) { this.matchingSpace = corrector; - this.matchedResult = tokenization; + + this.matchedResult = { + tokenCorrections: tokenization, + totalEditCount: tokenization.reduce((accum, curr) => accum + curr.knownCost, 0), + // If based on a legacy/custom model not using traversals, we don't + // support edit operations (for correction) beyond the direct results of + // the most recent input distribution. + totalEditableCodepoints: corrector?.correctableCodepoints ?? 0 + } } get spaceId(): number { - return this.matchingSpace.tokenization.spaceId; + return this.matchingSpace?.tokenization.spaceId; } - // /** - // * Gets the number of Damerau-Levenshtein edits needed to reach the node's - // * matchString from the output induced by the input sequence used to reach it. - // * - // * (This is scaled by `SearchSpace.EDIT_DISTANCE_COST_SCALE` when included in - // * `totalCost`.) - // */ - // get knownCost(): number { - // return this.node.editCount; - // } - - // /** - // * Gets the "input sampling cost" of the edge, which should be considered as the - // * negative log-likelihood of the input path taken to reach the node. - // */ - // get inputSamplingCost(): number { - // return this.node.inputSamplingCost; - // } + /** + * Gets the number of Damerau-Levenshtein edits needed to reach the node's + * matchString from the output induced by the input sequence used to reach it. + * + * (This is scaled by `SearchSpace.EDIT_DISTANCE_COST_SCALE` when included in + * `totalCost`.) + */ + get knownCost(): number { + return this.matchedResult.totalEditCount; + } + + /** + * Gets the "input sampling cost" of the edge, which should be considered as the + * negative log-likelihood of the input path taken to reach the node. + */ + get inputSamplingCost(): number { + return this.matchedResult.tokenCorrections.reduce((accum, curr) => accum + curr.inputSamplingCost, 0); + } /** * Gets the "total cost" of the edge, which should be considered as the @@ -40,6 +54,6 @@ export class TokenizationResultMapping implements CorrectionResultMapping total + curr.totalCost, 0); + return this.matchedResult.tokenCorrections.reduce((total, curr) => total + curr.totalCost, 0); } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts index b9f41e8f35a..bac6cceec49 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts @@ -135,7 +135,7 @@ export function transitionTokenizations( // Following call: is actually designed to build SubstitutionQuotientSpurs. const transitionedTokenization = rootTokenization.evaluateTransition(precomp[1], trueInput.id, bestProb); - const remadeTokenization = new ContextTokenization(transitionedTokenization.tokens, subset.transitionEdges.get(rootTokenization), transitionedTokenization.taillessTrueKeystroke); + const remadeTokenization = new ContextTokenization(transitionedTokenization.tokens, subset.transitionEdges.get(rootTokenization)); // If the last token is empty and has no flag for a revertable transition, // attempt to copy the previous token's revertable transition flag. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 1a514cad649..fb546e3196f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -1,15 +1,25 @@ import * as models from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { TransformUtils } from './transformUtils.js'; -import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, prependReversion, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; -import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; +import { + applySuggestionCasing, + composeIntermediatePredictions, + correctAndEnumerate, + createDefaultKeep, + dedupeSuggestions, + finalizeSuggestions, + predictionAutoSelect, + prependReversion, + processSimilarity, + toAnnotatedSuggestion, + tupleDisplayOrderSort +} from './predict-helpers.js'; +import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTracker } from './correction/context-tracker.js'; import { DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL } from './correction/distance-modeler.js'; import { ExecutionTimer } from './correction/execution-timer.js'; -import CasingForm = LexicalModelTypes.CasingForm; import Configuration = LexicalModelTypes.Configuration; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; @@ -125,24 +135,6 @@ export class ModelCompositor { const transitionId = inputTransform.id; this.initContextTracker(context, transitionId); - const allowBksp = TransformUtils.isBackspace(inputTransform); - const allowWhitespace = TransformUtils.isWhitespace(inputTransform); - - const postContext = models.applyTransform(inputTransform, context); - - // TODO: It would be best for the correctAndEnumerate method to return the - // suggestion's prefix, as it already has lots of logic oriented to this. - // The context-tracker used there with v14+ models can determine this more - // robustly. - const truePrefix = this.wordbreak(postContext); - // Only use of `truePrefix`. - const basePrefix = (allowBksp || allowWhitespace) ? truePrefix : this.wordbreak(context); - - // Used to restore whitespaces if operations would remove them. - const currentCasing: CasingForm = lexicalModel.languageUsesCasing - ? detectCurrentCasing(lexicalModel, postContext) - : null; - // Section 1: determine 'prediction roots' - enumerate corrections from most to least likely, // searching for results that yield viable predictions from the model. @@ -160,18 +152,22 @@ export class ModelCompositor { // Properly capitalizes the suggestions based on the existing context casing state. // This may result in duplicates if multiple casing options exist within the // lexicon for a word. (Example: "Apple" the company vs "apple" the fruit.) - for(let tuple of rawPredictions) { - if(currentCasing && currentCasing != 'lower') { - applySuggestionCasing(tuple.prediction.sample, basePrefix, this.lexicalModel, currentCasing); + if(lexicalModel.languageUsesCasing) { + for(let tuple of rawPredictions) { + tuple.components.forEach((component) => applySuggestionCasing(component, this.lexicalModel)); } } + // what if... we fuse suggestions together here, after the 'apply casing' step? + // deduplication, etc function fine from a fused-prediction perspective here. + // We want to dedupe before trimming the list so that we can present a full set // of viable distinct suggestions if available. - const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, rawPredictions, context); + const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, composeIntermediatePredictions(rawPredictions), context); // Needs "casing" to be applied first. - const hasExistingKeep = processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, transformDistribution[0]); + const postContext = postContextState?.context ?? models.applyTransform(inputTransform, context); + const hasExistingKeep = processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, postContext); // If no existing suggestion directly matches the user-visible version of // the token, also add a 'keep' suggestion (with `.matchesModel = false`) @@ -209,6 +205,12 @@ export class ModelCompositor { const transitionToRevert = this.contextTracker?.peek(revertableTransitionId); prependReversion(suggestions, transitionToRevert); + if(suggestions.filter((s) => s.tag == 'keep').length > 1) { + throw new Error(`Unexpected state: multiple keep suggestions exist: ${JSON.stringify(suggestions.filter((s) => s.tag == 'keep'))}`); + } else if(suggestions.filter((s) => s.tag == 'revert').length > 1) { + throw new Error(`Unexpected state: multiple revert suggestions exist! ${JSON.stringify(suggestions.filter((s) => s.tag == 'revert'))}`); + } + // Store the suggestions on the final token of the current context state (if it exists). // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root. if(postContextState) { diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts index 7008a74be55..03078bcfde5 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts @@ -71,7 +71,8 @@ export function determineModelTokenizer(model: LexicalModel) { if(model.wordbreaker) { return models.tokenize(model.wordbreaker, context); } else { - return null; + // Not ideal for pre-14.0 models, but it'll do for now. + return models.tokenize(wordBreakers.default, context); } } } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 92691c8d20b..0ea7880512b 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -4,17 +4,18 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { searchForProperty, WordBreakProperty } from '@keymanapp/models-wordbreakers'; import { TransformUtils } from './transformUtils.js'; -import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; +import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; +import { ContextToken, ContextTokenLike } from './correction/context-token.js'; import { ContextTokenization } from './correction/context-tokenization.js'; import { ContextTracker } from './correction/context-tracker.js'; -import { ContextToken } from './correction/context-token.js'; import { ContextState, determineContextSlideTransform } from './correction/context-state.js'; import { ContextTransition, TransitionReversionView } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; -import { getBestTokenMatches } from './correction/distance-modeler.js'; +import { EDIT_DISTANCE_COST_SCALE, getBestMatches } from './correction/distance-modeler.js'; +import { TokenizationCorrector, TokenResult } from './correction/tokenization-corrector.js'; +import { TokenizationResult, TokenizationResultMapping } from './correction/tokenization-result-mapping.js'; -import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; import Keep = LexicalModelTypes.Keep; @@ -73,23 +74,104 @@ export const CORRECTION_SEARCH_THRESHOLDS = { } /** - * Collates information related to suggestions during the suggestion generation - * process. + * Represents the minimum replacement range and effects required for + * suggestions. + * + * These values are based on properties of the transition from their base + * context-tokenization to their target-tokenization (and its represented + * context variant). */ -export type CorrectionPredictionTuple = { +export interface SuggestionReplacement { + /** + * Tokens lost from the base context-tokenization in the target + * context-tokenization due to the transition event. + * + * These are implicitly replaced when applying Suggestions. + */ + tokensToRemove: T[], + + /** + * Tokens added (after the removed tokens) to the base context-tokenization to + * produce the target context-tokenization. + * + * As these are "new" tokens generated by the transition, Suggestions should represent + * corrections and predictions rooted upon these tokens. + */ + tokensToPredict: T[], + + /** + * Indicates the total range of left-deletion needed when applying suggestions. + */ + deleteLeft: number, + + /** + * Indicates the id of the underlying context transition. + */ + transitionId?: number +} + +export interface TokenizedPredictionData { + /** + * The potential Suggestion + */ + prediction: Suggestion, + /** + * The correction upon which the Suggestion is based + */ + correction: string, + /** + * The unkeyed original string underlying the correction/prediction. + */ + casingRoot: string +} + +export interface CompositedPredictionData { /** * The potential Suggestion (or Keep) */ - prediction: ProbabilityMass, + prediction: Suggestion | Keep; /** * The correction upon which the Suggestion (or Keep) is based */ - correction: ProbabilityMass, + correction: string +} + +export interface PredictionProbabilities { + /** + * The probability of the word itself, separate from corrections, as + * determined by the LexicalModel itself. + */ + prediction: number; + /** - * The likelihood of the prediction - its lexical-model likelihood multiplied - * by the keystroke-sequence + correction likelihood. + * The probability of text-correction steps taken to build the correction upon + * which the prediction is based. */ - totalProb: number; + correction: number; + + /** + * The likelihood of the represented prediction, combining both the + * `prediction` and `correction` components into a single value. + */ + total: number; +} + +/** + * Tracks common intermediate prediction data, such as its underlying probabilities and its similarity to the actual context. + */ +export interface PredictionMetadata { + /** + * Tracks the relevant probability components contributing to a generated + * prediction. + */ + probabilities: PredictionProbabilities; + + /** + * Indicates that the 'suggestion' represents context changes that qualify for + * auto-selection. + */ + autoSelectable: boolean; + /** * How directly the prediction matches the current token in the context. * @@ -97,12 +179,33 @@ export type CorrectionPredictionTuple = { * available upon initial construction of this type. */ matchLevel?: SuggestionSimilarity; +} + +export interface TokenizedIntermediatePrediction { /** - * Text from the triggering input that should _not_ be affected by the - * prediction. + * Contains the tokenized components to be used to construct a full + * predictive-text Suggestion, as well as data about the source for each + * component. */ - preservationTransform?: Transform; -}; + components: TokenizedPredictionData[]; + /** + * Tracks common intermediate prediction data, such as its underlying probabilities and its similarity to the actual context. + */ + metadata: PredictionMetadata; +} + +export interface CompositedIntermediatePrediction { + /** + * Contains the fully composited predictive-text Suggestion and its underlying correction string. + */ + components: CompositedPredictionData; + /** + * Tracks common intermediate prediction data, such as its underlying probabilities and its similarity to the actual context. + */ + metadata: PredictionMetadata; +} + +type IntermediatePrediction = CompositedIntermediatePrediction | TokenizedIntermediatePrediction; /** * An enum to be used when categorizing the level of similarity between @@ -140,86 +243,74 @@ export enum SuggestionSimilarity { exact = 3 } -export function tupleDisplayOrderSort(a: CorrectionPredictionTuple, b: CorrectionPredictionTuple) { +export function tupleDisplayOrderSort(a: IntermediatePrediction, b: IntermediatePrediction) { // Similarity distance - const simDist = (b.matchLevel ?? 0) - (a.matchLevel ?? 0); + const simDist = (b.metadata.matchLevel ?? 0) - (a.metadata.matchLevel ?? 0); if(simDist != 0) { return simDist; } // Probability distance - return b.totalProb - a.totalProb; + return b.metadata.probabilities.total - a.metadata.probabilities.total; } -export async function correctAndEnumerateWithoutTraversals( +export function determineTraversallessCorrectionSequences( lexicalModel: LexicalModel, - transformDistribution: Distribution, + corrections: Distribution, context: Context -): Promise<{ - /** - * For models that support correction-search caching, this provides the - * cached object corresponding to this method's operation. - * - * Otherwise, is `null`. - */ - postContextState?: ContextState; +): PredictionParameters[] { + let returnedPredictionData: PredictionParameters[] = []; - /** - * The suggestions generated based on the user's input state. - */ - rawPredictions: CorrectionPredictionTuple[]; + const tokenizer = determineModelTokenizer(lexicalModel); - /** - * The id of a prior ContextTransition event that triggered a Suggestion found - * at the end of the Context. Will be undefined if no edits have occurred - * since the Suggestion was applied. - */ - revertableTransitionId?: number -}> { - const inputTransform = transformDistribution[0].sample; - let rawPredictions: CorrectionPredictionTuple[] = []; - - let predictionRoots: ProbabilityMass[]; - - // Only allow new-word suggestions if space was the most likely keypress. - const allowSpace = TransformUtils.isWhitespace(inputTransform); - const allowBksp = TransformUtils.isBackspace(inputTransform); + const tokenization = tokenizer(context); // issue at present if no tokens exist! + const tokenMapper = (t: models.Token) => { + return { + exampleInput: t.text, + codepointLength: KMWString.length(t.text) + } as ContextTokenLike; + } - // Generates raw prediction distributions for each valid input. Can only 'correct' - // against the final input. - // - // This is the old, 12.0-13.0 'correction' style. - if(allowSpace) { - // Detect start of new word; prevent whitespace loss here. - predictionRoots = [{sample: inputTransform, p: 1.0}]; - } else { - predictionRoots = transformDistribution.map((alt) => { - let transform = alt.sample; - - // Filter out special keys unless they're expected. - if(TransformUtils.isWhitespace(transform) && !allowSpace) { - return null; - } else if(TransformUtils.isBackspace(transform) && !allowBksp) { - return null; - } + for(let correction of corrections) { + // Step 1: determine tokenization effects. We can't use the + // ContextTokenization pattern due to the model's lack of LexiconTraversal + // support, though. + const transformId = correction.sample.id; + const postContext = models.applyTransform(correction.sample, context); + const postTokenization = tokenizer(postContext); + + const transitionEffects = determineSuggestionRange(tokenization.left.map(tokenMapper), postTokenization.left.map(tokenMapper), (a, b) => a.exampleInput == b.exampleInput); + transitionEffects.transitionId = correction.sample.id; + + // Build _multiple_ tokens to function as fake `TokenizationCorrector` + // results in order to generate a correction-root sequence. + const correctionRoots = transitionEffects.tokensToPredict.map((token, index) => { + const match: TokenResult = { + matchString: token.exampleInput, + // Key part: should be > 1 if the token is longer than just the most recent insert. + // Any extra tokens are only affected by the current input. + inputCount: index == 0 ? Math.max(1, KMWString.length(token.exampleInput) + 1 - KMWString.length(correction.sample.insert)) : 1, + inputSamplingCost: -Math.log(correction.p), + knownCost: 0, + totalCost: -Math.log(correction.p) + }; - return alt; + return match; }); - } - // Remove `null` entries. - predictionRoots = predictionRoots.filter(tuple => !!tuple); + // But, for now, only actually use the last one. + const suggestionParams = buildCorrectionSequence(transitionEffects, context, new TokenizationResultMapping([correctionRoots[correctionRoots.length - 1]], null)); + if(transformId !== undefined) { + suggestionParams.tokens.forEach((token) => token.correction.sample.id = transformId); + } - // Running in bulk over all suggestions, duplicate entries may be possible. - rawPredictions = predictFromCorrections(lexicalModel, predictionRoots, context); - if(allowSpace) { - rawPredictions.forEach((entry) => entry.preservationTransform = inputTransform); + returnedPredictionData.push({ + ...suggestionParams, + applyInPost: (p) => {} + }) } - return { - postContextState: null, - rawPredictions: rawPredictions - }; + return returnedPredictionData; } /** @@ -298,8 +389,7 @@ export function determineContextTransition( if(inputIsEmpty) { // Directly build a simple empty transition that duplicates the last seen state. // This should also clear the preservation transform if it exists! - const tokenization = new ContextTokenization(contextTracker.latest.final.tokenization.tokens); - const priorState = new ContextState(context, transition.final.model, tokenization); + const priorState = contextTracker.latest.final.transitionContextWindow(context); transition = new ContextTransition(priorState, inputTransform.id); transition.finalize(priorState, transformDistribution); } else if( @@ -320,192 +410,291 @@ export function determineContextTransition( } /** - * Determines where the context for prediction-generation should be rooted and how - * much of the context it should replace. - * @param transition - * @param lexicalModel + * Given two ContextTokenizations related by context transition, this function + * determines the tail-end range of the tokenization affected by the transition. + * @param userContextTokenization + * @param variantForSuggestions * @returns */ -export function determineSuggestionAlignment( - transition: ContextTransition, - tokenization: ContextTokenization, - lexicalModel: LexicalModel -): { - /** - * The context to use directly for generating predictions from the model. - */ - predictionContext: Context, - /** - * The total number of characters to delete from the token to be corrected. - */ - correctionDeleteLeft: number - /** - * The number of characters deleted from tokens aside from the one being corrected. - */ - committedDeleteLeft: number -} { - const transitionEdits = tokenization.transitionEdits; - const context = transition.base.context; - const postContext = transition.final.context; - const inputTransform = transition.inputDistribution[0].sample; - let deleteLeft: number; - - // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. - const wordbreak = determineModelWordbreaker(lexicalModel); +export function determineSuggestionRange( + userContextTokenization: T[], + variantForSuggestions: T[], + equalityChecker: (a: T, b: T) => boolean +): SuggestionReplacement { + // Add null/undefined guards to the equality checker. + const temp = equalityChecker; + equalityChecker = (a, b) => { + if(!a || !b) { + return false; + } - // Is the token under construction newly-constructed / is there no pre-existing root? - if(tokenization.taillessTrueKeystroke && transitionEdits?.addedNewTokens) { - return { - // If the new token is due to whitespace or due to a different input type - // that would likely imply a tokenization boundary, infer 'new word' mode. - // Apply any part of the context change that is not considered to be up - // for correction. - predictionContext: models.applyTransform(tokenization.taillessTrueKeystroke, context), - // As the word/token being corrected/predicted didn't originally exist, - // there's no part of it to 'replace'. (Suggestions are applied to the - // pre-transform state.) - correctionDeleteLeft: 0, - committedDeleteLeft: 0 - }; - // If the tokenized context length is shorter... sounds like a backspace (or similar). - } else if (transitionEdits?.removedOldTokens || TransformUtils.isBackspace(inputTransform)) { - /* Ooh, we've dropped context here. Almost certainly from a backspace or - * similar effect. Even if we drop multiple tokens... well, we know exactly - * how many chars were actually deleted - `inputTransform.deleteLeft`. Since - * we replace a word being corrected/predicted, we take length of the - * remaining context's tail token in addition to however far was deleted to - * reach that state. - */ - return { - predictionContext: models.applyTransform({...inputTransform, insert: ''}, context), - // Pre-apply delete-lefts, but do not include any inserted portion. - correctionDeleteLeft: KMWString.length(wordbreak(postContext)) - KMWString.length(inputTransform.insert), - committedDeleteLeft: inputTransform.deleteLeft - }; - } else { - // Suggestions are applied to the pre-input context, so get the token's original length. - // We're on the same token, so just delete its text for the replacement op. - deleteLeft = KMWString.length(wordbreak(context)); + return temp(a, b); } - // Did the wordbreaker (or similar) append a blank token before the caret? If so, - // preserve that by preventing corrections from triggering left-deletion. - if(tokenization.tail.isEmptyToken) { - deleteLeft = 0; + const deleteLeftCalc = (tokenSet: T[]) => { + return tokenSet.reduce((prev, curr) => prev + curr.codepointLength, 0); } - return { - predictionContext: context, - correctionDeleteLeft: deleteLeft, - committedDeleteLeft: 0 - }; -} + const tokenSetA = userContextTokenization.slice(); + const tokenSetB = variantForSuggestions.slice(); -/** - * Given two ContextTokenizations related by context transition, this function - * determines the tail-end range of the tokenization affected by the transition. - * @param userContextTokenization - * @param variantForSuggestions - * @returns - */ -export function determineSuggestionRange( - userContextTokenization: ContextTokenization, - variantForSuggestions: ContextTokenization -): { tokensToRemove: ContextToken[], tokensToPredict: ContextToken[] } { - // Assumption: spaceIds monotonically increase as new ones are generated. - // Given this, we backtrace on the token tails until finding a spot where the - // spaceIds match, dropping any that are newer than the last found in the - // other. - // - // We full-replace all tokens affected by an applied suggestion, so if there's - // a mismatch between the final form of a token, that implies that suggestions - // would replace the original form of the token anyway. - const tokenSetA = userContextTokenization.tokens.slice(); - const tokenSetB = variantForSuggestions.tokens.slice(); - - const tokensToRemove: ContextToken[] = []; - const tokensToPredict: ContextToken[] = []; - - const tailIdFor = (tokens: ContextToken[]) => tokens[tokens.length-1]?.spaceId ?? -1; - let tailOfA = tailIdFor(tokenSetA); - let tailOfB = tailIdFor(tokenSetB); - while(tailOfA != tailOfB) { - if(tailOfA < tailOfB) { - tokensToPredict.push(tokenSetB.pop()); - tailOfB = tailIdFor(tokenSetB); - } else { - tokensToRemove.push(tokenSetA.pop()); - tailOfA = tailIdFor(tokenSetA); + let aHeadIndexInB = tokenSetB.findIndex((t) => equalityChecker(t, tokenSetA[0])); + let bHeadIndexInA = tokenSetA.findIndex((t) => equalityChecker(t, tokenSetB[0])); + + if(aHeadIndexInB == -1 && bHeadIndexInA == -1) { + // Both are full replacements. + return { + tokensToRemove: tokenSetA, + tokensToPredict: tokenSetB, + deleteLeft: deleteLeftCalc(tokenSetA) } + } else if(aHeadIndexInB != 0 && bHeadIndexInA != 0) { + throw new Error("Leading edge of context should not differ in both tokenizations."); + } + + let tailOffset = 0; + while(equalityChecker(tokenSetA[bHeadIndexInA + tailOffset], tokenSetB[aHeadIndexInB + tailOffset])) { + tailOffset++; } - tokensToPredict.reverse(); + const tokensToRemove: T[] = tokenSetA.slice(bHeadIndexInA + tailOffset); + const tokensToPredict: T[] = tokenSetB.slice(aHeadIndexInB + tailOffset); // Can occur when backspacing to the end of a previous word. if(tokensToPredict.length == 0) { if(tokenSetA.length == 0 || tokenSetB.length == 0) { throw new Error("Invalid state - a tokenization is missing expected tokens"); } - tokensToRemove.push(tokenSetA.pop()); - tokensToPredict.push(tokenSetB.pop()); + tokensToRemove.unshift(tokenSetA[bHeadIndexInA + tailOffset - 1]); + tokensToPredict.unshift(tokenSetB[aHeadIndexInB + tailOffset - 1]); } - tokensToRemove.reverse(); - return { tokensToRemove, - tokensToPredict + tokensToPredict, + deleteLeft: deleteLeftCalc(tokensToRemove) + } +} + +/** + * Specifies the core, preprocessed data necessary for generating predictions, + * regardless of model type. + */ +export interface PredictionParameters { + /** + * The portion of context that should remain unchanged by generated suggestions + */ + rootContext: Context, + + /** + * A tokenization of the corrected part of the context, usable to generate + * suggestions. + * + * Note that each correction will be applied iteratively to the rootContext. + * That is, when suggesting based on the correction at index 1, the + * "unchanged" (root) context used for that suggestion will include the + * changes from the entry at index 0 (or possibly, a suggestion derived from it). + */ + tokens: { + correction: ProbabilityMass, + casingRoot: string, + autoSelectable: boolean + }[], + + deleteLeft: number; + + /** + * A closure to be applied to the generated suggestion's metadata. + * @param entry + * @returns + */ + applyInPost: (entry: TokenizedIntermediatePrediction) => void; +} + +export function buildCorrectionSequence( + transitionEffects: SuggestionReplacement, + context: Context, + tokenizationCorrection: TokenizationResultMapping +): Omit { + const { deleteLeft } = transitionEffects; + + const rootContext = models.applyTransform({insert: '', deleteLeft}, context); + + // Replace the existing context with the correction. + const orderedTokens = tokenizationCorrection.matchingSpace?.orderedTokens; + const tokens: PredictionParameters['tokens'] = []; + + for(let i = 0; i < tokenizationCorrection.matchedResult.tokenCorrections.length; i++) { + const correction = tokenizationCorrection.matchedResult.tokenCorrections[i]; + /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost + * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if + * there are significantly more likely words. We only need this to allow very minor fat-finger + * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on + * key borders. + * + * Technically, the probabilities this produces won't be normalized as-is... but there's no + * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when + * to apply it could become tricky, so it's simpler to leave out. + * + * Worst-case, it's possible to temporarily add normalization if a code deep-dive + * is needed in the future. + */ + const costFactor = (correction.inputCount <= 1) ? ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT : 1; + + const entry = { + sample: { + insert: correction.matchString, // insert correction string + deleteLeft: 0, + } as Transform, + p: Math.exp(-correction.totalCost * costFactor) + }; + + if(transitionEffects.transitionId !== undefined) { + entry.sample.id = transitionEffects.transitionId; + } + + tokens.push({ + correction: entry, + casingRoot: orderedTokens ? orderedTokens[i].exampleInput : entry.sample.insert, + autoSelectable: correctionValidForAutoSelect(entry.sample.insert) + }); } + + return { + rootContext, + tokens, + deleteLeft + }; } /** * This function takes in metadata about generated corrections (for models that - * implement Traversals) and uses that to construct predictions based upon those - * corrections. - * @param transition Context-transition data underlying the tokenization that led to the correction - * @param tokenization The tokenization from which the correction was generated. - * @param match The generated correction itself - the correction string and its cost - * @param costFactor A multiplicative factor used to adjust the cost when building prediction probabilities. + * implement Traversals) and uses that to produce the corresponding parameters + * to use for generating suggestions. + * @param transition Context-transition data underlying the tokenization that + * led to the correction + * @param tokenization The tokenization from which the correction was + * generated. + * @param match The generated correction itself - the correction string + * and its cost * @returns */ -export function buildAndMapPredictions( +export function determineTokenizedCorrectionSequence( transition: ContextTransition, tokenization: ContextTokenization, - // Originally, Readonly - but we only need these two components here. - match: Readonly<{matchString: string, totalCost: number}>, - costFactor: number -): CorrectionPredictionTuple[] { - const model = transition.final.model; - - // No matter the prediction, once we know the root of the prediction, we'll - // always 'replace' the same amount of text. We can handle this before the - // big 'prediction root' loop. - const { predictionContext, correctionDeleteLeft, committedDeleteLeft } = determineSuggestionAlignment(transition, tokenization, model); + match: TokenizationResultMapping +): PredictionParameters { + const applicationTarget = transition.base.displayTokenization; + const transitionParams = determineSuggestionRange(applicationTarget.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); + transitionParams.transitionId = transition.transitionId; + + const suggestionParams = buildCorrectionSequence(transitionParams, transition.base.context, match); + + // The correction should always be based on the most recent external + // transform/transcription ID. + if(transition.transitionId !== undefined) { + suggestionParams.tokens.forEach((t) => t.correction.sample.id = transition.transitionId); + } - let correction = match.matchString; - let rootCost = match.totalCost; + return { + ...suggestionParams, + applyInPost: (entry) => {} + }; +} - // Replace the existing context with the correction. - const correctionTransform: Transform = { - insert: correction, // insert correction string - deleteLeft: correctionDeleteLeft, - id: transition.transitionId // The correction should always be based on the most recent external transform/transcription ID. +/** + * Given the base 'display' tokenization and an array of target tokenizations + * for a context transition, this method determines the range needed for + * correction-search processes and builds the appropriate + * `TokenizationCorrector` instances for the search. + * @param transition + * @param tokenizations + * @param configuration Allows custom configuration for selecting correctable + * tokens within each TokenizationCorrector. Intended for use with unit + * testing. + * @returns + */ +export function prepareTokenizationSearch( + transition: ContextTransition, + tokenizations: ContextTokenization[], + configuration?: { + /** + * Should return true if the input index is within the appropriate range for + * correction. When called, the index of the first token in correction + * range will be provided to `rangeStart`. + * @param index + * @param rangeStart + */ + rangeValidator?: (index: number, rangeStart: number) => boolean, + /** + * Should return true if the token represents text valid for text correction + * processes, regardless of position. + * @param token + * @returns + */ + correctableValidator?: (token: ContextToken) => boolean } +) { + // Create duplicate of config parameter in order to prevent unwanted + // side-effects across multiple calls. + configuration = {...configuration}; - const predictionRoot = { - sample: correctionTransform, - p: Math.exp(-rootCost * costFactor) - }; + // Goal - determine what parts of each tokenization are searchable & prep them for correcion-search. + const tokenizationAnalyses = tokenizations.map((tokenization) => { + return { + tokenization: tokenization, + analysis: determineSuggestionRange(transition.base.displayTokenization.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId) + }; + }); - // Worth considering: extend Traversal to allow direct prediction lookups? - // let traversal = match.finalTraversal; // ... - let predictions = predictFromCorrections(model, [predictionRoot], predictionContext); - predictions.forEach((entry) => { - entry.preservationTransform = tokenization.taillessTrueKeystroke; - entry.prediction.sample.transform.deleteLeft += committedDeleteLeft; + // The token "removal" from each analysis is always based upon the + // base-state's tokenization - the same set of tokens. + // + // It is a consistent base from which all target tokenization variants are + // derived, first by removing the specified token count, then by adding on the + // edits + new additions. + + // Thus, the first step: what's the largest count removed by ANY transition + // variant? + const biggestCommonRemoval = tokenizationAnalyses.reduce( + (biggest, current) => biggest.length > current.analysis.tokensToRemove.length ? biggest : current.analysis.tokensToRemove, + [] as ContextTokenLike[] + ); + + configuration ??= {}; + // If a custom range validator is set, just use that one. + const rangeValidator = configuration.rangeValidator; + + configuration.correctableValidator ??= (token) => (token.codepointLength == 0 || correctionValidForAutoSelect(token.exampleInput)); + const tokenizationSetup = tokenizationAnalyses.map((tuple) => { + // While removed by at least one variant, this number of tokens was left in + // place, untouched, during the transition to the variant under + // consideration. These tokens are thus unaffected by the input whatsoever, + // though their probability may affect thresholding for the non-locked + // tokens. + const unaffectedTokenCount = biggestCommonRemoval.length - tuple.analysis.tokensToRemove.length; + // Unaffected tokens should still be part of the correction range; they'll + // just be marked noncorrectable. The edited + appended tokens (for actual + // correction and prediction), of course, are also part of that range. + const mutatedLength = tuple.analysis.tokensToPredict.length + unaffectedTokenCount; + + // Redefined here to capture this loop's value for `mutatedLength`. + + // If the token falls past the unaffected range... + // and, *for now*, is actually the tail token, then it may be corrected. + configuration.rangeValidator = rangeValidator ?? ((index, rangeStart) => { + return index >= rangeStart // is a modified token + && index == mutatedLength - 1 // TEMP: adjacent to the caret (TO BE REMOVED) + }); + return new TokenizationCorrector(tuple.tokenization, mutatedLength, (token, index) => { + // is within range for correction + return configuration.rangeValidator(index, unaffectedTokenCount) + // and is eligible for text-correction + && configuration.correctableValidator(token); + }); }); - return predictions; + return tokenizationSetup; } /** @@ -534,7 +723,7 @@ export async function correctAndEnumerate( /** * The suggestions generated based on the user's input state. */ - rawPredictions: CorrectionPredictionTuple[]; + rawPredictions: TokenizedIntermediatePrediction[]; /** * The id of a prior ContextTransition event that triggered a Suggestion found @@ -551,7 +740,13 @@ export async function correctAndEnumerate( // It's mostly here to support models compiled before Keyman 14.0, which was // when the `LexiconTraversal` pattern was established. if(!contextTracker) { - return correctAndEnumerateWithoutTraversals(lexicalModel, transformDistribution, context); + const predictionData = determineTraversallessCorrectionSequences(lexicalModel, transformDistribution, context); + return { + rawPredictions: predictionData.flatMap((entry) => { + const predictions = predictFromCorrectionSequence(lexicalModel, entry); + return predictions; + }) + }; } // 'else': the current, 14.0+ pattern, which is able to leverage @@ -579,56 +774,48 @@ export async function correctAndEnumerate( // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. // The 'eventual' logic will be significantly more complex, though still manageable. - const tokenizations = [transition.final.tokenization]; + const tokenizations = transition.final.tokenizations; const searchModules = tokenizations.map(t => t.tail.searchModule); + const preppedTokenizationSearch = prepareTokenizationSearch(transition, tokenizations); + // Only run the correction search when corrections are enabled. - let rawPredictions: CorrectionPredictionTuple[] = []; + let rawPredictions: TokenizedIntermediatePrediction[] = []; let bestCorrectionCost: number; - const correctionPredictionMap: Record> = {}; - for await(const match of getBestTokenMatches(searchModules, timer)) { - // Corrections obtained: now to predict from them! - const tokenization = tokenizations.find(t => t.spaceId == match.spaceId); - - // If our 'match' fully replaces the token, reject it and try again. - if(match.matchSequence.length != 0 && match.matchSequence.length == match.knownCost) { + for await(const match of getBestMatches(preppedTokenizationSearch, timer)) { + const { totalEditCount, totalEditableCodepoints } = match.matchedResult; + // If our 'match' fully replaces the tokens, reject it and try again. + // + // If the known edit count matches the total length of editable text, reject + // the suggestion source. Q: for any token, or across ALL tokens? If "for + // any", we need to distinguish between penalization where corrections + // couldn't be found and where there's actual edit cost. (2.5 edits does + // stand out a bit, but we should do something more robust.) + if(totalEditCount != 0 && totalEditableCodepoints == totalEditCount) { // TODO: double-check approach! continue; } - if(match.editCount > 0 && !searchModules.find(s => s.correctionsEnabled)) { + // Perhaps make a return object, add a cumulative 'editCount' property? + // Or, we could just sum it up here. + if(totalEditCount > 0 && !searchModules.find(s => s.correctionsEnabled)) { continue; } - /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost - * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if - * there are significantly more likely words. We only need this to allow very minor fat-finger - * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on - * key borders. - * - * Technically, the probabilities this produces won't be normalized as-is... but there's no - * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when - * to apply it could become tricky, so it's simpler to leave out. - * - * Worst-case, it's possible to temporarily add normalization if a code deep-dive - * is needed in the future. - */ - const costFactor = (tokenization.tail.inputCount <= 1) ? ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT : 1; + // Worth considering: extend Traversal to allow direct prediction lookups? + // let traversal = match.finalTraversal; - const predictions = buildAndMapPredictions(transition, tokenization, match, costFactor); + const tokenization = match.matchingSpace.tokenization; + const suggestionRange = determineSuggestionRange(transition.base.displayTokenization.tokens, tokenization.tokens, (a, b) => a.spaceId == b.spaceId); + suggestionRange.transitionId = transition.transitionId; + const predictionPrep = determineTokenizedCorrectionSequence(transition, tokenization, match); - // Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions. - if(predictions.length > 0 && bestCorrectionCost === undefined) { - bestCorrectionCost = match.totalCost * costFactor; - } + const predictions = predictFromCorrectionSequence(lexicalModel, predictionPrep); - // If we're getting the same prediction again, it's lower-cost. Update! - let oldPredictionSet = correctionPredictionMap[match.matchString]; - if(oldPredictionSet) { - rawPredictions = rawPredictions.filter((entry) => !oldPredictionSet.find((match) => entry.prediction.sample == match.sample)); + // Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions. + if(predictions.length > 0 && (bestCorrectionCost === undefined || bestCorrectionCost > match.totalCost)) { + bestCorrectionCost = match.totalCost; } - correctionPredictionMap[match.matchString] = predictions.map((entry) => entry.prediction); - rawPredictions = rawPredictions.concat(predictions); if(shouldStopSearchingEarly(bestCorrectionCost, match.totalCost, rawPredictions)) { @@ -649,7 +836,7 @@ export async function correctAndEnumerate( export function shouldStopSearchingEarly( bestCorrectionCost: number, currentCorrectionCost: number, - rawPredictions: CorrectionPredictionTuple[] + rawPredictions: TokenizedIntermediatePrediction[] ) { if(currentCorrectionCost >= bestCorrectionCost + CORRECTION_SEARCH_THRESHOLDS.MAX_SEARCH_THRESHOLD) { return true; @@ -665,7 +852,7 @@ export function shouldStopSearchingEarly( // If the best suggestion from the search's current tier fails to beat the worst // pending suggestion from previous tiers, assume all further corrections will // similarly fail to win; terminate the search-loop. - if(rawPredictions[ModelCompositor.MAX_SUGGESTIONS-1].totalProb > Math.exp(-currentCorrectionCost)) { + if(rawPredictions[ModelCompositor.MAX_SUGGESTIONS-1].metadata.probabilities.total > Math.exp(-currentCorrectionCost)) { return true; } } @@ -678,73 +865,207 @@ export function shouldStopSearchingEarly( * Given a generated set of corrections from the correction-search process, this * function searches the lexical model for valid predictions rooted on each. * - * While doing so, it also associates each prediction with metadata used for - * to "rank" and select the best predictions once the search is complete. This - * is performed at later stages. + * While doing so, it also associates each prediction with metadata used to + * "rank" and select the best predictions once the search is complete. This is + * performed at later stages. * @param lexicalModel - * @param corrections - * @param context + * @param predictionPrep * @returns */ -export function predictFromCorrections( +export function predictFromCorrectionSequence( lexicalModel: LexicalModel, - corrections: ProbabilityMass[], - context: Context -): CorrectionPredictionTuple[] { - let returnedPredictions: CorrectionPredictionTuple[] = []; - const wordbreak = determineModelWordbreaker(lexicalModel); + predictionPrep: PredictionParameters +): TokenizedIntermediatePrediction[] { + let successfulPredictions = 0; + const correctionTokens = predictionPrep.tokens; + const context = predictionPrep.rootContext; + let currentContext = context; + let prefixProb = 1; + + const predictionComponents = correctionTokens.map((correctionToken, i) => { + const correctionTransform = correctionToken.correction.sample; + let predictions = lexicalModel.predict(correctionTransform, currentContext); + const transitionId = correctionTransform.id; + + // Ensure codepointLength == prediction codepoint length if i does not match the tail! + // Filter out cases that do not conform to this condition. + if(i != correctionTokens.length - 1) { + predictions = predictions.filter((p) => { + const codepointLength = KMWString.length(correctionToken.correction.sample.insert); + return KMWString.length(p.sample.transform.insert) == codepointLength; + }); + } - for(let correction of corrections) { - let predictions = lexicalModel.predict(correction.sample, context); + // Failsafe: if there are no matching predictions, create a fake prediction + // matching the original text. + if(predictions.length != 0) { + successfulPredictions++; + } else { + const fallbackSuggestion = { + sample: { + transform: {...correctionTransform}, + displayAs: correctionTransform.insert + }, + // It's not found in the lexicon, so we'll take a low probability for it. + // + // Edit penalties will be applied via the correction component separately later on. + p: Math.exp(-EDIT_DISTANCE_COST_SCALE) + }; - const { sample: correctionTransform, p: correctionProb } = correction; - const correctionRoot = wordbreak(models.applyTransform(correction.sample, context)); + predictions.push(fallbackSuggestion); + } + + // Regardless of origin, overwrite the transform's deleteLeft value with what it should actually hold. + predictions.forEach((entry) => { + // Remove the `p` field from the Dummy model's mocked suggestions; these should not be emitted. + delete (entry.sample as Outcome).p; - let predictionSet = predictions.map((pair: ProbabilityMass) => { - // Let's not rely on the model to copy transform IDs. - // Only bother is there IS an ID to copy. - if(correctionTransform.id !== undefined) { - pair.sample.transform.id = correctionTransform.id; + entry.sample.transform.deleteLeft = correctionTransform.deleteLeft; + if(transitionId !== undefined) { + entry.sample.transform.id = transitionId; } + }); - let tuple: CorrectionPredictionTuple = { - prediction: pair, - correction: { - sample: correctionRoot, - p: correctionProb - }, - totalProb: pair.p * correctionProb, - matchLevel: SuggestionSimilarity.none + // Use traversals if possible - extract the most likely entry that is on the traversal, + // rather than predicting (and possibly extending) tokens not adjacent to the caret. + // + // Also, fall back to the actual correction string should prediction not be valid here. + const isLastToken = i == correctionTokens.length - 1; + const predictionsToReturn = isLastToken ? predictions : [predictions[0]]; + + if(!isLastToken) { + prefixProb *= predictions[0].p; + } + + // In case future models can use bigrams or similar strategies to adjust predictions based + // on prior tokens, we update the root context for following iterations with the current + // correction. (This could be improved by trying variations of valid predictions, but that's + // currently out-of-scope.) + currentContext = models.applyTransform(correctionToken.correction.sample, currentContext); + + return predictionsToReturn.map((prediction) => { + return { + prediction: prediction.sample, + correction: correctionTransform.insert, + casingRoot: correctionToken.casingRoot, + // This is tagged on as an addition because we need each final + // token-prediction's probability to be available in the next loop + // below. + predictionProb: prediction.p, + autoSelectable: correctionToken.autoSelectable }; - return tuple; }); + }); - returnedPredictions = returnedPredictions.concat(predictionSet); + if(successfulPredictions == 0) { + return []; } - return returnedPredictions; + // Constructs a common prefix for all but the final token's component. + const correctionCost = correctionTokens.reduce((accum, curr) => accum * curr.correction.p, 1); + const predictionPrefix = predictionComponents + .slice(0, predictionComponents.length-1) + .map((p) => p[0]); + + const completePredictionTuples: TokenizedIntermediatePrediction[] = predictionComponents[predictionComponents.length-1].map((tuple) => { + const predictionCost = tuple.predictionProb * prefixProb; + + const returnVal: TokenizedIntermediatePrediction = { + components: [...predictionPrefix, tuple], + metadata: { + probabilities: { + prediction: predictionCost, + correction: correctionCost, + total: predictionCost * correctionCost + }, + autoSelectable: tuple.autoSelectable, + matchLevel: SuggestionSimilarity.none + } + } + + returnVal.components[0].prediction.transform.deleteLeft = predictionPrep.deleteLeft; + + return returnVal; + }); + + completePredictionTuples.forEach((pt) => predictionPrep.applyInPost(pt)); + + return completePredictionTuples; } /** * Applies the specified casing-form to generated suggestions, leveraging the model's * defined casing behaviors to do so. - * @param suggestion - * @param baseWord + * @param predictionToken A tuple representing a single context token's prediction and base correction * @param lexicalModel - * @param casingForm */ -export function applySuggestionCasing(suggestion: Suggestion, baseWord: string, lexicalModel: LexicalModel, casingForm: CasingForm) { - // Step 1: does the suggestion replace the whole word? If not, we should extend the suggestion to do so. - let unchangedLength = KMWString.length(baseWord) - suggestion.transform.deleteLeft; +export function applySuggestionCasing(predictionToken: TokenizedPredictionData, lexicalModel: LexicalModel) { + // Step 0: our pattern for generating predictions and corrections already + // enforces that they encompass the whole word. + const suggestion = predictionToken.prediction; - if(unchangedLength > 0) { - suggestion.transform.deleteLeft += unchangedLength; - suggestion.transform.insert = KMWString.substr(baseWord, 0, unchangedLength) + suggestion.transform.insert; + // If we are using the context-tracking engine (when traversals are enabled), + // we just leverage the context token's exampleInput to determine casing. + // + // If it's not available, the correction entry reflects a word-broken piece of + // the original context, with its original casing - so we use that instead. + const casingRoot = predictionToken.casingRoot ? predictionToken.casingRoot : predictionToken.correction; + if(!casingRoot) { + // There's no text in place to verify casing expectations; just leave it + // unchanged. + return; } + // Step 1: detect the original token's casing + const casingForm = detectCurrentCasing(lexicalModel, { + left: casingRoot, + startOfBuffer: true, + endOfBuffer: true + }); + // Step 2: Now that the transform affects the whole word, we may safely apply casing rules. - suggestion.transform.insert = lexicalModel.applyCasing(casingForm, suggestion.transform.insert); - suggestion.displayAs = lexicalModel.applyCasing(casingForm, suggestion.displayAs); + if(casingForm && casingForm != 'lower') { + suggestion.transform.insert = lexicalModel.applyCasing(casingForm, suggestion.transform.insert); + suggestion.displayAs = lexicalModel.applyCasing(casingForm, suggestion.displayAs); + } +} + +/** + * Composes a set of `IntermediateTokenizedPrediction`s, merging the tokenized + * data into corresponding `CompositedIntermediatePrediction`s representing the + * full range of affected context. + * @param predictions + * @returns + */ +export function composeIntermediatePredictions(predictions: TokenizedIntermediatePrediction[]): CompositedIntermediatePrediction[] { + return predictions.map((predictionData) => { + const components = predictionData.components; + + const reduceBaseTransform: Transform = { + insert: '', + deleteLeft: 0 + } + const transformId = predictionData.components[0].prediction.transform.id; + if(transformId !== undefined) { + reduceBaseTransform.id = transformId; + } + + return { + components: components.reduce((total, current) => { + const mergedTransform = models.buildMergedTransform(total.prediction.transform, current.prediction.transform); + const mergedDisplayAs = total.prediction.displayAs + current.prediction.displayAs + + return { + prediction: {...total.prediction, transform: mergedTransform, displayAs: mergedDisplayAs}, + correction: total.correction + current.correction + } + }, { + prediction: {...components[0].prediction, transform: reduceBaseTransform, displayAs: ''}, + correction: '' + }), + metadata: predictionData.metadata + }; + }); } /** @@ -760,17 +1081,17 @@ export function applySuggestionCasing(suggestion: Suggestion, baseWord: string, */ export function dedupeSuggestions( lexicalModel: LexicalModel, - rawPredictions: CorrectionPredictionTuple[], + rawPredictions: CompositedIntermediatePrediction[], context: Context ) { const wordbreak = determineModelWordbreaker(lexicalModel); - let suggestionDistribMap: {[key: string]: CorrectionPredictionTuple} = {}; - let suggestionDistribution: CorrectionPredictionTuple[] = []; + let suggestionDistribMap: {[key: string]: CompositedIntermediatePrediction} = {}; + let suggestionDistribution: CompositedIntermediatePrediction[] = []; // Deduplicator + annotator of 'keep' suggestions. for(let tuple of rawPredictions) { - const predictedWord = wordbreak(models.applyTransform(tuple.prediction.sample.transform, context)); + const predictedWord = wordbreak(models.applyTransform(tuple.components.prediction.transform, context)); // Assumption: suggestions that have the same net result should have the // same displayAs string. (We could try to pick the one with highest net @@ -780,7 +1101,7 @@ export function dedupeSuggestions( // Merge 'em! const existingSuggestion = suggestionDistribMap[predictedWord]; if(existingSuggestion) { - existingSuggestion.totalProb += tuple.totalProb; + existingSuggestion.metadata.probabilities.total += tuple.metadata.probabilities.total; } else { suggestionDistribMap[predictedWord] = tuple; } @@ -808,66 +1129,61 @@ export function dedupeSuggestions( * current text * - any other suggestion * + * @param lexicalModel * @param suggestionDistribution - * @param context - * @param trueInput inputTransform + its assigned probability + * @param baseContext + * @param finalContext * @returns true if an existing suggestion fulfills the role of 'keep'; * otherwise, false. */ export function processSimilarity( lexicalModel: LexicalModel, - suggestionDistribution: CorrectionPredictionTuple[], - context: Context, - trueInput: ProbabilityMass + suggestionDistribution: CompositedIntermediatePrediction[], + baseContext: Context, + finalContext: Context ): boolean { - const { sample: inputTransform } = trueInput; const wordbreak = determineModelWordbreaker(lexicalModel); - const postContext = models.applyTransform(inputTransform, context); - const truePrefix = wordbreak(postContext); - const keyed = (text: string) => lexicalModel.toKey ? lexicalModel.toKey(text) : text; const keyCased = (text: string) => lexicalModel.applyCasing ? lexicalModel.applyCasing('lower', text) : text; - const keyedPrefix = keyed(truePrefix); - const lowercasedPrefix = keyCased(truePrefix); + const keyedTarget = keyed(finalContext.left); + const lowercasedTarget = keyCased(finalContext.left); let keepOption: Outcome; - for(let tuple of suggestionDistribution) { - // Don't set it unnecessarily; this can have side-effects in some automated tests. - if(inputTransform.id !== undefined) { - tuple.prediction.sample.transform.id = inputTransform.id; - } + // If there are no suggestions found, we can't validate that the underlying + // correction was an empty token. + let allCorrectionsEmpty: boolean = suggestionDistribution.length > 0 + ? true + : wordbreak(finalContext) == ''; - const predictedWord = wordbreak(models.applyTransform(tuple.prediction.sample.transform, context)); + for(let tuple of suggestionDistribution) { + const appliedContext = models.applyTransform(tuple.components.prediction.transform, baseContext); + allCorrectionsEmpty &&= tuple.components.correction == ''; // Is the suggestion an exact match (or, "similar enough") to the // actually-typed context? If so, we wish to note this fact and to // prioritize such a suggestion over suggestions that are not. - if(keyed(tuple.correction.sample) == keyedPrefix) { - if(predictedWord == truePrefix) { - // Exact match: it's a perfect 'keep' suggestion. - tuple.matchLevel = SuggestionSimilarity.exact; - keepOption = toAnnotatedSuggestion(lexicalModel, tuple.prediction.sample, 'keep', models.QuoteBehavior.noQuotes); - - // Indicates that this suggestion exists directly within the lexical - // model as a valid suggestion. (We actively display it if it's an - // exact match, but hide it if not, only preserving it for reversions - // if/when needed.) - keepOption.matchesModel = true; - Object.assign(tuple.prediction.sample, keepOption); - keepOption = tuple.prediction.sample as Outcome; - } else if(keyCased(predictedWord) == lowercasedPrefix) { - // Case-insensitive match. No diacritic differences; the ONLY difference is casing. - tuple.matchLevel = SuggestionSimilarity.sameText; - } else if(keyed(predictedWord) == keyedPrefix) { - // Diacritic-insensitive / exact-key match. - tuple.matchLevel = SuggestionSimilarity.sameKey; - } else { - tuple.matchLevel = SuggestionSimilarity.none; - } + if(appliedContext.left == finalContext.left) { + // Exact match: it's a perfect 'keep' suggestion. + tuple.metadata.matchLevel = SuggestionSimilarity.exact; + keepOption = toAnnotatedSuggestion(lexicalModel, tuple.components.prediction, 'keep', models.QuoteBehavior.noQuotes); + + // Indicates that this suggestion exists directly within the lexical + // model as a valid suggestion. (We actively display it if it's an + // exact match, but hide it if not, only preserving it for reversions + // if/when needed.) + keepOption.matchesModel = true; + Object.assign(tuple.components.prediction, keepOption); + keepOption = tuple.components.prediction as Outcome; + } else if(keyCased(appliedContext.left) == lowercasedTarget) { + // Case-insensitive match. No diacritic differences; the ONLY difference is casing. + tuple.metadata.matchLevel = SuggestionSimilarity.sameText; + } else if(keyed(appliedContext.left) == keyedTarget) { + // Diacritic-insensitive / exact-key match. + tuple.metadata.matchLevel = SuggestionSimilarity.sameKey; } else { - tuple.matchLevel = SuggestionSimilarity.none; + tuple.metadata.matchLevel = SuggestionSimilarity.none; } } @@ -875,12 +1191,12 @@ export function processSimilarity( // // No actual 'keep' needed if the current context token is empty, so we say we // have a 'keep' for that case, even though there isn't really one. - return !!(keepOption || truePrefix == ''); + return !!(keepOption || allCorrectionsEmpty); } /** * Generates metadata for a new 'keep' suggestion based solely upon the existing - * context. + * context and the most likely input. * * This method is designed for use when no appropriate 'keep' suggestion was * generated by the correction-search process. @@ -893,22 +1209,31 @@ export function createDefaultKeep( lexicalModel: LexicalModel, context: Context, trueInput: ProbabilityMass -): CorrectionPredictionTuple { +): CompositedIntermediatePrediction { const { sample: inputTransform, p: inputTransformProb } = trueInput; const wordbreak = determineModelWordbreaker(lexicalModel); + const tokenizer = determineModelTokenizer(lexicalModel); const postContext = models.applyTransform(inputTransform, context); const truePrefix = wordbreak(postContext); - // Generate a full-word 'keep' replacement like other suggestions when one is not otherwise - // produced; we want to replace the full token in the same manner used for other suggestions. - const basePrefixLength = KMWString.length(truePrefix) - KMWString.length(inputTransform.insert) + inputTransform.deleteLeft; - const keepTransform = { - insert: truePrefix, - deleteLeft: basePrefixLength - }; + const tokenization = tokenizer(context); + const postTokenization = tokenizer(postContext); + + const tokenMapper = (t: models.Token) => { + return { + exampleInput: t.text, + codepointLength: KMWString.length(t.text) + } as ContextTokenLike; + } + const transitionEffects = determineSuggestionRange(tokenization.left.map(tokenMapper), postTokenization.left.map(tokenMapper), (a, b) => a.exampleInput == b.exampleInput); + + const keepTransition = { + insert: transitionEffects.tokensToPredict.reduce((accum, curr) => accum + curr.exampleInput, ''), + deleteLeft: transitionEffects.tokensToRemove.reduce((accum, curr) => accum + curr.codepointLength, 0) + } + let keepSuggestion = models.transformToSuggestion(keepTransition); - let keepSuggestion = models.transformToSuggestion(keepTransform); // This is the one case where the transform doesn't insert the full word; we need to override the displayAs param. keepSuggestion.displayAs = truePrefix; @@ -920,19 +1245,19 @@ export function createDefaultKeep( // Insert our synthetic keepOption as a prediction tuple. return { - // Product of the two p's below. - totalProb: inputTransformProb * MAX_PROB, - prediction: { - sample: keepOption, - // We always show the keep option if it doesn't directly match, - // so max probability is fine. - p: MAX_PROB, + components: { + prediction: keepOption, + correction: truePrefix }, - correction: { - sample: truePrefix, - p: inputTransformProb * MAX_PROB - }, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: MAX_PROB, + correction: inputTransformProb, + total: inputTransformProb * MAX_PROB + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; } @@ -965,12 +1290,12 @@ export function correctionValidForAutoSelect(correction: string) { return false; } -export function predictionAutoSelect(suggestionDistribution: CorrectionPredictionTuple[]) { +export function predictionAutoSelect(suggestionDistribution: CompositedIntermediatePrediction[]) { if(suggestionDistribution.length == 0) { return; } - const keepOption = suggestionDistribution[0].prediction.sample as Outcome; + const keepOption = suggestionDistribution[0].components.prediction as Outcome; if(keepOption.tag == 'keep' && keepOption.matchesModel) { // Do not auto-select 'keep' suggestions'; there's no need to apply them. // @@ -986,19 +1311,19 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio if(suggestionDistribution.length == 1) { // Prevent auto-acceptance when the root doesn't meet validation criteria. - if(!correctionValidForAutoSelect(suggestionDistribution[0].correction.sample)) { + if(!suggestionDistribution[0].metadata.autoSelectable) { return; } // Mark for auto-acceptance; there are no alternatives. - suggestionDistribution[0].prediction.sample.autoAccept = true; + suggestionDistribution[0].components.prediction.autoAccept = true; return; } // Is it reasonable to auto-accept any of our suggestions? const bestSuggestion = suggestionDistribution[0]; - const baseCorrection = bestSuggestion.correction.sample; + const baseCorrection = bestSuggestion.components.correction; if(baseCorrection.length == 0) { // If the correction is rooted on an empty root, there's no basis for // auto-correcting to this suggestion. @@ -1007,8 +1332,8 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio // Find the highest probability for any correction that led to a valid prediction. // No need to full-on re-sort everything, though. - const bestCorrection = suggestionDistribution.reduce((prev, current) => prev?.correction.p > current.correction.p ? prev : current, null).correction; - if(bestCorrection.p > bestSuggestion.correction.p) { + const bestCorrectionP = suggestionDistribution.reduce((prev, current) => Math.max(prev, current.metadata.probabilities.correction), 0); + if(bestCorrectionP > bestSuggestion.metadata.probabilities.correction) { // Here, the best suggestion didn't come from the best correction. // Is it actually reasonable to auto-correct? We're probably just very // biased toward its frequency. (Maybe a threshold should be considered?) @@ -1019,28 +1344,28 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio // - such as replacing `cant` with `can't` if the latter is much more frequent - // we may wish to group matchLevel values below by 'mapping' them with an appropriate // function. (Both on the next line and within the reduce functor.) - const bestSuggestionTier = bestSuggestion.matchLevel; + const bestSuggestionTier = bestSuggestion.metadata.matchLevel; // compare best vs other probabilities of compatible tier. const probSum = suggestionDistribution.reduce((accum, current) => { // If the suggestion is from a different similarity tier, do not count it against // the required auto-select probability ratio threshold. That threshold should // only apply within the suggestion's tier. - return accum + (current.matchLevel == bestSuggestionTier ? current.totalProb : 0) + return accum + (current.metadata.matchLevel == bestSuggestionTier ? current.metadata.probabilities.total : 0) }, 0); - const proportionOfBest = bestSuggestion.totalProb / probSum; + const proportionOfBest = bestSuggestion.metadata.probabilities.total / probSum; if(proportionOfBest < AUTOSELECT_PROPORTION_THRESHOLD) { return; } - if(!correctionValidForAutoSelect(bestSuggestion.correction.sample)) { + if(!bestSuggestion.metadata.autoSelectable) { return; } // compare correction-cost aspects? We disable if the base correction is lower than best, // but should we do other comparisons too? - bestSuggestion.prediction.sample.autoAccept = true; + bestSuggestion.components.prediction.autoAccept = true; } /** @@ -1061,7 +1386,7 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio */ export function finalizeSuggestions( lexicalModel: LexicalModel, - deduplicatedSuggestionTuples: CorrectionPredictionTuple[], + deduplicatedSuggestionTuples: CompositedIntermediatePrediction[], context: Context, inputTransform: Transform, verbose?: boolean @@ -1070,43 +1395,20 @@ export function finalizeSuggestions( const tokenize = determineModelTokenizer(lexicalModel); const suggestions = deduplicatedSuggestionTuples.map((tuple) => { - const prediction = tuple.prediction; - - // If this is a suggestion after any form of wordbreak input, make sure we preserve any components - // from prior tokens! - // - // Note: may need adjustment if/when supporting phrase-level correction. - if(tuple.preservationTransform) { - const presDL = tuple.preservationTransform.deleteLeft; - const mergedTransform = models.buildMergedTransform(tuple.preservationTransform, prediction.sample.transform); - // Any preserved delete-left is applied early because it directly affects the suggestion - // root; we need to remove that preserved delete-left here. - if(presDL > 0) { - mergedTransform.deleteLeft -= presDL; - } - if(prediction.sample.transform.id !== undefined) { - mergedTransform.id = prediction.sample.transform.id; - } - - // Temporarily and locally drops 'readonly' semantics so that we can reassign the transform. - // See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers - let mutableSuggestion = prediction.sample as {-readonly [transform in keyof Suggestion]: Suggestion[transform]}; - - // Assignment via by-reference behavior, as suggestion is an object - mutableSuggestion.transform = mergedTransform; - } + const prediction = tuple.components.prediction; + const probs = tuple.metadata.probabilities; if(!verbose) { return { - ...prediction.sample, - p: tuple.totalProb + ...prediction, + p: probs.total }; } else { const sample: Outcome = { - ...prediction.sample, - p: tuple.totalProb, - "lexical-p": prediction.p, - "correction-p": tuple.correction.p + ...prediction, + p: probs.total, + "lexical-p": probs.prediction, + "correction-p": probs.correction } return sample; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index bcaf4692bf7..c5959e919b7 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -7,6 +7,9 @@ export * from './correction/context-transition.js'; export * from './correction/correction-searchable.js'; export * from './correction/correction-result-mapping.js'; export * from './correction/distance-modeler.js'; +export * from './correction/deletion-quotient-spur.js'; +export * from './correction/insertion-quotient-spur.js'; +export * from './correction/substitution-quotient-spur.js'; export * from './correction/execution-timer.js'; export * from './correction/search-quotient-cluster.js'; export * from './correction/search-quotient-spur.js'; diff --git a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.ts b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.ts index 15faf2189fd..530cce052f8 100644 --- a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.ts +++ b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.ts @@ -21,32 +21,32 @@ LMLayerWorker.loadModel(new models.DummyModel({ const appleDummySuggestionSets: Suggestion[][] = [[ // Set 1: { - transform: { insert: 'e', deleteLeft: 0}, + transform: { insert: 'apple', deleteLeft: 0}, displayAs: 'apple', }, { - transform: { insert: 'y', deleteLeft: 0}, + transform: { insert: 'apply', deleteLeft: 0}, displayAs: 'apply' }, { - transform: { insert: 'es', deleteLeft: 0}, + transform: { insert: 'apples', deleteLeft: 0}, displayAs: 'apples' } ], [ // Set 2: { - transform: { insert: 'e', deleteLeft: 0}, + transform: { insert: 'apple', deleteLeft: 0}, displayAs: 'apple', tag: 'keep' }, { - transform: { insert: 'y', deleteLeft: 0}, + transform: { insert: 'apply', deleteLeft: 0}, displayAs: 'apply' }, { - transform: { insert: 's', deleteLeft: 1}, + transform: { insert: 'apps', deleteLeft: 1}, displayAs: 'apps' } ], [ // Set 3: { - transform: { insert: 'ied', deleteLeft: 2}, + transform: { insert: 'applied', deleteLeft: 2}, displayAs: 'applied' } ], [ @@ -101,7 +101,7 @@ describe("PredictionContext", () => { suggestions = updateFake.secondCall.args[0]; assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apples']); assert.isNotOk(suggestions.find((obj) => obj.tag == 'keep')); - assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 0)); + assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 4)); textStore.insertTextBeforeCaret('e'); // appl| + e = apple let transcription = textStore.buildTranscriptionFrom(initialTextStore, null, true); @@ -113,7 +113,7 @@ describe("PredictionContext", () => { suggestions = updateFake.thirdCall.args[0]; assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apps']); assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, 'apple'); - assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'apps'); + assert.isOk(suggestions.find((obj) => obj.displayAs == 'apps')); }); it('ignores outdated predictions', async function () { @@ -140,14 +140,17 @@ describe("PredictionContext", () => { suggestions = updateFake.secondCall.args[0]; assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apples']); assert.isNotOk(suggestions.find((obj) => obj.tag == 'keep')); - assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 0)); + assert.isNotOk(suggestions.find((obj) => obj.transform.deleteLeft != 4)); + textStore = SyntheticTextStore.from(initialTextStore); + textStore.insertTextBeforeCaret('e'); const baseTranscription = textStore.buildTranscriptionFrom(initialTextStore, null, true); // Mocking: corresponds to the second set of mocked predictions - round 2 of // 'apple', 'apply', 'apples'. const skippedPromise = langProcessor.predict(baseTranscription, dummiedGetLayer()); + textStore = SyntheticTextStore.from(initialTextStore); textStore.insertTextBeforeCaret('e'); // appl| + e = apple const finalTranscription = textStore.buildTranscriptionFrom(initialTextStore, null, true); @@ -219,7 +222,7 @@ describe("PredictionContext", () => { suggestions = updateFake.firstCall.args[0]; assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['apple', 'apply', 'apps']); assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, 'apple'); - assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'apps'); + assert.isOk(suggestions.find((obj) => obj.displayAs == 'apps')); // Now for the real test. previousTextState = SyntheticTextStore.from(textState); // snapshot it! diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts index 86780824ac6..11517476939 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildAlphabeticClusteredFixture.ts @@ -13,9 +13,9 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs' import { models, - LegacyQuotientSpur, SearchQuotientCluster, - LegacyQuotientRoot + SearchQuotientRoot, + SubstitutionQuotientSpur } from '@keymanapp/lm-worker/test-index'; import Distribution = LexicalModelTypes.Distribution; @@ -31,7 +31,7 @@ const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); * @returns */ export const buildAlphabeticClusterFixtures = () => { - const rootPath = new LegacyQuotientRoot(testModel); + const rootPath = new SearchQuotientRoot(testModel); // consonant-cluster 1, insert 1, delete 0 const distrib_c1_i1d0: Distribution = [ @@ -48,9 +48,9 @@ export const buildAlphabeticClusterFixtures = () => { ]; // keystrokes 1, codepoints 1, total inserts 1, delete 0 - const path_k1c1_i1d0 = new LegacyQuotientSpur(rootPath, distrib_c1_i1d0, distrib_c1_i1d0[0]); + const path_k1c1_i1d0 = new SubstitutionQuotientSpur(rootPath, distrib_c1_i1d0, distrib_c1_i1d0[0]); // keystrokes 1, codepoints 2, total inserts 2, delete 0 - const path_k1c2_i2d0 = new LegacyQuotientSpur(rootPath, distrib_c1_i2d0, distrib_c1_i1d0[0]); + const path_k1c2_i2d0 = new SubstitutionQuotientSpur(rootPath, distrib_c1_i2d0, distrib_c1_i1d0[0]); // Second input @@ -62,8 +62,8 @@ export const buildAlphabeticClusterFixtures = () => { { sample: { insert: 'u', deleteLeft: 0, deleteRight: 0, id: 12 }, p: 0.1 }, ]; - const path_k2c2_i2d0 = new LegacyQuotientSpur(path_k1c1_i1d0, distrib_v1_i1d0, distrib_v1_i1d0[0]); - const path_k2c3_i3d0 = new LegacyQuotientSpur(path_k1c2_i2d0, distrib_v1_i1d0, distrib_v1_i1d0[0]); + const path_k2c2_i2d0 = new SubstitutionQuotientSpur(path_k1c1_i1d0, distrib_v1_i1d0, distrib_v1_i1d0[0]); + const path_k2c3_i3d0 = new SubstitutionQuotientSpur(path_k1c2_i2d0, distrib_v1_i1d0, distrib_v1_i1d0[0]); // Third input const distrib_v2_i1d0: Distribution = [ @@ -90,15 +90,15 @@ export const buildAlphabeticClusterFixtures = () => { { sample: { insert: 'úú', deleteLeft: 1, deleteRight: 0, id: 13 }, p: 0.02 }, ]; // 0.2 total - const path_k3c2_i3d1 = new LegacyQuotientSpur(path_k2c2_i2d0, distrib_v2_i1d1, distrib_v2_i1d0[0]); + const path_k3c2_i3d1 = new SubstitutionQuotientSpur(path_k2c2_i2d0, distrib_v2_i1d1, distrib_v2_i1d0[0]); - const path_k3c3_i3d0 = new LegacyQuotientSpur(path_k2c2_i2d0, distrib_v2_i1d0, distrib_v2_i1d0[0]); - const path_k3c3_i4d1a = new LegacyQuotientSpur(path_k2c2_i2d0, distrib_v2_i2d1, distrib_v2_i1d0[0]); - const path_k3c3_i4d1b = new LegacyQuotientSpur(path_k2c3_i3d0, distrib_v2_i1d1, distrib_v2_i1d0[0]); + const path_k3c3_i3d0 = new SubstitutionQuotientSpur(path_k2c2_i2d0, distrib_v2_i1d0, distrib_v2_i1d0[0]); + const path_k3c3_i4d1a = new SubstitutionQuotientSpur(path_k2c2_i2d0, distrib_v2_i2d1, distrib_v2_i1d0[0]); + const path_k3c3_i4d1b = new SubstitutionQuotientSpur(path_k2c3_i3d0, distrib_v2_i1d1, distrib_v2_i1d0[0]); // both are built on path k1c2 (splits at index 1) - const path_k3c4_i4d0 = new LegacyQuotientSpur(path_k2c3_i3d0, distrib_v2_i1d0, distrib_v2_i1d0[0]); - const path_k3c4_i5d1 = new LegacyQuotientSpur(path_k2c3_i3d0, distrib_v2_i2d1, distrib_v2_i1d0[0]); + const path_k3c4_i4d0 = new SubstitutionQuotientSpur(path_k2c3_i3d0, distrib_v2_i1d0, distrib_v2_i1d0[0]); + const path_k3c4_i5d1 = new SubstitutionQuotientSpur(path_k2c3_i3d0, distrib_v2_i2d1, distrib_v2_i1d0[0]); const cluster_k3c3 = new SearchQuotientCluster([path_k3c3_i3d0, path_k3c3_i4d1a, path_k3c3_i4d1b]); // both are built on path k1c2. @@ -116,13 +116,13 @@ export const buildAlphabeticClusterFixtures = () => { { sample: { insert: 'vw', deleteLeft: 0, deleteRight: 0, id: 14 }, p: 0.1 } ]; - const path_k4c4_i2 = new LegacyQuotientSpur(path_k3c2_i3d1, distrib_c2_i2d0, distrib_c2_i2d0[0]); - const path_k4c4_i1 = new LegacyQuotientSpur(cluster_k3c3, distrib_c2_i1d0, distrib_c2_i2d0[0]); + const path_k4c4_i2 = new SubstitutionQuotientSpur(path_k3c2_i3d1, distrib_c2_i2d0, distrib_c2_i2d0[0]); + const path_k4c4_i1 = new SubstitutionQuotientSpur(cluster_k3c3, distrib_c2_i1d0, distrib_c2_i2d0[0]); - const path_k4c5_i2 = new LegacyQuotientSpur(cluster_k3c3, distrib_c2_i2d0, distrib_c2_i2d0[0]); - const path_k4c5_i1 = new LegacyQuotientSpur(cluster_k3c4, distrib_c2_i1d0, distrib_c2_i2d0[0]); + const path_k4c5_i2 = new SubstitutionQuotientSpur(cluster_k3c3, distrib_c2_i2d0, distrib_c2_i2d0[0]); + const path_k4c5_i1 = new SubstitutionQuotientSpur(cluster_k3c4, distrib_c2_i1d0, distrib_c2_i2d0[0]); - const path_k4c6 = new LegacyQuotientSpur(cluster_k3c4, distrib_c2_i2d0, distrib_c2_i2d0[0]); + const path_k4c6 = new SubstitutionQuotientSpur(cluster_k3c4, distrib_c2_i2d0, distrib_c2_i2d0[0]); const cluster_k4c4 = new SearchQuotientCluster([path_k4c4_i2, path_k4c4_i1]); const cluster_k4c5 = new SearchQuotientCluster([path_k4c5_i2, path_k4c5_i1]); @@ -135,8 +135,8 @@ export const buildAlphabeticClusterFixtures = () => { { sample: { insert: 'z', deleteLeft: 0, deleteRight: 0, id: 15 }, p: 0.4 } ]; - const path_k5c6_a = new LegacyQuotientSpur(cluster_k4c4, distrib_c3_i2d0, distrib_c3_i2d0[0]); - const path_k5c6_b = new LegacyQuotientSpur(cluster_k4c5, distrib_c3_i1d0, distrib_c3_i2d0[0]); + const path_k5c6_a = new SubstitutionQuotientSpur(cluster_k4c4, distrib_c3_i2d0, distrib_c3_i2d0[0]); + const path_k5c6_b = new SubstitutionQuotientSpur(cluster_k4c5, distrib_c3_i1d0, distrib_c3_i2d0[0]); const cluster_k5c6 = new SearchQuotientCluster([path_k5c6_a, path_k5c6_b]); @@ -164,6 +164,7 @@ export const buildAlphabeticClusterFixtures = () => { distrib_c3_i2d0 } }, + root: rootPath, paths: { 1: { path_k1c1_i1d0, diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildCantLinearFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildCantLinearFixture.ts index 3de7986562c..640c6c206e6 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/buildCantLinearFixture.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildCantLinearFixture.ts @@ -8,7 +8,7 @@ * divergence occurring within the fixture. */ -import { LegacyQuotientRoot, LegacyQuotientSpur, models } from "@keymanapp/lm-worker/test-index"; +import { SearchQuotientRoot, SubstitutionQuotientSpur, models } from "@keymanapp/lm-worker/test-index"; import { jsonFixture } from "@keymanapp/common-test-resources/model-helpers.mjs"; import TrieModel = models.TrieModel; @@ -19,31 +19,31 @@ const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); * Build a linear fixture that models the word 'cant' and words close to that. */ export function buildCantLinearFixture() { - const rootPath = new LegacyQuotientRoot(testModel); + const rootPath = new SearchQuotientRoot(testModel); const distrib1 = [ { sample: {insert: 'c', deleteLeft: 0, id: 11}, p: 0.5 }, { sample: {insert: 'r', deleteLeft: 0, id: 11}, p: 0.4 }, { sample: {insert: 't', deleteLeft: 0, id: 11}, p: 0.1 } ]; - const path1 = new LegacyQuotientSpur(rootPath, distrib1, distrib1[0]); + const path1 = new SubstitutionQuotientSpur(rootPath, distrib1, distrib1[0]); const distrib2 = [ { sample: {insert: 'a', deleteLeft: 0, id: 12}, p: 0.7 }, { sample: {insert: 'e', deleteLeft: 0, id: 12}, p: 0.3 } ]; - const path2 = new LegacyQuotientSpur(path1, distrib2, distrib2[0]); + const path2 = new SubstitutionQuotientSpur(path1, distrib2, distrib2[0]); const distrib3 = [ { sample: {insert: 'n', deleteLeft: 0, id: 13}, p: 0.8 }, { sample: {insert: 'r', deleteLeft: 0, id: 13}, p: 0.2 } ]; - const path3 = new LegacyQuotientSpur(path2, distrib3, distrib3[0]); + const path3 = new SubstitutionQuotientSpur(path2, distrib3, distrib3[0]); const distrib4 = [ { sample: {insert: 't', deleteLeft: 0, id: 14}, p: 1 } ]; - const path4 = new LegacyQuotientSpur(path3, distrib4, distrib4[0]); + const path4 = new SubstitutionQuotientSpur(path3, distrib4, distrib4[0]); return { paths: [null, path1, path2, path3, path4], diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts new file mode 100644 index 00000000000..014723ae39a --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.tests.ts @@ -0,0 +1,41 @@ +/** + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-10 + * + * This file validates that the fixture corresponding to the quotient-graph-doc examples + * meets certain expectations and properties relied upon in other unit tests. + */ + +import { assert } from "chai"; + +import { buildQuotientDocFixture } from "./buildQuotientDocFixture.js"; + +describe('buildQuotientDocFixture() fixture', () => { + it('constructs paths properly', () => { + const {searchRoot, nodes} = buildQuotientDocFixture(); + + [searchRoot, nodes.sc1, nodes.sc2].forEach((n) => { + assert.equal(n.inputCount, 0); + }); + [nodes.k1c0, nodes.k1c1, nodes.k1c2, nodes.k1c3].forEach((n) => { + assert.equal(n.inputCount, 1); + }); + [nodes.k2c0, nodes.k2c1, nodes.k2c2, nodes.k2c3].forEach((n) => { + assert.equal(n.inputCount, 2); + }); + + [searchRoot, nodes.k1c0, nodes.k2c0].forEach((n) => { + assert.equal(n.codepointLength, 0); + }); + [nodes.sc1, nodes.k1c1, nodes.k2c1].forEach((n) => { + assert.equal(n.codepointLength, 1); + }); + [nodes.sc2, nodes.k1c2, nodes.k2c2].forEach((n) => { + assert.equal(n.codepointLength, 2); + }); + [nodes.k1c3, nodes.k2c3].forEach((n) => { + assert.equal(n.codepointLength, 3); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts new file mode 100644 index 00000000000..d9623c27dc7 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/buildQuotientDocFixture.ts @@ -0,0 +1,248 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-09 + * + * This file defines a unit-text fixture designed for testing + * the internal mechanisms of a search quotient graph built from + * quotient-spurs specialized for each of the three main edit-distance + * operation types. + */ + +import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { + DeletionQuotientSpur, + generateSubsetId, + InsertionQuotientSpur, + models, + SearchQuotientCluster, + SearchQuotientRoot, + SubstitutionQuotientSpur +} from '@keymanapp/lm-worker/test-index'; + +import Distribution = LexicalModelTypes.Distribution; +import Transform = LexicalModelTypes.Transform; +import TrieModel = models.TrieModel; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + + +/** + * Builds a text fixture matching the [final quotient-graph example]( + * ../../../../../../../engine/predictive-text/worker-thread/docs/correction-search-graph.md) + * documenting the internal SearchQuotientNode design. + * @returns + */ +export function buildQuotientDocFixture() { + const searchRoot = new SearchQuotientRoot(testModel); + let idSeed = 0; + + const key1Id = idSeed++; + const abDistrib: Distribution = [ + { sample: { insert: 'a', deleteLeft: 0, id: key1Id }, p: .45 }, + { sample: { insert: 'b', deleteLeft: 0, id: key1Id }, p: .35 } + ]; + + const cdDistrib: Distribution = [ + { sample: { insert: 'cd', deleteLeft: 0, id: key1Id }, p: .2 } + ]; + + const sc1 = new InsertionQuotientSpur(searchRoot); + const sc2 = new InsertionQuotientSpur(sc1); + + // K1C0 + const k1c0 = new DeletionQuotientSpur(searchRoot, abDistrib.concat(cdDistrib), { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + + // K1C1 + const k1c1_del = new DeletionQuotientSpur(sc1, abDistrib.concat(cdDistrib), { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c1_ab = new SubstitutionQuotientSpur(searchRoot, abDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c1_ins = new InsertionQuotientSpur(k1c0); + const k1c1 = new SearchQuotientCluster([k1c1_del, k1c1_ab, k1c1_ins]); + + const k1c2_del = new DeletionQuotientSpur(sc2, abDistrib.concat(cdDistrib), { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c2_ab = new SubstitutionQuotientSpur(sc1, abDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c2_cd = new SubstitutionQuotientSpur(searchRoot, cdDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c2_ins = new InsertionQuotientSpur(k1c1); + const k1c2 = new SearchQuotientCluster([k1c2_del, k1c2_ab, k1c2_cd, k1c2_ins]); + + const k1c3_ab = new SubstitutionQuotientSpur(sc2, abDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c3_cd = new SubstitutionQuotientSpur(sc1, cdDistrib, { + segment: { + transitionId: key1Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: abDistrib[0].p + }); + const k1c3_ins = new InsertionQuotientSpur(k1c2); + const k1c3 = new SearchQuotientCluster([k1c3_ab, k1c3_cd, k1c3_ins]); + + // Onto keystroke 2. + + const key2Id = idSeed++; + const efDistrib: Distribution = [ + { sample: { insert: 'e', deleteLeft: 0, id: key2Id }, p: .4 }, + { sample: { insert: 'f', deleteLeft: 0, id: key2Id }, p: .3 } + ]; + + const ghDistrib: Distribution = [ + { sample: { insert: 'gh', deleteLeft: 0, id: key2Id }, p: .3 } + ]; + + const k2c0 = new DeletionQuotientSpur(k1c0, efDistrib.concat(ghDistrib), { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + + const k2c1_del = new DeletionQuotientSpur(k1c1, efDistrib.concat(ghDistrib), { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c1_ef = new SubstitutionQuotientSpur(k1c0, efDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c1_ins = new InsertionQuotientSpur(k2c0); + const k2c1 = new SearchQuotientCluster([k2c1_del, k2c1_ef, k2c1_ins]); + + const k2c2_del = new DeletionQuotientSpur(k1c2, efDistrib.concat(ghDistrib), { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c2_ef = new SubstitutionQuotientSpur(k1c1, efDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c2_gh = new SubstitutionQuotientSpur(k1c0, ghDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c2_ins = new InsertionQuotientSpur(k2c1); + const k2c2 = new SearchQuotientCluster([k2c2_del, k2c2_ef, k2c2_gh, k2c2_ins]); + + const k2c3_del = new DeletionQuotientSpur(k1c3, efDistrib.concat(ghDistrib), { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c3_ef = new SubstitutionQuotientSpur(k1c2, efDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c3_gh = new SubstitutionQuotientSpur(k1c1, ghDistrib, { + segment: { + transitionId: key2Id, + start: 0 + }, + // Deletions always get their own unique subset ID. + subsetId: generateSubsetId(), + bestProbFromSet: efDistrib[0].p + }); + const k2c3_ins = new InsertionQuotientSpur(k2c2); + const k2c3 = new SearchQuotientCluster([k2c3_del, k2c3_ef, k2c3_gh, k2c3_ins]); + + return { + searchRoot, + spurs: {sc1, sc2, k1c1_ab, k1c2_ab, k1c2_cd, k1c2_ins, k1c3_ab, k1c3_cd, k1c3_ins, k2c1_del, k2c2_del, k2c2_ef, k2c3_ef, k2c3_gh, k2c3_ins}, + nodes: {sc1, sc2, k1c0, k1c1, k1c2, k1c3, k2c0, k2c1, k2c2, k2c3} + }; +} diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts index 8ea47e9029f..d3df0742d50 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.tests.ts @@ -9,9 +9,13 @@ import { assert } from 'chai'; +import { DeletionQuotientSpur, InsertionQuotientSpur } from '@keymanapp/lm-worker/test-index'; + import { constituentPaths } from "./constituentPaths.js"; +import { toSpurTypeSequence } from './toSpurTypeSequence.js'; import { buildCantLinearFixture } from './buildCantLinearFixture.js'; import { buildAlphabeticClusterFixtures } from './buildAlphabeticClusteredFixture.js'; +import { buildQuotientDocFixture } from './buildQuotientDocFixture.js'; describe('constituentPaths', () => { it('includes a single entry array when all parents are SearchQuotientSpurs', () => { @@ -42,4 +46,50 @@ describe('constituentPaths', () => { return p; })); }); + + describe('for the final quotient-graph doc example', () => { + it('handles insertion-only quotient-graph paths', () => { + const { sc2 } = buildQuotientDocFixture().nodes; + + const sc2Constituents = constituentPaths(sc2); + assert.equal(sc2Constituents.length, 1); + sc2Constituents.forEach(s => s.forEach(p => assert.isTrue(p instanceof InsertionQuotientSpur))); + }); + + it('handles deletion-only quotient-graph paths', () => { + const { k2c0 } = buildQuotientDocFixture().nodes; + + const k2c0Constituents = constituentPaths(k2c0); + assert.equal(k2c0Constituents.length, 1); + k2c0Constituents.forEach(s => s.forEach(p => assert.isTrue(p instanceof DeletionQuotientSpur))); + }); + + it('does not emit sequences with inserts immediately following deletes', () => { + const { k2c3 } = buildQuotientDocFixture().nodes; + + const k2c3Constituents = constituentPaths(k2c3); + + const shouldNotOccur = k2c3Constituents.find((seq) => { + const typeSeq = toSpurTypeSequence(seq); + return typeSeq.find((type, index) => { + return type == 'delete' && typeSeq[index+1] == 'insert'; + }); + }); + assert.isNotOk(shouldNotOccur); + }); + + it('does emit sequences with deletes immediately following inserts', () => { + const { k2c3 } = buildQuotientDocFixture().nodes; + + const k2c3Constituents = constituentPaths(k2c3); + + const shouldOccur = k2c3Constituents.find((seq) => { + const typeSeq = toSpurTypeSequence(seq); + return typeSeq.find((type, index) => { + return type == 'insert' && typeSeq[index+1] == 'delete'; + }); + }); + assert.isOk(shouldOccur); + }); + }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts index 9c01297f0ce..38325078719 100644 --- a/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/constituentPaths.ts @@ -8,6 +8,8 @@ */ import { + DeletionQuotientSpur, + InsertionQuotientSpur, SearchQuotientCluster, SearchQuotientNode, SearchQuotientRoot, @@ -29,6 +31,28 @@ export function constituentPaths(node: SearchQuotientNode): SearchQuotientSpur[] const parentPaths = constituentPaths(node.parents[0]); let pathsToExtend = parentPaths; + if(node instanceof InsertionQuotientSpur) { + pathsToExtend = pathsToExtend.filter(s => { + const tail = s[s.length - 1]; + + // Deletion nodes and modules should always be ordered after those for + // insertion in order to avoid duplicating search paths. (Insertions may + // stick to the right of a root, while deletions always process inputs; insertions + // may thus precede deletions.) + // + // Also, internally, insertion edges are not built after deletion (or empty) edges. + if(tail instanceof DeletionQuotientSpur) { + return false; + } else if(tail.insertLength == 0 && tail.leftDeleteLength == 0) { + // Insertions should also not appear after empty nodes; there's no net + // difference between inserting before and inserting after. + return false; + } + + return true; + }); + } + if(parentPaths.length > 0) { return pathsToExtend.map(p => { p.push(node); diff --git a/web/src/test/auto/headless/engine/predictive-text/helpers/toSpurTypeSequence.ts b/web/src/test/auto/headless/engine/predictive-text/helpers/toSpurTypeSequence.ts new file mode 100644 index 00000000000..74c9ea182cb --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/helpers/toSpurTypeSequence.ts @@ -0,0 +1,20 @@ +import { + DeletionQuotientSpur, + InsertionQuotientSpur, + SearchQuotientNode, + SubstitutionQuotientSpur +} from "@keymanapp/lm-worker/test-index"; + +export function toSpurTypeSequence(spurs: SearchQuotientNode[]): ('insert' | 'delete' | 'substitute' | 'legacy')[] { + return spurs.map(s => { + if(s instanceof InsertionQuotientSpur) { + return 'insert'; + } else if(s instanceof DeletionQuotientSpur) { + return 'delete'; + } else if(s instanceof SubstitutionQuotientSpur) { + return 'substitute'; + } else { + return 'legacy'; + } + }) +} \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts index 3bc636c4128..0aa4f9551ed 100644 --- a/web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts @@ -175,7 +175,7 @@ describe('Tokenization functions', function() { }); it('properly handles empty-context cases', function() { - // Wordbreaking on a empty space => no word. + // Wordbreaking on a empty space => no word, but empty initial token. let context = { left: '', startOfBuffer: true, right: '', endOfBuffer: true @@ -184,7 +184,7 @@ describe('Tokenization functions', function() { let tokenization = models.tokenize(wordBreakers.default, context); let expectedResult: models.Tokenization = { - left: [], + left: [{text: '', isWhitespace: false}], right: [], caretSplitsToken: false }; @@ -193,11 +193,11 @@ describe('Tokenization functions', function() { }); it('properly handles null context cases', function() { - // Wordbreaking on a empty space => no word. + // Wordbreaking on a empty space => no word, but empty initial token. let tokenization = models.tokenize(wordBreakers.default, null); let expectedResult: models.Tokenization = { - left: [], + left: [{text: '', isWhitespace: false}], right: [], caretSplitsToken: false }; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts index d1346e82148..ef8be703cff 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts @@ -36,7 +36,7 @@ describe('ContextState', () => { assert.equal(state.context, context); assert.equal(state.model, plainModel); - assert.isOk(state.tokenization); + assert.isOk(state.displayTokenization); assert.isUndefined(state.isManuallyApplied); assert.isNotOk(state.suggestions); assert.isNotOk(state.appliedSuggestionId); @@ -46,36 +46,36 @@ describe('ContextState', () => { it('creates one empty token for an empty context', () => { let context = { left: '', right: '', startOfBuffer: true, endOfBuffer: true }; let state = new ContextState(context, plainModel); - assert.isOk(state.tokenization); - assert.equal(state.tokenization.tokens.length, 1); - assert.equal(state.tokenization.tail.exampleInput, ''); + assert.isOk(state.displayTokenization); + assert.equal(state.displayTokenization.tokens.length, 1); + assert.equal(state.displayTokenization.tail.exampleInput, ''); }); it('creates tokens for initial text (without ending whitespace)', () => { let context = { left: 'the quick brown fox', right: '', startOfBuffer: true, endOfBuffer: true }; let state = new ContextState(context, plainModel); - assert.isOk(state.tokenization); - assert.equal(state.tokenization.tokens.length, 7); - assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']); + assert.isOk(state.displayTokenization); + assert.equal(state.displayTokenization.tokens.length, 7); + assert.deepEqual(state.displayTokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']); let context2 = { left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; let state2 = new ContextState(context2, plainModel); - assert.deepEqual(state2.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(state2.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); }); it('creates tokens for initial text (with extra empty token for ending whitespace)', () => { let context = { left: 'the quick brown fox ', right: '', startOfBuffer: true, endOfBuffer: true }; let state = new ContextState(context, plainModel); - assert.isOk(state.tokenization); - assert.equal(state.tokenization.tokens.length, 9); - assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']); + assert.isOk(state.displayTokenization); + assert.equal(state.displayTokenization.tokens.length, 9); + assert.deepEqual(state.displayTokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']); let context2 = { left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; let state2 = new ContextState(context2, plainModel); - assert.deepEqual(state2.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(state2.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); }); }); @@ -97,7 +97,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); }); it("properly matches and aligns when no context changes occur (after whitespace)", function() { @@ -116,7 +116,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); }); it("properly matches and aligns when lead token is removed (end of word)", function() { @@ -135,7 +135,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // if(!newContextMatch.final.tokenization.alignment.canAlign) { // assert.fail("context alignment failed"); @@ -160,7 +160,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // if(!newContextMatch.final.tokenization.alignment.canAlign) { // assert.fail("context alignment failed"); @@ -185,7 +185,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // if(!newContextMatch.final.tokenization.alignment.canAlign) { // assert.fail("context alignment failed"); @@ -207,7 +207,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // if(!newContextMatch.final.tokenization.alignment.canAlign) { // assert.fail("context alignment failed"); @@ -230,18 +230,16 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve the added whitespace when predicting a token that follows after it. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // The 'wordbreak' transform let state = newContextMatch?.final; // space transform - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); // empty transform - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1); - assert.isTrue(state.tokenization.tail.searchModule instanceof SearchQuotientSpur); - assert.deepEqual((state.tokenization.tail.searchModule as SearchQuotientSpur).lastInput, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1); + assert.isTrue(state.displayTokenization.tail.searchModule instanceof SearchQuotientSpur); + assert.deepEqual((state.displayTokenization.tail.searchModule as SearchQuotientSpur).lastInput, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); }); it("properly matches and aligns when whitespace before final empty token is extended", function() { @@ -257,19 +255,17 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve the added whitespace when predicting a token that follows after it. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // The 'wordbreak' transform let state = newContextMatch?.final; // Two whitespaces, one of which is new! - const preTail = state.tokenization.tokens[state.tokenization.tokens.length - 2]; + const preTail = state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2]; assert.equal(preTail.searchModule.inputCount, 2); assert.deepEqual((preTail.searchModule.parents[0] as SearchQuotientSpur).lastInput, [{sample: transform, p: 1}]); - assert.equal(state.tokenization.tail.searchModule.inputCount, 1); - assert.isTrue(state.tokenization.tail.searchModule instanceof SearchQuotientSpur); - assert.deepEqual((state.tokenization.tail.searchModule as SearchQuotientSpur).lastInput, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); + assert.equal(state.displayTokenization.tail.searchModule.inputCount, 1); + assert.isTrue(state.displayTokenization.tail.searchModule instanceof SearchQuotientSpur); + assert.deepEqual((state.displayTokenization.tail.searchModule as SearchQuotientSpur).lastInput, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); }); it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() { @@ -285,7 +281,7 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isOk(newContextMatch?.final); - assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.deepEqual(newContextMatch?.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); }); it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { @@ -301,13 +297,12 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: '', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // The 'wordbreak' transform let state = newContextMatch.final; - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1); }) // Needs improved context-state management (due to 2x tokens) @@ -327,15 +322,13 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve the added whitespace when predicting a token that follows after it. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // The 'wordbreak' transform let state = newContextMatch.final; - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); assert.equal( - state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1 + state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1 ); // if(!newContextMatch.final.tokenization.alignment.canAlign) { @@ -358,15 +351,13 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, [{sample: transform, p: 1}]); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve all text preceding the new token when applying a suggestion. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: 'd ', deleteLeft: 0}); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // The 'wordbreak' transform let state = newContextMatch.final; - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); assert.equal( - state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1 + state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1 ); }); @@ -383,14 +374,12 @@ describe('ContextState', () => { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = baseState.analyzeTransition(existingContext, [{sample: transform, p: 1}]); assert.isNotNull(newContextMatch?.final); - assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - // We want to preserve all text preceding the new token when applying a suggestion. - assert.deepEqual(newContextMatch.final.tokenization.taillessTrueKeystroke, { insert: 'tor ', deleteLeft: 0 }); + assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens); // The 'wordbreak' transform let state = newContextMatch.final; - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchModule.inputCount, 1); - assert.equal(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 2].searchModule.inputCount, 1); + assert.equal(state.displayTokenization.tokens[state.displayTokenization.tokens.length - 1].searchModule.inputCount, 1); }); it('handles case where tail token is split into three rather than two', function() { @@ -416,7 +405,7 @@ describe('ContextState', () => { let problemContextMatch = baseState.analyzeTransition({left: "text'", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}]); assert.isNotNull(problemContextMatch); - assert.deepEqual(problemContextMatch.final.tokenization.exampleInput, ['text', '\'', '"']); + assert.deepEqual(problemContextMatch.final.displayTokenization.exampleInput, ['text', '\'', '"']); }); }); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts index b7ff05135b6..628ada95de7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts @@ -15,7 +15,7 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs' import { LexicalModelTypes } from '@keymanapp/common-types'; import { KMWString } from 'keyman/common/web-utils'; -import { ContextToken, ExecutionTimer, generateSubsetId, getBestMatches, InputSegment, LegacyQuotientRoot, models, SearchQuotientSpur } from '@keymanapp/lm-worker/test-index'; +import { ContextToken, ExecutionTimer, generateSubsetId, getBestMatches, InputSegment, LegacyQuotientRoot, LegacyQuotientSpur, models, SearchQuotientSpur } from '@keymanapp/lm-worker/test-index'; import { quotientPathHasInputs } from "../../helpers/quotientPathHasInputs.js"; @@ -146,36 +146,45 @@ describe('ContextToken', function() { const srcTransform = { insert: "can't", deleteLeft: 0, deleteRight: 0, id: 1 }; const srcSubsetId = generateSubsetId(); - const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); - const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); - const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); - token1.addInput({ - segment: { - transitionId: srcTransform.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetId - }, [{sample: {insert: 'can', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); - - token2.addInput({ - segment: { - transitionId: srcTransform.id, - start: 3 - }, - bestProbFromSet: 1, - subsetId: srcSubsetId - }, [{sample: {insert: "'", deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); - - token3.addInput({ - segment: { - transitionId: srcTransform.id, - start: 4 - }, - bestProbFromSet: 1, - subsetId: srcSubsetId - }, [{sample: {insert: 't', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: {insert: 'can', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}], { + segment: { + transitionId: srcTransform.id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetId + } + )); + + token2 = new ContextToken(new LegacyQuotientSpur( + token2.searchModule, + [{sample: {insert: "'", deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}], { + segment: { + transitionId: srcTransform.id, + start: 3 + }, + bestProbFromSet: 1, + subsetId: srcSubsetId + } + )); + + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: {insert: 't', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}], { + segment: { + transitionId: srcTransform.id, + start: 4 + }, + bestProbFromSet: 1, + subsetId: srcSubsetId + } + )); const merged = ContextToken.merge([token1, token2, token3]); assert.equal(merged.exampleInput, "can't"); @@ -204,67 +213,85 @@ describe('ContextToken', function() { ]; // apples - const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); // and - const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); // sour - const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); // grapes - const token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); - const tokensToMerge = [token1, token2, token3, token4] + let token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); - token1.addInput({ - segment: { - transitionId: srcTransforms[0].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[0] - }, [{sample: srcTransforms[0], p: 1}]); - token1.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - - token2.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 1 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: "and", deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - - token3.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 4 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - token3.addInput({ - segment: { - transitionId: srcTransforms[2].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[2] - }, [{sample: srcTransforms[2], p: 1}]); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: srcTransforms[0], p: 1}], { + segment: { + transitionId: srcTransforms[0].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[0] + } + )); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); - token4.addInput({ - segment: { - transitionId: srcTransforms[3].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[3] - }, [{sample: srcTransforms[3], p: 1}]); + token2 = new ContextToken(new LegacyQuotientSpur( + token2.searchModule, + [{sample: {insert: "and", deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 1 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 4 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: srcTransforms[2], p: 1}], { + segment: { + transitionId: srcTransforms[2].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[2] + } + )); + + token4 = new ContextToken(new LegacyQuotientSpur( + token4.searchModule, + [{sample: srcTransforms[3], p: 1}], { + segment: { + transitionId: srcTransforms[3].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[3] + } + )); + + const tokensToMerge = [token1, token2, token3, token4]; const merged = ContextToken.merge(tokensToMerge); assert.equal(merged.exampleInput, "applesandsourgrapes"); assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({ @@ -293,68 +320,86 @@ describe('ContextToken', function() { generateSubsetId() ]; - // apples - const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); + // apples + let token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); // and - const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); // sour - const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); + let token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); // grapes - const token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); - const tokensToMerge = [token1, token2, token3, token4] + let token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); - token1.addInput({ - segment: { - transitionId: srcTransforms[0].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[0] - }, [{sample: srcTransforms[0], p: 1}]); - token1.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - - token2.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 1 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: toMathematicalSMP("and"), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - - token3.addInput({ - segment: { - transitionId: srcTransforms[1].id, - start: 4 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[1] - }, [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); - token3.addInput({ - segment: { - transitionId: srcTransforms[2].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[2] - }, [{sample: srcTransforms[2], p: 1}]); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: srcTransforms[0], p: 1}], { + segment: { + transitionId: srcTransforms[0].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[0] + } + )); + token1 = new ContextToken(new LegacyQuotientSpur( + token1.searchModule, + [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); - token4.addInput({ - segment: { - transitionId: srcTransforms[3].id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: srcSubsetIds[3] - }, [{sample: srcTransforms[3], p: 1}]); + token2 = new ContextToken(new LegacyQuotientSpur( + token2.searchModule, + [{sample: {insert: toMathematicalSMP("and"), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 1 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); + + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}], { + segment: { + transitionId: srcTransforms[1].id, + start: 4 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[1] + } + )); + token3 = new ContextToken(new LegacyQuotientSpur( + token3.searchModule, + [{sample: srcTransforms[2], p: 1}], { + segment: { + transitionId: srcTransforms[2].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[2] + } + )); + + token4 = new ContextToken(new LegacyQuotientSpur( + token4.searchModule, + [{sample: srcTransforms[3], p: 1}], { + segment: { + transitionId: srcTransforms[3].id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: srcSubsetIds[3] + } + )); + const tokensToMerge = [token1, token2, token3, token4]; const merged = ContextToken.merge(tokensToMerge); assert.equal(merged.exampleInput, toMathematicalSMP("applesandsourgrapes")); assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({ @@ -390,15 +435,18 @@ describe('ContextToken', function() { ] ] - const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); + let tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({ - segment: { - transitionId: keystrokeDistributions[i][0].sample.id, - start: 0 - }, bestProbFromSet: .75, - subsetId: generateSubsetId() - }, keystrokeDistributions[i]); + tokenToSplit = new ContextToken(new LegacyQuotientSpur( + tokenToSplit.searchModule, + keystrokeDistributions[i], { + segment: { + transitionId: keystrokeDistributions[i][0].sample.id, + start: 0 + }, bestProbFromSet: .75, + subsetId: generateSubsetId() + } + )); }; assert.equal(tokenToSplit.sourceRangeKey, 'T11+T12+T13+T14'); @@ -433,16 +481,19 @@ describe('ContextToken', function() { const splitTextArray = ['big', 'large', 'transform']; const subsetId = generateSubsetId(); - const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); + let tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({ - segment: { - transitionId: keystrokeDistributions[i][0].sample.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId - }, keystrokeDistributions[i]); + tokenToSplit = new ContextToken(new LegacyQuotientSpur( + tokenToSplit.searchModule, + keystrokeDistributions[i], { + segment: { + transitionId: keystrokeDistributions[i][0].sample.id, + start: 0 + }, + bestProbFromSet: 1, + subsetId + } + )); }; assert.equal(tokenToSplit.sourceRangeKey, `T${keystrokeDistributions[0][0].sample.id}`); @@ -504,16 +555,19 @@ describe('ContextToken', function() { generateSubsetId() ]; - const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); + let tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({ - segment: { - transitionId: keystrokeDistributions[i][0].sample.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: subsetIds[i] - }, keystrokeDistributions[i]); + tokenToSplit = new ContextToken(new LegacyQuotientSpur( + tokenToSplit.searchModule, + keystrokeDistributions[i], { + segment: { + transitionId: keystrokeDistributions[i][0].sample.id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: subsetIds[i] + } + )); }; assert.equal(tokenToSplit.exampleInput, 'largelongtransforms'); @@ -631,16 +685,19 @@ describe('ContextToken', function() { generateSubsetId() ]; - const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); + let tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({ - segment: { - transitionId: keystrokeDistributions[i][0].sample.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: subsetIds[i] - }, keystrokeDistributions[i]); + tokenToSplit = new ContextToken(new LegacyQuotientSpur( + tokenToSplit.searchModule, + keystrokeDistributions[i], { + segment: { + transitionId: keystrokeDistributions[i][0].sample.id, + start: 0 + }, + bestProbFromSet: 1, + subsetId: subsetIds[i] + } + )); }; assert.equal(tokenToSplit.exampleInput, toMathematicalSMP('largelongtransforms')); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 992d829997e..e858b76e66e 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -26,10 +26,10 @@ import { ExtendedEditOperation, generateSubsetId, models, - TransitionEdge, SearchQuotientSpur, traceInsertEdits, - determineTaillessTrueKeystroke + LegacyQuotientSpur, + TransitionEdge } from '@keymanapp/lm-worker/test-index'; import Transform = LexicalModelTypes.Transform; @@ -51,13 +51,15 @@ function toTransitionToken(text: string, transitionId?: number) { let isWhitespace = text == ' '; let token = ContextToken.fromRawText(plainModel, ''); const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed }; - token.addInput({ + token = new ContextToken(new LegacyQuotientSpur( + token.searchModule, + [ { sample: textAsTransform, p: 1 } ], { segment: { transitionId: textAsTransform.id, start: 0 }, bestProbFromSet: 1, subsetId: generateSubsetId() - }, [ { sample: textAsTransform, p: 1 } ]); + })); token.isWhitespace = isWhitespace; return token; } @@ -98,7 +100,6 @@ describe('ContextTokenization', function() { let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text)))); assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); - assert.isNotOk(tokenization.transitionEdits); assert.equal(tokenization.tail.exampleInput, 'day'); assert.isFalse(tokenization.tail.isWhitespace); }); @@ -106,36 +107,11 @@ describe('ContextTokenization', function() { it("constructs from a token array + alignment data", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const tokens = rawTextTokens.map((text => toTransitionToken(text))); - const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; - // We _could_ flesh this out a bit more... but it's not really needed for this test. - const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec); - let transitionEdits: TransitionEdge = { - alignment: { - merges: [], - splits: [], - unmappedEdits: [], - edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)}, - removedTokenCount: 0 - }, - inputs: [{sample: (() => { - const map = new Map(); - map.set(0, emptyTransform); - return map; - })(), p: 1}], - inputSubsetId: generateSubsetId() - }; - - let tokenization = new ContextTokenization(tokens, transitionEdits, null /* dummy val */); + let tokenization = new ContextTokenization(tokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); - assert.isOk(tokenization.transitionEdits); - assert.deepEqual(tokenization.transitionEdits, { - addedNewTokens: false, - removedOldTokens: false, - editedTokenCount: 1 - }); assert.equal(tokenization.tail.exampleInput, 'day'); assert.isFalse(tokenization.tail.isWhitespace); }); @@ -163,7 +139,7 @@ describe('ContextTokenization', function() { inputSubsetId: generateSubsetId() }; - let baseTokenization = new ContextTokenization(tokens, transitionEdits, null /* dummy val */); + let baseTokenization = new ContextTokenization(tokens, transitionEdits); let cloned = new ContextTokenization(baseTokenization); assert.sameOrderedMembers( @@ -2539,111 +2515,4 @@ describe('ContextTokenization', function() { assert.deepEqual(results, expectedMap); }); }); - - describe('determineTaillessTrueKeystroke', () => { - it('handles simple tail-token extensions correctly', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(0, { insert: '', deleteLeft: 0 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.isNotOk(preservedTransform); - }); - - it('handles simple tail-terminating whitespace inputs correctly', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(1, { insert: ' ', deleteLeft: 0 }); - tokenizedInput.set(2, { insert: '', deleteLeft: 0 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.deepEqual(preservedTransform, { - insert: ' ', - deleteLeft: 0 - }); - }); - - it('handles simple tail-token char deletions correctly', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(0, { insert: '', deleteLeft: 1 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.isNotOk(preservedTransform); - }); - - it('handles tail whitespace-token deletions correctly', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(-1, { insert: '', deleteLeft: 1 }); - tokenizedInput.set(0, { insert: '', deleteLeft: 0 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.isNotOk(preservedTransform); - }); - - it('handles multi-token insert with small delete correctly', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(0, { insert: 'a', deleteLeft: 1 }); - tokenizedInput.set(1, { insert: ' ', deleteLeft: 0 }); - tokenizedInput.set(2, { insert: 'bc', deleteLeft: 0 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.deepEqual(preservedTransform, { - insert: 'a ', - deleteLeft: 1 - }); - }); - - it('handles multi-token delete with small insert correctly', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(-2, { insert: 'a', deleteLeft: 1 }); - tokenizedInput.set(-1, { insert: '', deleteLeft: 1 }); - tokenizedInput.set(0, { insert: '', deleteLeft: 1 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.isNotOk(preservedTransform); - }); - - it('handles multi-token insertion/deletion input correctly (1)', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(-2, { insert: 'a', deleteLeft: 1 }); - tokenizedInput.set(-1, { insert: ' ', deleteLeft: 1 }); - tokenizedInput.set(0, { insert: 'b', deleteLeft: 1 }); - tokenizedInput.set(1, { insert: ' ', deleteLeft: 0 }); - tokenizedInput.set(2, { insert: 'c', deleteLeft: 0 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.deepEqual(preservedTransform, { - insert: 'a b ', - deleteLeft: 3 - }); - }); - - it('handles multi-token insertion/deletion input correctly (2)', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(-4, { insert: 'a', deleteLeft: 1 }); - tokenizedInput.set(-3, { insert: ' ', deleteLeft: 1 }); - tokenizedInput.set(-2, { insert: 'b', deleteLeft: 1 }); - tokenizedInput.set(-1, { insert: ' ', deleteLeft: 1 }); - tokenizedInput.set(0, { insert: '', deleteLeft: 0 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.deepEqual(preservedTransform, { - insert: 'a b ', - deleteLeft: 4 - }); - }); - - it('handles multi-token insertion/deletion input correctly (3)', () => { - const tokenizedInput: Map = new Map(); - tokenizedInput.set(-4, { insert: 'a', deleteLeft: 1 }); - tokenizedInput.set(-3, { insert: ' ', deleteLeft: 1 }); - tokenizedInput.set(-2, { insert: 'b', deleteLeft: 1 }); - tokenizedInput.set(-1, { insert: '', deleteLeft: 1 }); - tokenizedInput.set(0, { insert: '', deleteLeft: 0 }); - - const preservedTransform = determineTaillessTrueKeystroke(tokenizedInput); - assert.deepEqual(preservedTransform, { - insert: 'a ', - deleteLeft: 2 - }); - }); - }); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts index 050ab56c4a6..327613758d2 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts @@ -76,10 +76,10 @@ describe('ContextTracker', function() { assert.equal(postContextMatch.final.appliedSuggestionId, baseSuggestion.id); // Penultimate token corresponds to whitespace, which does not have a 'raw' representation. - assert.equal(postContextMatch.final.tokenization.tokens[postContextMatch.final.tokenization.tokens.length - 2].exampleInput, ' '); + assert.equal(postContextMatch.final.displayTokenization.tokens[postContextMatch.final.displayTokenization.tokens.length - 2].exampleInput, ' '); // Final token is empty (follows a wordbreak) - assert.equal(postContextMatch.final.tokenization.tail.exampleInput, ''); + assert.equal(postContextMatch.final.displayTokenization.tail.exampleInput, ''); }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts index 2477a80f18e..2ee2c1cf50d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts @@ -25,9 +25,9 @@ var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), function assertClonedStateMatch(a: ContextState, b: ContextState) { assert.notEqual(a, b); - assert.notEqual(a.tokenization, b.tokenization); - assert.notSameOrderedMembers(a.tokenization.tokens, b.tokenization.tokens); - assert.sameOrderedMembers(a.tokenization.exampleInput, b.tokenization.exampleInput); + assert.notEqual(a.displayTokenization, b.displayTokenization); + assert.notSameOrderedMembers(a.displayTokenization.tokens, b.displayTokenization.tokens); + assert.sameOrderedMembers(a.displayTokenization.exampleInput, b.displayTokenization.exampleInput); assert.deepEqual(a.suggestions, b.suggestions); } @@ -50,7 +50,7 @@ describe('ContextTransition', () => { const transition = new ContextTransition(baseState, 1); assert.sameOrderedMembers( - transition.base.tokenization.tokens.map((t) => t.exampleInput), + transition.base.displayTokenization.tokens.map((t) => t.exampleInput), ['hello', ' ', 'world', ' ', ''] ); assert.equal(transition.transitionId, 1); @@ -67,7 +67,7 @@ describe('ContextTransition', () => { const transition = new ContextTransition(baseState, 1); assert.sameOrderedMembers( - transition.base.tokenization.tokens.map((t) => t.exampleInput), + transition.base.displayTokenization.tokens.map((t) => t.exampleInput), ['hello', ' ', 'world', ' ', ''] ); @@ -141,17 +141,17 @@ describe('ContextTransition', () => { assert.notEqual(appliedTransition.base, transition); assert.isOk(appliedTransition.appended); assert.notEqual(appliedTransition.appended, transition); - assert.sameOrderedMembers(appliedTransition.base.final.tokenization.exampleInput, [ + assert.sameOrderedMembers(appliedTransition.base.final.displayTokenization.exampleInput, [ 'hello', ' ', 'world' ]); - assert.sameOrderedMembers(appliedTransition.appended.final.tokenization.exampleInput, [ + assert.sameOrderedMembers(appliedTransition.appended.final.displayTokenization.exampleInput, [ 'hello', ' ', 'world', ' ', '' ]); assert.equal(appliedTransition.base.final.appliedSuggestionId, suggestions[0].id); assert.equal(appliedTransition.appended.final.appliedSuggestionId, suggestions[0].id); // 3 long, only last token was edited. - appliedTransition.base.final.tokenization.tokens.forEach((token, index) => { + appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 2) { assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { @@ -159,7 +159,7 @@ describe('ContextTransition', () => { } }); - appliedTransition.appended.final.tokenization.tokens.forEach((token, index) => { + appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 2) { assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { @@ -223,17 +223,17 @@ describe('ContextTransition', () => { assert.notEqual(appliedTransition.base, transition); assert.isOk(appliedTransition.appended); assert.notEqual(appliedTransition.appended, transition); - assert.sameOrderedMembers(appliedTransition.base.final.tokenization.exampleInput, [ + assert.sameOrderedMembers(appliedTransition.base.final.displayTokenization.exampleInput, [ 'hello', ' ', 'world', ' ', 'the' ]); - assert.sameOrderedMembers(appliedTransition.appended.final.tokenization.exampleInput, [ + assert.sameOrderedMembers(appliedTransition.appended.final.displayTokenization.exampleInput, [ 'hello', ' ', 'world', ' ', 'the', ' ', '' ]); assert.equal(appliedTransition.base.final.appliedSuggestionId, suggestions[0].id); assert.equal(appliedTransition.appended.final.appliedSuggestionId, suggestions[0].id); // 3 long, only last token was edited. - appliedTransition.base.final.tokenization.tokens.forEach((token, index) => { + appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 4) { assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { @@ -241,7 +241,7 @@ describe('ContextTransition', () => { } }); - appliedTransition.appended.final.tokenization.tokens.forEach((token, index) => { + appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 4) { assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index b924d9422cb..44d55fde4c9 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -20,7 +20,7 @@ import { buildEdgeWindow, ContextToken, ContextTokenization, - generateSubsetId, + LegacyQuotientSpur, legacySubsetKeyer, models, precomputationSubsetKeyer, @@ -230,16 +230,11 @@ describe('precomputationSubsetKeyer', function() { [...tokenization.tokens, (() => { const token = ContextToken.fromRawText(plainModel, 'da'); // source text: 'date' - token.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ + const dist = [ {sample: {insert: 'te', deleteLeft: 0, id: 13}, p: 1} - ]); - return token; + ]; + const space = new LegacyQuotientSpur(token.searchModule, dist, dist[0]); + return new ContextToken(space); })()], { insert: 's', deleteLeft: 0, deleteRight: 0 }, false @@ -262,16 +257,11 @@ describe('precomputationSubsetKeyer', function() { [...tokenization.tokens, (() => { const token = ContextToken.fromRawText(plainModel, 'da'); // source text: 'date' - token.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ - {sample: {insert: 't', deleteLeft: 0}, p: 1} - ]); - return token; + const dist = [ + {sample: {insert: 't', deleteLeft: 0, id: 13}, p: 1} + ]; + const space = new LegacyQuotientSpur(token.searchModule, dist, dist[0]); + return new ContextToken(space); })()], { insert: 'es', deleteLeft: 0, deleteRight: 0, id: 14 }, false @@ -306,17 +296,15 @@ describe('precomputationSubsetKeyer', function() { ...buildEdgeWindow( [...tokenization.tokens, (() => { const token = ContextToken.fromRawText(plainModel, 'da'); - token.isPartial = true; // source text: 'dat' - token.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [{sample: {insert: 'ts', deleteLeft: 0, id: 13}, p: 1} - ]); - return token; + const dist = [ + {sample: {insert: 'ts', deleteLeft: 0, id: 13}, p: 1} + ]; + const space = new LegacyQuotientSpur(token.searchModule, dist, dist[0]); + let token2 = new ContextToken(space); + token2.isPartial = true; + + return token2; })()], { insert: 'e', deleteLeft: 1, deleteRight: 0, id: 14 }, false @@ -338,18 +326,15 @@ describe('precomputationSubsetKeyer', function() { ...buildEdgeWindow( [...tokenization.tokens, (() => { const token = ContextToken.fromRawText(plainModel, 'da'); - token.isPartial = true; // source text: 'dat' - token.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ + const dist = [ {sample: {insert: 't', deleteLeft: 0, id: 13}, p: 1} - ]); - return token; + ]; + const space = new LegacyQuotientSpur(token.searchModule, dist, dist[0]); + let token2 = new ContextToken(space); + token2.isPartial = true; + + return token2; })()], { insert: 'e', deleteLeft: 0, deleteRight: 0, id: 14 }, false @@ -796,27 +781,25 @@ describe('TokenizationSubsetBuilder', function() { const trueSourceTransform: Transform = { insert: 'é', deleteLeft: 1, id: 13 }; - const fourCharTailToken = new ContextToken(baseTokenization.tail); - fourCharTailToken.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ - { sample: trueSourceTransform, p: .6 } - ]); - - const fiveCharTailToken = new ContextToken(baseTokenization.tail); - fiveCharTailToken.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: 1, - subsetId: generateSubsetId() - }, [ + let fourCharTailToken = new ContextToken(baseTokenization.tail); + let fourCharTailDist = [{sample: trueSourceTransform, p: .6}]; + let fourCharTailSpace = new LegacyQuotientSpur( + fourCharTailToken.searchModule, + fourCharTailDist, + fourCharTailDist[0] + ); + fourCharTailToken = new ContextToken(fourCharTailSpace); + + let fiveCharTailToken = new ContextToken(baseTokenization.tail); + let fiveCharTailDist = [ { sample: { insert: 's', deleteLeft: 0, id: 13 }, p: .4 } - ]); + ]; + let fiveCharTailSpace = new LegacyQuotientSpur( + fiveCharTailToken.searchModule, + fiveCharTailDist, + fiveCharTailDist[0] + ); + fiveCharTailToken = new ContextToken(fiveCharTailSpace); const subsetBuilder = new TokenizationSubsetBuilder(); const fourCharTokenization = new ContextTokenization([...baseTokenization.tokens.slice(0, -1), fourCharTailToken]); @@ -845,27 +828,25 @@ describe('TokenizationSubsetBuilder', function() { const trueSourceTransform: Transform = { insert: 'é', deleteLeft: 1, id: 13 }; - const twoCharTailToken = new ContextToken(baseTokenization.tail); - twoCharTailToken.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: .6, - subsetId: generateSubsetId() - }, [ - { sample: trueSourceTransform, p: .6 } - ]); - - const threeCharTailToken = new ContextToken(baseTokenization.tail); - threeCharTailToken.addInput({ - segment: { - transitionId: 13, - start: 0 - }, bestProbFromSet: .6, - subsetId: generateSubsetId() - }, [ - { sample: { insert: 'a', deleteLeft: 0, id: 13}, p: .4 } - ]); + let twoCharTailToken = new ContextToken(baseTokenization.tail); + let twoCharTailDist = [{sample: trueSourceTransform, p: .6}]; + let twoCharTailSpace = new LegacyQuotientSpur( + twoCharTailToken.searchModule, + twoCharTailDist, + twoCharTailDist[0] + ); + twoCharTailToken = new ContextToken(twoCharTailSpace); + + let threeCharTailToken = new ContextToken(baseTokenization.tail); + let threeCharTailDist = [ + { sample: { insert: 'a', deleteLeft: 0, id: 13 }, p: .4 } + ]; + let threeCharTailSpace = new LegacyQuotientSpur( + threeCharTailToken.searchModule, + threeCharTailDist, + threeCharTailDist[0] + ); + threeCharTailToken = new ContextToken(threeCharTailSpace); const subsetBuilder = new TokenizationSubsetBuilder(); const twoCharTokenization = new ContextTokenization([...baseTokenization.tokens.slice(0, -1), twoCharTailToken]); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts index 3349c09a7bf..9044f971da5 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts @@ -187,11 +187,7 @@ function generateFixtureForTokenizationOutboundTransition ( // CURRENTLY NOT DONE: adding new or replacement tokens for text to be placed after 'quotientNodeToExtend'. - const transitionedTokenization = new ContextTokenization( - srcTokenization.tokens.slice(0, srcTokenization.tokens.length - 1 + relativeTailIndex).concat(token), - tokenizationEdge, - null - ); + const transitionedTokenization = new ContextTokenization(srcTokenization.tokens.slice(0, srcTokenization.tokens.length - 1 + relativeTailIndex).concat(token)); return { /** @@ -366,7 +362,6 @@ function assertMatchingToken(actual: ContextToken, expected: ContextToken, msg: function assertMatchingTokenization(actual: ContextTokenization, expected: ContextTokenization, msg: string) { assert.equal(actual.tokens.length, expected.tokens.length, msg); assert.deepEqual(actual.exampleInput, expected.exampleInput, msg); - assert.deepEqual(actual.transitionEdits, expected.transitionEdits, msg); for(let j=0; j < actual.tokens.length; j++) { const nestedMsg = `${msg}, token ${j}`; @@ -571,7 +566,7 @@ describe('transitionTokenizations', () => { }, p: 1 }] - const precomputedTransition = precomputeTransitions([baseState.tokenization], dist); + const precomputedTransition = precomputeTransitions(baseState.tokenizations, dist); const result = transitionTokenizations(precomputedTransition.subsets, dist); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts new file mode 100644 index 00000000000..e9020b51ec7 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/deletion-quotient-spur.tests.ts @@ -0,0 +1,210 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-06 + * + * This file defines tests for the DeletionQuotientSpur class of the + * predictive-text correction-search engine's search graph. + */ + +import { assert } from 'chai'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { + DeletionQuotientSpur, + models, + SearchQuotientRoot, + SubstitutionQuotientSpur +} from '@keymanapp/lm-worker/test-index'; + +import TrieModel = models.TrieModel; + +import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; +import { buildQuotientDocFixture } from '../../helpers/buildQuotientDocFixture.js'; + +import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + +describe('DeletionQuotientSpur', () => { + describe('constructor', () => { + it('may be extended from root path', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + + const extendedPath = new DeletionQuotientSpur(rootPath, leadEdgeDistribution, leadEdgeDistribution[0]); + + assert.equal(extendedPath.inputCount, 1); + assert.equal(extendedPath.codepointLength, 0); + assert.isNumber(extendedPath.spaceId); + assert.notEqual(extendedPath.spaceId, rootPath.spaceId); + assert.deepEqual(extendedPath.bestExample, { text: '', p: 1 } ); + + assert.deepEqual(extendedPath.parents, [rootPath]); + assert.deepEqual(extendedPath.inputs, leadEdgeDistribution); + + // Assert the root is unchanged. + assert.equal(rootPath.inputCount, 0); + + // Should (still) have codepointLength == 0 once it's defined. + assert.deepEqual(rootPath.bestExample, {text: '', p: 1}); + assert.deepEqual(rootPath.parents, []); + }); + + it('may be built from arbitrary prior SearchQuotientSpur', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + const inputClone = leadEdgeDistribution.map(e => ({...e})); + + const length1Path = new SubstitutionQuotientSpur( + rootPath, + leadEdgeDistribution, + leadEdgeDistribution[0] + ); + + const tailEdgeDistribution = [ + {sample: {insert: 'r', deleteLeft: 0, id: 17 }, p: 0.6}, + {sample: {insert: 'e', deleteLeft: 0, id: 17 }, p: 0.25}, + {sample: {insert: 'h', deleteLeft: 0, id: 17 }, p: 0.15} + ]; + + const length2Path = new DeletionQuotientSpur( + length1Path, + tailEdgeDistribution, + tailEdgeDistribution[0] + ); + + // Verify that the prior distribution remains fully unaltered. + assert.deepEqual(leadEdgeDistribution, inputClone); + + assert.equal(length2Path.inputCount, 2); + assert.equal(length2Path.codepointLength, 1); + assert.isNumber(length2Path.spaceId); + assert.notEqual(length2Path.spaceId, length1Path.spaceId); + assert.deepEqual(length2Path.bestExample, length1Path.bestExample); + assert.deepEqual(length2Path.parents, [length1Path]); + assert.deepEqual(length2Path.inputs, tailEdgeDistribution); + assert.deepEqual(length2Path.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + }, { + transitionId: tailEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + assert.equal(length1Path.inputCount, 1); + assert.equal(length1Path.codepointLength, 1); + assert.isNumber(length1Path.spaceId); + assert.notEqual(length1Path.spaceId, rootPath.spaceId); + assert.deepEqual(length1Path.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(length1Path.parents, [rootPath]); + assert.deepEqual(length1Path.inputs, leadEdgeDistribution); + }); + }); + + describe('.edgeKey', () => { + it('is different for different delete locations', () => { + const {k1c0, k2c0} = buildQuotientDocFixture().nodes; + + assert.notEqual(k2c0.edgeKey, k1c0.edgeKey); + }); + + it('is different from the parent node\'s key', () => { + const { k1c1_ab, k2c1_del, k1c2_cd, k2c2_del } = buildQuotientDocFixture().spurs; + + assert.notEqual(k2c1_del.edgeKey, k1c1_ab.edgeKey); + assert.notEqual(k2c2_del.edgeKey, k1c2_cd.edgeKey); + }); + }); + + describe('handleNextNode()', () => { + it('does not output results that directly match inputs', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const matchTargets = [ + 'can', + 'car', + 'cen', // 'cent' and 'center' are supported in this test model. + ]; + const analysis = analyzeQuotientNodeResults(deletionPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that substitute inputs', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const matchTargets = [ + // Replacement of first char + 'man', + 'far', + // Replacement of second char + 'con', // 'consider' and variants thereof are also supported. + 'cor', // 'corner' + // Replacement of third char + 'cal', // 'call' + ]; + const analysis = analyzeQuotientNodeResults(deletionPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that insert characters as needed', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const matchTargets = [ + 'can', // 'can' + (insert) 'n' (=> 'cannot') + 'car', // 'car' + (insert) 'e' + 'ran', // 'ran' + (insert) 'g' (=> 'range') + 'cann', // 'can' + (insert) 'n' (=> 'cannot') + 'care', // 'car' + (insert) 'e' + 'rang', // 'ran' + (insert) 'g' (=> 'range') + ]; + const analysis = analyzeQuotientNodeResults(deletionPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('outputs results that delete incoming keystrokes as needed', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const matchTargets = [ + // Deletes the last char. + 'ca', + 'ce', + 're' + ]; + const analysis = analyzeQuotientNodeResults(deletionPath, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.sameMembers(analysis.missing, []); + assert.isEmpty(analysis.foundWithDuplicates); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts index bf45e3b94e1..50198b4b2a0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts @@ -1,6 +1,16 @@ import { assert } from 'chai'; -import { CORRECTION_SEARCH_THRESHOLDS, CorrectionPredictionTuple, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index"; +import { CORRECTION_SEARCH_THRESHOLDS, TokenizedIntermediatePrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index"; + +function mockTokenizedPrediction(value: number) { + return { + metadata: { + probabilities: { + total: value + } + } + } as TokenizedIntermediatePrediction +} describe('correction-search: shouldStopSearchingEarly', () => { it('stops early once new corrections are less likely than currently discovered predictions', () => { @@ -12,12 +22,7 @@ describe('correction-search: shouldStopSearchingEarly', () => { assert.equal(predictionProbs.length, ModelCompositor.MAX_SUGGESTIONS, "test setup no longer valid"); // The only part for each entry we actually care about here: .totalProb. - /** @type {import('#./predict-helpers.js').CorrectionPredictionTuple[]} */ - const predictions = predictionProbs.map((entry) => { - return { - totalProb: entry - } as CorrectionPredictionTuple - }); + const predictions = predictionProbs.map((entry) => mockTokenizedPrediction(entry)); // Thresholding is performed in log-space. // 0.0501 and 0.0499 are offset on each side of 0.05, the last value in the array defined above. @@ -33,8 +38,8 @@ describe('correction-search: shouldStopSearchingEarly', () => { // // Can technically run the method with an empty array, but the actual scenario would have // at least one prediction present in the "found predictions" array. - assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, [{ totalProb: Math.exp(-1) } as CorrectionPredictionTuple])); - assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, [{ totalProb: Math.exp(-1) } as CorrectionPredictionTuple])); + assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, [mockTokenizedPrediction(Math.exp(-1))])); + assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, [mockTokenizedPrediction(Math.exp(-1))])); }); it('stops checking corrections earlier when enough predictions have been found', () => { @@ -43,11 +48,7 @@ describe('correction-search: shouldStopSearchingEarly', () => { // The only part for each entry we actually care about here: .totalProb. /** @type {import('#./predict-helpers.js').CorrectionPredictionTuple[]} */ - const predictions = predictionProbs.map((entry) => { - return { - totalProb: entry - } as CorrectionPredictionTuple - }); + const predictions = predictionProbs.map((entry) => mockTokenizedPrediction(entry)); const baseCost = 1; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts new file mode 100644 index 00000000000..074f261d349 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/insertion-quotient-spur.tests.ts @@ -0,0 +1,192 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-06 + * + * This file defines tests for the InsertionQuotientSpur class of the + * predictive-text correction-search engine's search graph. + */ + +import { assert } from 'chai'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { + DeletionQuotientSpur, + InsertionQuotientSpur, + models, + SearchQuotientRoot, + SubstitutionQuotientSpur +} from '@keymanapp/lm-worker/test-index'; + +import TrieModel = models.TrieModel; + +import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; +import { buildQuotientDocFixture } from '../../helpers/buildQuotientDocFixture.js'; + +import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + +describe('InsertionQuotientSpur', () => { + describe('constructor', () => { + it('may be extended from root path', () => { + const rootPath = new SearchQuotientRoot(testModel); + const extendedPath = new InsertionQuotientSpur(rootPath); + + assert.equal(extendedPath.inputCount, 0); + assert.equal(extendedPath.codepointLength, 1); + assert.isNumber(extendedPath.spaceId); + assert.notEqual(extendedPath.spaceId, rootPath.spaceId); + assert.equal(extendedPath.bestExample.text.length, 1); + assert.equal(extendedPath.bestExample.p, 1); + + assert.deepEqual(extendedPath.parents, [rootPath]); + assert.deepEqual(extendedPath.inputs, null); + + // Assert the root is unchanged. + assert.equal(rootPath.inputCount, 0); + + // Should (still) have codepointLength == 0 once it's defined. + assert.deepEqual(rootPath.bestExample, {text: '', p: 1}); + assert.deepEqual(rootPath.parents, []); + }); + + it('may be built from arbitrary prior SearchQuotientSpur', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + const inputClone = leadEdgeDistribution.map(e => ({...e})); + + const length1Path = new SubstitutionQuotientSpur( + rootPath, + leadEdgeDistribution, + leadEdgeDistribution[0] + ); + + const length2Path = new InsertionQuotientSpur(length1Path); + + // Verify that the prior distribution remains fully unaltered. + assert.deepEqual(leadEdgeDistribution, inputClone); + + assert.equal(length2Path.inputCount, 1); + assert.equal(length2Path.codepointLength, 2); + assert.isNumber(length2Path.spaceId); + assert.notEqual(length2Path.spaceId, length1Path.spaceId); + assert.equal(length2Path.bestExample.text.length, 2); + assert.equal(length2Path.bestExample.p, leadEdgeDistribution[0].p); + assert.deepEqual(length2Path.parents, [length1Path]); + assert.deepEqual(length2Path.inputs, null); + assert.deepEqual(length2Path.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + assert.equal(length1Path.inputCount, 1); + assert.equal(length1Path.codepointLength, 1); + assert.isNumber(length1Path.spaceId); + assert.notEqual(length1Path.spaceId, rootPath.spaceId); + assert.deepEqual(length1Path.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(length1Path.parents, [rootPath]); + assert.deepEqual(length1Path.inputs, leadEdgeDistribution); + }); + }); + + describe('.edgeKey', () => { + it('is different for different insert locations', () => { + const {sc1, sc2} = buildQuotientDocFixture().nodes; + assert.notEqual(sc1.edgeKey, sc2.edgeKey); + }); + + it('is different for an insert and the spur with the most recent processed input', () => { + const { k1c2_cd, k1c3_ins } = buildQuotientDocFixture().spurs; + assert.notEqual(k1c2_cd.edgeKey, k1c3_ins.edgeKey); + + const k1c2_cd_ins = new InsertionQuotientSpur(k1c2_cd); + assert.notEqual(k1c2_cd_ins.edgeKey, k1c2_cd.edgeKey); + }); + }); + + describe('handleNextNode()', () => { + it('outputs results that insert characters as needed', () => { + const canPath = buildCantLinearFixture().paths[3]; + const followingInsert = new InsertionQuotientSpur(canPath); + + const matchTargets = [ + 'cann', // 'can' + (insert) 'n' (=> 'cannot') + 'care', // 'car' + (insert) 'e' + 'rang', // 'ran' + (insert) 'g' (=> 'range') + ]; + const analysis = analyzeQuotientNodeResults(followingInsert, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.sameMembers(analysis.missing, []); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('outputs results extending prior inserts if insertion spur is parent', () => { + const canPath = buildCantLinearFixture().paths[3]; + const firstInsert = new InsertionQuotientSpur(canPath); + const followingInsert = new InsertionQuotientSpur(firstInsert) + + const matchTargets = [ + 'canno', // 'can' + (insert) 'n' (=> 'cannot') + 'range', // 'ran' + (insert) 'g' (=> 'range') + ]; + const analysis = analyzeQuotientNodeResults(followingInsert, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.sameMembers(analysis.missing, []); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that delete incoming keystrokes as needed', () => { + const canPath = buildCantLinearFixture().paths[3]; + const followingInsert = new InsertionQuotientSpur(canPath); + + const matchTargets = [ + // Delete only first + 'an', // (delete) 'c'/'r'/'t' + 'an', for 'and' and 'any' + 'en', // (delete) 'c'/'r'/'t' + 'en', for 'end', + // Even delete second + 'n', // model possesses words starting with just 'n' + 't', // ... and 't'. + // Delete only third + 'ca', + 'ce', + ]; + const analysis = analyzeQuotientNodeResults(followingInsert, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results when immediately following a deletion spur edit', () => { + const caPath = buildCantLinearFixture().paths[2]; + const distrib = buildCantLinearFixture().distributions[2]; // for the third entry. + const deletionPath = new DeletionQuotientSpur(caPath, distrib, distrib[0]); + + const followingInsert = new InsertionQuotientSpur(deletionPath); + + for( + let searchResult = followingInsert.handleNextNode(); + searchResult.type != 'none'; + searchResult = followingInsert.handleNextNode() + ) { + // While the insertion quotient node will forward results from the deletion version, + // it should never actually build search routes that go delete -> insert directly. + if(searchResult.type == 'complete') { + assert.notEqual(searchResult.mapping.lastEdgeType, 'insertion'); + assert.notEqual(searchResult.mapping.spaceId, followingInsert.spaceId); + } + } + // We should reach the end of the loop without once processing an actual 'insert' case. + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts index 0989ab9d64e..21bdd5026e6 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts @@ -17,13 +17,46 @@ import { models } from '@keymanapp/lm-worker/test-index'; -import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; import TrieModel = models.TrieModel; const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); +// Similar to the fixture version, but with the LegacyQuotientRoot & LegacyQuotientSpur classes. +export function buildCantLinearFixture() { + const rootPath = new LegacyQuotientRoot(testModel); + + const distrib1 = [ + { sample: {insert: 'c', deleteLeft: 0, id: 11}, p: 0.5 }, + { sample: {insert: 'r', deleteLeft: 0, id: 11}, p: 0.4 }, + { sample: {insert: 't', deleteLeft: 0, id: 11}, p: 0.1 } + ]; + const path1 = new LegacyQuotientSpur(rootPath, distrib1, distrib1[0]); + + const distrib2 = [ + { sample: {insert: 'a', deleteLeft: 0, id: 12}, p: 0.7 }, + { sample: {insert: 'e', deleteLeft: 0, id: 12}, p: 0.3 } + ]; + const path2 = new LegacyQuotientSpur(path1, distrib2, distrib2[0]); + + const distrib3 = [ + { sample: {insert: 'n', deleteLeft: 0, id: 13}, p: 0.8 }, + { sample: {insert: 'r', deleteLeft: 0, id: 13}, p: 0.2 } + ]; + const path3 = new LegacyQuotientSpur(path2, distrib3, distrib3[0]); + + const distrib4 = [ + { sample: {insert: 't', deleteLeft: 0, id: 14}, p: 1 } + ]; + const path4 = new LegacyQuotientSpur(path3, distrib4, distrib4[0]); + + return { + paths: [null, path1, path2, path3, path4], + distributions: [distrib1, distrib2, distrib3, distrib4] + }; +} + describe('LegacyQuotientSpur', () => { describe('constructor', () => { it('initializes from a lexical model', () => { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts index ee6ee088e65..9dec509ad5b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts @@ -15,14 +15,15 @@ import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { + EDIT_DISTANCE_COST_SCALE, generateSubsetId, - LegacyQuotientSpur, models, PathInputProperties, PathResult, QuotientNodeFinalizer, SearchQuotientNode, SearchQuotientRoot, + SubstitutionQuotientSpur, TokenResultMapping } from '@keymanapp/lm-worker/test-index'; @@ -76,10 +77,9 @@ function buildFixture_therefore(): SearchQuotientNode[] { }; }); - // TODO: Use SubstitutionQuotientSpur instead! let quotientNodes: SearchQuotientNode[] = [new SearchQuotientRoot(plainModel)]; for(let i=0; i < 9; i++) { - quotientNodes.push(new LegacyQuotientSpur(quotientNodes[i], distributions[i], inputSources[i])); + quotientNodes.push(new SubstitutionQuotientSpur(quotientNodes[i], distributions[i], inputSources[i])); } return quotientNodes; @@ -99,7 +99,7 @@ describe('QuotientNodeFinalizer', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p)); + assert.approximately(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p), Number.EPSILON * 1000); assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); } else { @@ -107,14 +107,16 @@ describe('QuotientNodeFinalizer', () => { } searchResult = therefo.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. do { searchResult = therefo.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + assert.equal(searchResult.type, 'none'); }); it('finds only corrections when predictions are forbidden', () => { @@ -130,6 +132,11 @@ describe('QuotientNodeFinalizer', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { assert.isAbove(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p)); + + // There are two codepoints missing that are necessary to complete a + // full word with the represented prefix. Check that the penalty is set + // appropriately, accounting for floating-point precision issues. + assert.isAtLeast(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p) + EDIT_DISTANCE_COST_SCALE * 1.99); assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); } else { @@ -137,14 +144,16 @@ describe('QuotientNodeFinalizer', () => { } searchResult = therefo.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); do { searchResult = therefo.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. + assert.equal(searchResult.type, 'none'); }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts index 152f0630c2e..a2a48a76d40 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-cluster.tests.ts @@ -118,16 +118,6 @@ const determineTargetSplitSequences = (constituentPaths: SearchQuotientSpur[][], describe('SearchQuotientCluster', () => { describe('constructor()', () => { - it('initializes from LegacySearchRoot', () => { - const path = new LegacyQuotientRoot(testModel); - const cluster = new SearchQuotientCluster([path]); - assert.equal(cluster.inputCount, 0); - assert.equal(cluster.codepointLength, 0); - assert.isNumber(cluster.spaceId); - assert.deepEqual(cluster.bestExample, {text: '', p: 1}); - assert.deepEqual(cluster.parents, [path]); - }); - it('initializes from arbitrary SearchQuotientSpur', () => { const rootPath = new LegacyQuotientRoot(testModel); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts index a8d62c3a7f2..81042fd308d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts @@ -26,8 +26,10 @@ import { } from '@keymanapp/lm-worker/test-index'; import { constituentPaths } from '../../helpers/constituentPaths.js'; +import { toSpurTypeSequence } from '../../helpers/toSpurTypeSequence.js'; import { quotientPathHasInputs } from '../../helpers/quotientPathHasInputs.js'; import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; +import { buildQuotientDocFixture } from '../../helpers/buildQuotientDocFixture.js'; import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform; @@ -1143,6 +1145,132 @@ describe('SearchQuotientSpur', () => { assert.deepEqual((head as LegacyQuotientSpur).inputSource, headTarget.inputSource); assert.deepEqual((tail as LegacyQuotientSpur).inputSource, tailTarget.inputSource); }); + + + describe('correctly handles the quotient-path doc example', () => { + it('splits properly at index 0', () => { + const fixture = buildQuotientDocFixture(); + const k2c3 = fixture.nodes.k2c3; + + const splits = k2c3.split(0); + assert.equal(splits.length, 1); + assert.equal(splits[0][0], fixture.searchRoot); + assert.isTrue(splits[0][1].isSameNode(k2c3)); + }); + + it('splits properly at index 1', () => { + const nodes = buildQuotientDocFixture().nodes; + const k2c3 = nodes.k2c3; + + const splits = k2c3.split(1); + /* + * Unironically, actually 5. + * + * 3 are "clean", with 2 "dirty" - the "dirty" two both split in the + * middle of input for the first keystroke. + */ + assert.equal(splits.length, 5); + assert.sameDeepMembers(splits.map(s => s.map(n => n.inputCount)), [ + // clean + [0, 2], // 1 inserted char, then both inputs + [1, 1], // 1 std keystroke, then the other input + [2, 0], // BOTH keystrokes processed (with one deleted), then insertions afterward + // dirty + [1, 2], // 1/2 keystroke inserted, other 1/2 is in tail + [2, 1] // first keystroke deleted, second keystroke 1/2 inserted with remainder in tail + ]); + + splits.forEach(s => { + assert.equal(s[0].codepointLength, 1); + assert.equal(s[1].codepointLength, 2); + }); + + const cleanSplit0KeyHead = splits.find(s => s[0].inputCount == 0 && s[1].inputCount == 2); + const cleanSplit1KeyHead = splits.find(s => s[0].inputCount == 1 && s[1].inputCount == 1); + const cleanSplit2KeyHead = splits.find(s => s[0].inputCount == 2 && s[1].inputCount == 0); + + assert.equal(constituentPaths(cleanSplit0KeyHead[0]).length, 1); + assert.sameDeepMembers(constituentPaths(cleanSplit0KeyHead[0]).map(toSpurTypeSequence), [ + ['insert'] + ]); + assert.equal(constituentPaths(cleanSplit1KeyHead[0]).length, 2); + assert.sameDeepMembers(constituentPaths(cleanSplit1KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'delete'], + ['substitute'] + ]); + assert.equal(constituentPaths(cleanSplit2KeyHead[0]).length, 3); + assert.sameDeepMembers(constituentPaths(cleanSplit2KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'delete', 'delete'], + ['substitute', 'delete'], + ['delete', 'substitute'] + ]); + }); + + it('splits properly at index 2', () => { + const nodes = buildQuotientDocFixture().nodes; + const k2c3 = nodes.k2c3; + + const splits = k2c3.split(2); + /* + * Unironically, actually 5. + * + * 3 are "clean", with 2 "dirty" - the "dirty" two both split in the + * middle of input for the first keystroke. + */ + assert.equal(splits.length, 5); + assert.sameDeepMembers(splits.map(s => s.map(n => n.inputCount)), [ + // clean + [0, 2], // 1 inserted char, then both inputs + [1, 1], // 1 std keystroke, then the other input + [2, 0], // BOTH keystrokes processed (with one deleted), then insertions afterward + // dirty + [1, 2], //insert, then 1/2 keystroke inserted, other 1/2 is in tail + [2, 1] // first keystroke deleted or short, second keystroke 1/2 inserted with remainder in tail + ]); + + splits.forEach(s => { + assert.equal(s[0].codepointLength, 2); + assert.equal(s[1].codepointLength, 1); + }); + + const cleanSplit0KeyHead = splits.find(s => s[0].inputCount == 0 && s[1].inputCount == 2); + const cleanSplit1KeyHead = splits.find(s => s[0].inputCount == 1 && s[1].inputCount == 1); + const cleanSplit2KeyHead = splits.find(s => s[0].inputCount == 2 && s[1].inputCount == 0); + + assert.equal(constituentPaths(cleanSplit0KeyHead[0]).length, 1); + assert.sameDeepMembers(constituentPaths(cleanSplit0KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'insert'] + ]); + assert.equal(constituentPaths(cleanSplit1KeyHead[0]).length, 4); + assert.sameDeepMembers(constituentPaths(cleanSplit1KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'insert', 'delete'], // is too high an edit-cost, though... + ['substitute', 'insert'], + ['insert', 'substitute'], + ['substitute'] + ]); + assert.equal(constituentPaths(cleanSplit2KeyHead[0]).length, 8); + assert.sameDeepMembers(constituentPaths(cleanSplit2KeyHead[0]).map(toSpurTypeSequence), [ + ['insert', 'insert', 'delete', 'delete'], // is too high an edit-cost, though... + ['substitute', 'substitute'], + ['substitute', 'delete'], + ['delete', 'substitute'], + ['insert', 'substitute', 'delete'], + ['substitute', 'insert', 'delete'], + ['insert', 'delete', 'substitute'], + ['delete', 'substitute', 'insert'] + ]); + }); + + it('splits properly at index 3', () => { + const fixture = buildQuotientDocFixture(); + const k2c3 = fixture.nodes.k2c3; + + const splits = k2c3.split(3); + assert.equal(splits.length, 1); + assert.equal(splits[0][0], k2c3); + assert.isTrue(splits[0][1].isSameNode(fixture.searchRoot)); + }); + }); }); // Placed after `split()` because many cases mock a reversal of split-test results. diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts new file mode 100644 index 00000000000..8419c4ab4bc --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/substitution-quotient-spur.tests.ts @@ -0,0 +1,303 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-04 + * + * This file defines tests for the SubstitutionQuotientSpur class of the + * predictive-text correction-search engine's search graph. + */ + +import { assert } from 'chai'; + +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { + generateSubsetId, + models, + SearchQuotientRoot, + SubstitutionQuotientSpur +} from '@keymanapp/lm-worker/test-index'; + +import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js'; +import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js'; + +import TrieModel = models.TrieModel; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + +describe('SubstitutionQuotientSpur', () => { + describe('constructor', () => { + it('may be extended from root path', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + + const extendedPath = new SubstitutionQuotientSpur(rootPath, leadEdgeDistribution, leadEdgeDistribution[0]); + + assert.equal(extendedPath.inputCount, 1); + assert.equal(extendedPath.codepointLength, 1); + assert.isNumber(extendedPath.spaceId); + assert.notEqual(extendedPath.spaceId, rootPath.spaceId); + assert.deepEqual(extendedPath.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(extendedPath.parents, [rootPath]); + assert.deepEqual(extendedPath.inputs, leadEdgeDistribution); + assert.deepEqual(extendedPath.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + // Assert the root is unchanged. + assert.equal(rootPath.inputCount, 0); + // Should (still) have codepointLength == 0 once it's defined. + assert.deepEqual(rootPath.bestExample, {text: '', p: 1}); + assert.deepEqual(rootPath.parents, []); + }); + + it('may be built from arbitrary prior SearchQuotientSpur', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + const inputClone = leadEdgeDistribution.map(e => ({...e})); + + const length1Path = new SubstitutionQuotientSpur( + rootPath, + leadEdgeDistribution, + leadEdgeDistribution[0] + ); + + const tailEdgeDistribution = [ + {sample: {insert: 'r', deleteLeft: 0, id: 17 }, p: 0.6}, + {sample: {insert: 'e', deleteLeft: 0, id: 17 }, p: 0.25}, + {sample: {insert: 'h', deleteLeft: 0, id: 17 }, p: 0.15} + ]; + + const length2Path = new SubstitutionQuotientSpur( + length1Path, + tailEdgeDistribution, + tailEdgeDistribution[0] + ); + + // Verify that the prior distribution remains fully unaltered. + assert.deepEqual(leadEdgeDistribution, inputClone); + + assert.equal(length2Path.inputCount, 2); + assert.equal(length2Path.codepointLength, 2); + assert.isNumber(length2Path.spaceId); + assert.notEqual(length2Path.spaceId, length1Path.spaceId); + assert.deepEqual(length2Path.bestExample, {text: 'tr', p: leadEdgeDistribution[0].p * tailEdgeDistribution[0].p}); + assert.deepEqual(length2Path.parents, [length1Path]); + assert.deepEqual(length2Path.inputs, tailEdgeDistribution); + assert.deepEqual(length2Path.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + }, { + transitionId: tailEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + assert.equal(length1Path.inputCount, 1); + assert.equal(length1Path.codepointLength, 1); + assert.isNumber(length1Path.spaceId); + assert.notEqual(length1Path.spaceId, rootPath.spaceId); + assert.deepEqual(length1Path.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(length1Path.parents, [rootPath]); + assert.deepEqual(length1Path.inputs, leadEdgeDistribution); + }); + + it('may extend with a Transform inserting multiple codepoints', () => { + const rootPath = new SearchQuotientRoot(testModel); + + const leadEdgeDistribution = [ + {sample: {insert: 't', deleteLeft: 0, id: 13 }, p: 0.5}, + {sample: {insert: 'a', deleteLeft: 0, id: 13 }, p: 0.3}, + {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} + ]; + const inputClone = leadEdgeDistribution.map(e => ({...e})); + + const length1Path = new SubstitutionQuotientSpur( + rootPath, + leadEdgeDistribution, + leadEdgeDistribution[0] + ); + + const tailEdgeDistribution = [ + {sample: {insert: 'ri', deleteLeft: 0, id: 17 }, p: 0.6}, + {sample: {insert: 'er', deleteLeft: 0, id: 17 }, p: 0.25}, + {sample: {insert: 'hi', deleteLeft: 0, id: 17 }, p: 0.15} + ]; + + const length2Path = new SubstitutionQuotientSpur( + length1Path, + tailEdgeDistribution, + tailEdgeDistribution[0] + ); + + // Verify that the prior distribution remains fully unaltered. + assert.deepEqual(leadEdgeDistribution, inputClone); + + assert.equal(length2Path.inputCount, 2); + assert.equal(length2Path.codepointLength, 3); + assert.isNumber(length2Path.spaceId); + assert.notEqual(length2Path.spaceId, length1Path.spaceId); + assert.deepEqual(length2Path.bestExample, {text: 'tri', p: leadEdgeDistribution[0].p * tailEdgeDistribution[0].p}); + assert.deepEqual(length2Path.parents, [length1Path]); + assert.deepEqual(length2Path.inputs, tailEdgeDistribution); + assert.deepEqual(length2Path.inputSegments, [ + { + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 + }, { + transitionId: tailEdgeDistribution[0].sample.id, + start: 0 + } + ]); + + assert.equal(length1Path.inputCount, 1); + assert.equal(length1Path.codepointLength, 1); + assert.isNumber(length1Path.spaceId); + assert.notEqual(length1Path.spaceId, rootPath.spaceId); + assert.deepEqual(length1Path.bestExample, {text: 't', p: 0.5}); + assert.deepEqual(length1Path.parents, [rootPath]); + assert.deepEqual(length1Path.inputs, leadEdgeDistribution); + }); + }); + + describe('.edgeKey', () => { + it('changes when input source subset IDs differ', () => { + const root = new SearchQuotientRoot(testModel); + + const {distributions} = buildCantLinearFixture(); + const inputSrc = { + segment: { + transitionId: distributions[0][0].sample.id, + start: 0 + }, + subsetId: generateSubsetId(), + bestProbFromSet: distributions[0][0].p + }; + + const spur1 = new SubstitutionQuotientSpur(root, distributions[0], { + ...inputSrc, + subsetId: generateSubsetId() + }); + const spur2 = new SubstitutionQuotientSpur(root, distributions[0], { + ...inputSrc, + subsetId: generateSubsetId() + }); + + assert.notEqual(spur1.edgeKey, spur2.edgeKey); + }); + + it('changes when different parts of the same input source are used', () => { + const root = new SearchQuotientRoot(testModel); + + const {distributions} = buildCantLinearFixture(); + const inputSrc = { + segment: { + transitionId: distributions[0][0].sample.id, + start: 0 + }, + subsetId: generateSubsetId(), + bestProbFromSet: distributions[0][0].p + }; + + const spur1 = new SubstitutionQuotientSpur(root, distributions[0], inputSrc); + const spur2 = new SubstitutionQuotientSpur(root, distributions[0], { + ...inputSrc, + segment: { + ...inputSrc.segment, + end: 1 + } + }); + const spur3 = new SubstitutionQuotientSpur(root, distributions[0], { + ...inputSrc, + segment: { + ...inputSrc.segment, + start: inputSrc.segment.start + 1 + } + }); + + assert.notEqual(spur1.edgeKey, spur2.edgeKey); + assert.notEqual(spur2.edgeKey, spur3.edgeKey); + assert.notEqual(spur3.edgeKey, spur1.edgeKey); + }); + }); + + describe('handleNextNode()', () => { + it('outputs results that directly match inputs', () => { + const canPath = buildCantLinearFixture().paths[3]; + + const matchTargets = [ + 'can', + 'car', + 'cen', // 'cent' and 'center' are supported in this test model. + ]; + const analysis = analyzeQuotientNodeResults(canPath, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('outputs results that substitute inputs', () => { + const canPath = buildCantLinearFixture().paths[3]; + + const matchTargets = [ + // Replacement of first char + 'man', + 'far', + // Replacement of second char + 'con', // 'consider' and variants thereof are also supported. + 'cor', // 'corner' + // Replacement of third char + 'cal', // 'call' + ]; + const analysis = analyzeQuotientNodeResults(canPath, matchTargets); + + assert.sameMembers(analysis.found, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that insert characters as needed', () => { + const canPath = buildCantLinearFixture().paths[3]; + + const matchTargets = [ + 'char', // 'c' + (insert) 'h' + 'ar' + 'than', // 't' + (insert) 'h' + 'an' + ]; + const analysis = analyzeQuotientNodeResults(canPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + + it('does not output results that delete incoming keystrokes as needed', () => { + const canPath = buildCantLinearFixture().paths[3]; + + const matchTargets = [ + // Delete only first + 'an', // (delete) 'c'/'r'/'t' + 'an', for 'and' and 'any' + 'en', // (delete) 'c'/'r'/'t' + 'en', for 'end', + // Even delete second + 'n', // model possesses words starting with just 'n' + 't' // ... and 't'. + ]; + const analysis = analyzeQuotientNodeResults(canPath, matchTargets); + + assert.sameMembers(analysis.found, []); + assert.sameMembers(analysis.missing, matchTargets); + assert.isEmpty(analysis.foundWithDuplicates); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts index 711f494f085..4b7577662c0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts @@ -21,18 +21,20 @@ import { ExecutionTimer, generateSubsetId, getBestMatches, - LegacyQuotientSpur, models, PathInputProperties, PathResult, SearchQuotientNode, SearchQuotientRoot, + SubstitutionQuotientSpur, TokenizationCorrector, TokenResult, - TokenizationResultMapping + TokenizationResultMapping, + TokenizationResult } from '@keymanapp/lm-worker/test-index'; import Distribution = LexicalModelTypes.Distribution; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import TrieModel = models.TrieModel; import Transform = LexicalModelTypes.Transform; @@ -86,22 +88,21 @@ function buildFixture_therefore() { const therefTokens: ContextToken[] = []; // as in "therefore" const the_efTokens: ContextToken[] = []; // as in "the effect" - // TODO: Use SubstitutionQuotientSpur instead! let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=0; i < 3; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); } the_efTokens.push(new ContextToken(firstTokenNode, false)); - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, [distributions[3][1]], { + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, [distributions[3][1]], { ...inputSources[3], subsetId: generateSubsetId() }); // whitespace token alternate - using the ' ' input instead. const whitespaceToken = new ContextToken( - new LegacyQuotientSpur( + new SubstitutionQuotientSpur( new SearchQuotientRoot(plainModel), [distributions[3][0]], { ...inputSources[3], subsetId: generateSubsetId() } @@ -112,11 +113,11 @@ function buildFixture_therefore() { let secondTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=4; i < distributions.length; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], { + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], { ...inputSources[i], subsetId: generateSubsetId() }); - secondTokenNode = new LegacyQuotientSpur(secondTokenNode, distributions[i], { + secondTokenNode = new SubstitutionQuotientSpur(secondTokenNode, distributions[i], { ...inputSources[i], subsetId: generateSubsetId() }) @@ -171,17 +172,16 @@ function buildFixture_terminalWhitespace() { const fullTokens: ContextToken[] = []; const lastToken: ContextToken[] = []; - // TODO: Use SubstitutionQuotientSpur instead! let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=0; i < 5; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); } fullTokens.push(new ContextToken(firstTokenNode, false)); // whitespace token alternate - using the ' ' input instead. const whitespaceToken = new ContextToken( - new LegacyQuotientSpur( + new SubstitutionQuotientSpur( new SearchQuotientRoot(plainModel), distributions[5], inputSources[5], @@ -303,7 +303,7 @@ describe('TokenizationCorrector', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; + const tokenResults = mapping.matchedResult.tokenCorrections; assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); assert.equal(tokenResults.length, 1); @@ -316,14 +316,63 @@ describe('TokenizationCorrector', () => { } searchResult = instance.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); do { searchResult = instance.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. + assert.equal(searchResult.type, 'none'); + }); + + it('returns no result when a single correctable token lacks a model match', () => { + const fixture = buildFixture_therefore(); + + const theref = fixture.theref.tail; + const xInput: ProbabilityMass = { + sample: { + insert: 'x', + deleteLeft: 0, + id: 123 + }, + p: 1 + } + const therefx = new SubstitutionQuotientSpur(theref.searchModule, [xInput], xInput); + const yInput: ProbabilityMass = { + sample: { + insert: 'y', + deleteLeft: 0, + id: 124 + }, + p: 1 + } + const therefxy = new SubstitutionQuotientSpur(therefx, [yInput], yInput); + const zInput: ProbabilityMass = { + sample: { + insert: 'z', + deleteLeft: 0, + id: 125 + }, + p: 1 + } + const therefxyz = new ContextToken(new SubstitutionQuotientSpur(therefxy, [zInput], zInput)); + const therefxyzTokenization = new ContextTokenization([therefxyz]); + + const instance = new TokenizationCorrector( + therefxyzTokenization, + 1, + fixture.filter + ); + + let searchResult: PathResult; + do { + searchResult = instance.handleNextNode(); + } while(searchResult.type == 'intermediate'); + + assert.equal(searchResult.type, 'none'); }); it('finds corrections for a group of tokens with two correctable', () => { @@ -346,7 +395,7 @@ describe('TokenizationCorrector', () => { let firstResults: ReadonlyArray; if(searchResult.type == 'complete') { const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; + const tokenResults = mapping.matchedResult.tokenCorrections; firstResults = tokenResults; assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); @@ -369,7 +418,7 @@ describe('TokenizationCorrector', () => { searchResult = instance.handleNextNode(); if(searchResult.type == 'complete') { const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; + const tokenResults = mapping.matchedResult.tokenCorrections; // Verify that the first (bound) token is not altered further. // It should receive no further correction attempts. @@ -380,7 +429,7 @@ describe('TokenizationCorrector', () => { } while(searchResult.type != 'none'); }); - it('immediately returns a single result when the only represented token is uncorrectable', () => { + it('immediately returns with no result when the only represented token is uncorrectable', () => { const fixture = buildFixture_terminalWhitespace(); const tokenization = fixture.spaceOnly; @@ -392,13 +441,7 @@ describe('TokenizationCorrector', () => { ); const searchResult = instance.handleNextNode(); - assert.equal(searchResult.type, 'complete'); - if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.matchedResult[0].matchString, ' '); - } - - const nilResult = instance.handleNextNode(); - assert.equal(nilResult.type, 'none'); + assert.equal(searchResult.type, 'none'); }); it('returns a single result when the final token is uncorrectable', () => { @@ -419,8 +462,8 @@ describe('TokenizationCorrector', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.matchedResult[0].matchString, 'space'); - assert.equal(searchResult.mapping.matchedResult[1].matchString, ' '); + assert.equal(searchResult.mapping.matchedResult.tokenCorrections[0].matchString, 'space'); + assert.equal(searchResult.mapping.matchedResult.tokenCorrections[1].matchString, ' '); } const nilResult = instance.handleNextNode(); @@ -437,20 +480,20 @@ describe('TokenizationCorrector', () => { let haveSeenSingleTokenCorrection = false; let haveSeenThreeTokenCorrection = false; for await(let phraseMatch of getBestMatches< - ReadonlyArray, + TokenizationResult, TokenizationResultMapping, TokenizationCorrector >(correctors, buildTestTimer())) { - if(phraseMatch.matchedResult.length == 1) { + if(phraseMatch.matchedResult.tokenCorrections.length == 1) { if(!haveSeenSingleTokenCorrection) { - assert.sameOrderedMembers(phraseMatch.matchedResult.map((t) => t.matchString), ['theref' /* -ore */]); + assert.sameOrderedMembers(phraseMatch.matchedResult.tokenCorrections.map((t) => t.matchString), ['theref' /* -ore */]); } haveSeenSingleTokenCorrection = true; - } else if(phraseMatch.matchedResult.length == 3) { + } else if(phraseMatch.matchedResult.tokenCorrections.length == 3) { if(!haveSeenThreeTokenCorrection) { - assert.sameOrderedMembers(phraseMatch.matchedResult.map((t) => t.matchString), ['the', ' ', 'ef' /* -fort */]); + assert.sameOrderedMembers(phraseMatch.matchedResult.tokenCorrections.map((t) => t.matchString), ['the', ' ', 'ef' /* -fort */]); } haveSeenThreeTokenCorrection = true; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts index 882390591a9..afcb02af09a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts @@ -1,6 +1,6 @@ import { assert } from 'chai'; -import { AUTOSELECT_PROPORTION_THRESHOLD, CorrectionPredictionTuple, predictionAutoSelect, SuggestionSimilarity, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; +import { AUTOSELECT_PROPORTION_THRESHOLD, CompositedIntermediatePrediction, predictionAutoSelect, SuggestionSimilarity, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; /* * Preconditions: * - there should always be a 'keep' option. Now, whether or not that option @@ -9,7 +9,7 @@ import { AUTOSELECT_PROPORTION_THRESHOLD, CorrectionPredictionTuple, predictionA */ describe('predictionAutoSelect', () => { it(`does not throw when no suggestions are available`, () => { - const predictions: CorrectionPredictionTuple[] = []; + const predictions: CompositedIntermediatePrediction[] = []; const originalPredictions = [].concat(predictions); assert.doesNotThrow(() => predictionAutoSelect(predictions)); @@ -17,14 +17,10 @@ describe('predictionAutoSelect', () => { }); it(`selects nothing if solitary 'keep' suggestion does match the model`, () => { - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ { - correction: { - sample: 'apple', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'e', @@ -33,9 +29,16 @@ describe('predictionAutoSelect', () => { matchesModel: true, displayAs: 'apple' }, - p: 1 + correction: 'apple', }, - totalProb: 1 + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 + }, + autoSelectable: true + } } ]; @@ -43,19 +46,15 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); it(`does not select suggestions if the root correction has no letters`, () => { - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ { - correction: { - sample: '5', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { tag: 'keep', transform: { insert: '5', @@ -64,17 +63,20 @@ describe('predictionAutoSelect', () => { matchesModel: false, displayAs: '5' }, - p: 0.01 + correction: '5' }, - totalProb: 0.01 + metadata: { + probabilities: { + prediction: 0.01, + correction: 1, + total: 0.01 + }, + autoSelectable: false + } }, { - correction: { - sample: '5', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { transform: { insert: '5th', deleteLeft: 0 @@ -82,9 +84,16 @@ describe('predictionAutoSelect', () => { matchesModel: true, displayAs: '5th' }, - p: 0.8 + correction: '5' }, - totalProb: 0.8 + metadata: { + probabilities: { + prediction: 0.8, + correction: 1, + total: 0.8 + }, + autoSelectable: false + } } ]; @@ -92,19 +101,15 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); it(`does not select solitary 'keep' suggestion that doesn't match the model`, () => { - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ { - correction: { - sample: 'appl', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'l', @@ -113,9 +118,16 @@ describe('predictionAutoSelect', () => { matchesModel: false, displayAs: '"appl"' }, - p: 1 + correction: 'appl' }, - totalProb: 1 + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 + }, + autoSelectable: true + } } ]; @@ -123,18 +135,14 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); it(`selects nothing for 'keep' suggestion that does match the model even with alternatives`, () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const keepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'n', @@ -143,65 +151,81 @@ describe('predictionAutoSelect', () => { matchesModel: true, displayAs: 'thin' }, - p: .05 + correction: 'thin' }, - totalProb: .04 + metadata: { + probabilities: { + prediction: .05, + correction: .8, + total: .05 * .8 + }, + autoSelectable: true + } } - const highestNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const highestNonKeepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'nk', deleteLeft: 0 }, displayAs: 'think' }, - p: .55 + correction: 'thin' }, - totalProb: .44 + metadata: { + probabilities: { + prediction: .55, + correction: .8, + total: .55 * .8 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ng', deleteLeft: 0 }, displayAs: 'thing' }, - p: .4 + correction: 'thin' }, - totalProb: .32 + metadata: { + probabilities: { + prediction: .4, + correction: .8, + total: .4 * .8 + }, + autoSelectable: true + } }, { - correction: { - sample: 'thic', - p: .2 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ck', deleteLeft: 0 }, displayAs: 'thick' }, - p: 1 + correction: 'thic' }, - totalProb: .2 + metadata: { + probabilities: { + prediction: 1, + correction: .2, + total: 1 * .2 + }, + autoSelectable: true + } } ]; @@ -209,18 +233,14 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const keepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'n', @@ -229,40 +249,50 @@ describe('predictionAutoSelect', () => { displayAs: '"thin"', matchesModel: false }, - p: .05 + correction: 'thin' }, - totalProb: .04 + metadata: { + probabilities: { + prediction: .05, + correction: .8, + total: .8 * .05 + }, + autoSelectable: true + } } // To 'win', a suggestion (currently) needs at least twice the probability of the sum of all alternatives. // This threshold may be subject to change. // // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts. - const onlyNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const onlyNonKeepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'nk', deleteLeft: 0 }, displayAs: 'think' }, - p: .01 + correction: 'thin' }, - totalProb: .008 + metadata: { + probabilities: { + prediction: .01, + correction: .8, + total: .01 * .8 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, onlyNonKeepSuggestion ]; - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isBelow(onlyNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); + const totalProb = predictions.reduce((accum, current) => accum + current.metadata.probabilities.total, 0); + assert.isBelow(onlyNonKeepSuggestion.metadata.probabilities.total, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); predictions.sort(tupleDisplayOrderSort); @@ -270,18 +300,14 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.equal(autoselected, onlyNonKeepSuggestion); }); it(`does not select non-'keep' without sufficient winning probability`, () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const keepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'n', @@ -290,74 +316,90 @@ describe('predictionAutoSelect', () => { displayAs: '"thin"', matchesModel: false }, - p: .05 + correction: 'thin' }, - totalProb: .04 + metadata: { + probabilities: { + prediction: .05, + correction: .8, + total: .05 * .8 + }, + autoSelectable: true + } } // To 'win', a suggestion (currently) needs at least twice the probability of the sum of all alternatives. // This threshold may be subject to change. // // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts. - const highestNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const highestNonKeepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'nk', deleteLeft: 0 }, displayAs: 'think' }, - p: .55 + correction: 'thin' }, - totalProb: .44 + metadata: { + probabilities: { + prediction: .55, + correction: .8, + total: .55 * .8 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ng', deleteLeft: 0 }, displayAs: 'thing' }, - p: .4 + correction: 'thin' }, - totalProb: .32 + metadata: { + probabilities: { + prediction: .4, + correction: .8, + total: .4 * .8 + }, + autoSelectable: true + } }, { - correction: { - sample: 'thic', - p: .2 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ck', deleteLeft: 0 }, displayAs: 'thick' }, - p: 1 + correction: 'thic' }, - totalProb: .2 + metadata: { + probabilities: { + prediction: 1, + correction: .2, + total: 1 * .2 + }, + autoSelectable: true + } } ]; - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isBelow(highestNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); + const totalProb = predictions.reduce((accum, current) => accum + current.metadata.probabilities.total, 0); + assert.isBelow(highestNonKeepSuggestion.metadata.probabilities.total, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); predictions.sort(tupleDisplayOrderSort); @@ -365,18 +407,14 @@ describe('predictionAutoSelect', () => { assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepOrderedMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); it(`does select non-'keep' with sufficient winning probability`, () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .8 - }, - prediction: { - sample: { + const keepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'n', @@ -385,87 +423,99 @@ describe('predictionAutoSelect', () => { displayAs: '"thin"', matchesModel: false }, - p: .05 + correction: 'thin' }, - totalProb: .04 + metadata: { + probabilities: { + prediction: .05, + correction: .8, + total: .05 * .8 + }, + autoSelectable: true + } } - const highestNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thin', - p: .9 - }, - prediction: { - sample: { + const highestNonKeepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'nk', deleteLeft: 0 }, displayAs: 'think' }, - p: .75 + correction: 'thin' }, - totalProb: .675 + metadata: { + probabilities: { + prediction: .75, + correction: .9, + total: .75 * .9 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, highestNonKeepSuggestion, { - correction: { - sample: 'thin', - p: .9 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ng', deleteLeft: 0 }, displayAs: 'thing' }, - p: .2 + correction: 'thin' }, - totalProb: .18 + metadata: { + probabilities: { + prediction: .2, + correction: .9, + total: .2 * .9 + }, + autoSelectable: true + } }, { - correction: { - sample: 'thic', - p: .1 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'ck', deleteLeft: 0 }, displayAs: 'thick' }, - p: 1 + correction: 'thic' }, - totalProb: .1 + metadata: { + probabilities: { + prediction: 1, + correction: .1, + total: 1 * .1 + }, + autoSelectable: true + } } ]; - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isAbove(highestNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); + const totalProb = predictions.reduce((accum, current) => accum + current.metadata.probabilities.total, 0); + assert.isAbove(highestNonKeepSuggestion.metadata.probabilities.total, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); const originalPredictions = [].concat(predictions); assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.equal(autoselected, highestNonKeepSuggestion); }); it('ignores non key-matched suggestions when key-matched suggestions exist', () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'cant', - p: 1 - }, - prediction: { - sample: { + const keepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 't', @@ -474,51 +524,64 @@ describe('predictionAutoSelect', () => { displayAs: '"cant"', matchesModel: false }, - p: 1 + correction: 'cant' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: true, + matchLevel: SuggestionSimilarity.exact + } } - const expectedSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'cant', - p: 1 - }, - prediction: { - sample: { + const expectedSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: '\'t', deleteLeft: 0 }, displayAs: "can't" }, - p: .2 + correction: 'cant' }, - totalProb: .2, - matchLevel: SuggestionSimilarity.sameKey + metadata: { + probabilities: { + prediction: .2, + correction: 1, + total: .2 * 1 + }, + autoSelectable: true, + matchLevel: SuggestionSimilarity.sameKey + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, expectedSuggestion, { - correction: { - sample: 'cant', - p: 1 - }, - prediction: { - sample: { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'teen', deleteLeft: 0 }, displayAs: 'canteen' }, - p: .8 + correction: 'cant' }, - totalProb: .8, - matchLevel: SuggestionSimilarity.none + metadata: { + probabilities: { + prediction: .8, + correction: 1, + total: .8 * 1 + }, + autoSelectable: true, + matchLevel: SuggestionSimilarity.none + } } ]; @@ -527,20 +590,16 @@ describe('predictionAutoSelect', () => { assert.sameDeepMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.equal(autoselected, expectedSuggestion); }); // The idea: avoid "over-correcting" when a potential correction has a // super-high-frequency word. it('does not auto-select suggestion if its root correction is not most likely', () => { - const keepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thi', - p: .7 - }, - prediction: { - sample: { + const keepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { tag: 'keep', transform: { // can be null / "mocked out" insert: 'i', @@ -549,61 +608,74 @@ describe('predictionAutoSelect', () => { displayAs: '"thi"', matchesModel: false }, - p: .05 + correction: 'thi' }, - totalProb: .035 + metadata: { + probabilities: { + prediction: .05, + correction: .7, + total: .05 * .7 + }, + autoSelectable: true + } } - const highestCorrectionSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'thi', - p: .7 - }, - prediction: { - sample: { + const highestCorrectionSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'in', deleteLeft: 0 }, displayAs: 'thin' }, - p: .1 + correction: 'thi', }, - totalProb: .07 + metadata: { + probabilities: { + prediction: .1, + correction: .7, + total: .1 * .7 + }, + autoSelectable: true + } }; - const highestNonKeepSuggestion: CorrectionPredictionTuple = { - correction: { - sample: 'the', - p: .3 - }, - prediction: { - sample: { + const highestNonKeepSuggestion: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { // can be null / "mocked out" insert: 'e', deleteLeft: 0 }, displayAs: 'the' }, - p: 1 + correction: 'the' }, - totalProb: .3 + metadata: { + probabilities: { + prediction: 1, + correction: .3, + total: 1 * .3 + }, + autoSelectable: true + } }; - const predictions: CorrectionPredictionTuple[] = [ + const predictions: CompositedIntermediatePrediction[] = [ keepSuggestion, highestNonKeepSuggestion, highestCorrectionSuggestion ]; - const totalProb = predictions.reduce((accum, current) => accum + current.totalProb, 0); - assert.isAbove(highestNonKeepSuggestion.totalProb, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); + const totalProb = predictions.reduce((accum, current) => accum + current.metadata.probabilities.total, 0); + assert.isAbove(highestNonKeepSuggestion.metadata.probabilities.total, totalProb * AUTOSELECT_PROPORTION_THRESHOLD, 'test setup is no longer valid'); const originalPredictions = [].concat(predictions); assert.doesNotThrow(() => predictionAutoSelect(predictions)); assert.sameDeepMembers(predictions, originalPredictions); - const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); assert.isNotOk(autoselected); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/build-and-map-predictions.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/build-and-map-predictions.tests.ts deleted file mode 100644 index 73d8276e132..00000000000 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/build-and-map-predictions.tests.ts +++ /dev/null @@ -1,220 +0,0 @@ - -/* - * Keyman is copyright (C) SIL Global. MIT License. - * - * Created by jahorton on 2026-07-23 - * - * This file contains tests designed to validate the behavior of the - * buildAndMapPredictions helper function class and its integration with the - * lower-level predictive-text helpers. - */ - - -import { assert } from 'chai'; - -import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; -import { LexicalModelTypes } from '@keymanapp/common-types'; -import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; -import { TrieModel } from '@keymanapp/models-templates'; - -import { buildAndMapPredictions, buildEdgeWindow, ContextState, ContextToken, ContextTokenization, ContextTransition, generateSubsetId, LegacyQuotientRoot, LegacyQuotientSpur, models, predictFromCorrections } from "@keymanapp/lm-worker/test-index"; - -import Context = LexicalModelTypes.Context; -import Distribution = LexicalModelTypes.Distribution; -import ProbabilityMass = LexicalModelTypes.ProbabilityMass; -import Transform = LexicalModelTypes.Transform; - -const plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), - {wordBreaker: defaultBreaker}); - -describe('buildAndMapPredictions', () => { - it('adds the preservation transform to all generated predictions', () => { - const context: Context = { - left: 'th', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const correctionDistribution: Distribution = [{ - sample: { - insert: 'e', - deleteLeft: 0 - }, - p: 0.6 - } - ]; - - const basePredictions = predictFromCorrections(plainModel, correctionDistribution, context); - basePredictions.forEach((entry) => assert.isNotOk(entry.preservationTransform)); - - // must construct the taillessTrueKeystroke appropriately. - const tailless = { insert: 'TEST', deleteLeft: 0 }; - const tokenization = new ContextTokenization([ContextToken.fromRawText(plainModel, 'th', true)], null, tailless); - const transition = new ContextTransition(new ContextState(context, plainModel, tokenization), 0); - - const targetTokenization = new ContextTokenization([new ContextToken(new LegacyQuotientSpur(tokenization.tail.searchModule, correctionDistribution, correctionDistribution[0]))]); - transition.finalize(new ContextState(models.applyTransform(correctionDistribution[0].sample, context), plainModel, targetTokenization), correctionDistribution); - - const mappedPredictions = buildAndMapPredictions( - transition, - transition.base.displayTokenization, - {matchString: 'the', totalCost: 0}, - 1 - ); - - assert.deepEqual(mappedPredictions.map((tuple) => tuple.prediction), basePredictions.map((tuple) => tuple.prediction)); - mappedPredictions.forEach((tuple) => assert.isOk(tuple.preservationTransform)); - mappedPredictions.forEach((tuple) => tuple.preservationTransform == tailless); - }); - - it('properly handles empty prediction roots from deleted same-token codepoints', () => { - const context: Context = { - left: 'the a', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const correctionDistribution: Distribution = [{ - sample: { - insert: '', - deleteLeft: 1 - }, - p: 1 - } - ]; - - const basePredictions = predictFromCorrections(plainModel, correctionDistribution, context); - - // must construct the taillessTrueKeystroke appropriately. - const tokenization = new ContextTokenization([ - ContextToken.fromRawText(plainModel, 'the', false), - ContextToken.fromRawText(plainModel, ' ', false), - ContextToken.fromRawText(plainModel, 'a', true) - ]); - const transition = new ContextTransition(new ContextState(context, plainModel, tokenization), 0); - - const targetTokenization = new ContextTokenization([ - tokenization.tokens[0], - tokenization.tokens[1], - new ContextToken(new LegacyQuotientRoot(plainModel)) - ]); - transition.finalize(new ContextState(models.applyTransform(correctionDistribution[0].sample, context), plainModel, targetTokenization), correctionDistribution); - - const mappedPredictions = buildAndMapPredictions( - transition, - transition.base.displayTokenization, - {matchString: '', totalCost: 0}, - 1 - ); - - assert.deepEqual(mappedPredictions.map((tuple) => tuple.prediction), basePredictions.map((tuple) => tuple.prediction)); - }); - - it('properly handles empty prediction roots caused by backspacing one of multiple spaces', () => { - const context: Context = { - left: 'the ', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const correctionDistribution: Distribution = [{ - sample: { - insert: '', - deleteLeft: 1 - }, - p: 1 - } - ]; - - const basePredictions = predictFromCorrections(plainModel, correctionDistribution, context); - - // must construct the taillessTrueKeystroke appropriately. - const tokenization = new ContextTokenization([ - ContextToken.fromRawText(plainModel, 'the', false), - ContextToken.fromRawText(plainModel, ' ', false), - ContextToken.fromRawText(plainModel, '', true) - ]); - const transition = new ContextTransition(new ContextState(context, plainModel, tokenization), 0); - - const targetTokenization = new ContextTokenization([ - tokenization.tokens[0], - new ContextToken(new LegacyQuotientSpur(tokenization.tokens[1].searchModule, correctionDistribution, correctionDistribution[0])), - new ContextToken(new LegacyQuotientRoot(plainModel)) - ]); - transition.finalize(new ContextState(models.applyTransform(correctionDistribution[0].sample, context), plainModel, targetTokenization), correctionDistribution); - - const mappedPredictions = buildAndMapPredictions( - transition, - transition.base.displayTokenization, - {matchString: '', totalCost: 0}, - 1 - ); - - assert.deepEqual(mappedPredictions.map((tuple) => tuple.prediction), basePredictions.map((tuple) => tuple.prediction)); - }); - - it('properly handles contexts made empty by input backspace', () => { - const context: Context = { - left: 't', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const correctionDistribution = [{ - sample: { - insert: '', - deleteLeft: 1, - deleteRight: 0 - }, - p: 1 - } - ]; - - // must construct the taillessTrueKeystroke appropriately. - const tokenization = new ContextTokenization([ - ContextToken.fromRawText(plainModel, 't', true) - ]); - const transition = new ContextTransition(new ContextState(context, plainModel, tokenization), 0); - - const targetTokenization = new ContextTokenization([ - new ContextToken(new LegacyQuotientRoot(plainModel)) - ], { - alignment: { - merges: [], - splits: [], - unmappedEdits: [], - edgeWindow: { - ...buildEdgeWindow(tokenization.tokens, correctionDistribution[0].sample, false), - retokenization: [''], - retokenizationText: '' - }, - removedTokenCount: 0 - }, - inputs: (() => { - const val: ProbabilityMass>[] = [{ - sample: new Map(), - p: correctionDistribution[0].p - }]; - - val[0].sample.set(0, correctionDistribution[0].sample); - - return val; - })(), - inputSubsetId: generateSubsetId() - }, null); - transition.finalize(new ContextState(models.applyTransform(correctionDistribution[0].sample, context), plainModel, targetTokenization), correctionDistribution); - - const mappedPredictions = buildAndMapPredictions( - transition, - transition.final.displayTokenization, - {matchString: '', totalCost: 0}, - 1 - ); - - mappedPredictions.forEach((tuple) => assert.equal(tuple.prediction.sample.transform.deleteLeft, 1)); - }); -}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts index 6b5981c8343..8bed658ed4c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts @@ -12,7 +12,7 @@ import { assert } from 'chai'; import { LexicalModelTypes } from "@keymanapp/common-types"; import * as wordBreakers from '@keymanapp/models-wordbreakers'; -import { CorrectionPredictionTuple, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index"; import CasingFunction = LexicalModelTypes.CasingFunction; import Context = LexicalModelTypes.Context; @@ -91,8 +91,47 @@ const testModelWithCasing = new DummyModel({ // No suggestions needed here, so we don't define any. }); -describe('produceKeep', () => { - it(`creates an 'exact'-match suggestion based on primary input and current context`, () => { +describe('createDefaultKeep', () => { + it(`creates an 'exact'-match suggestion based on context when no change occurs and no match is found`, () => { + const transformId = 314159; + + const context: Context = { + left: 'appl', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const expectedKeep: CompositedIntermediatePrediction = { + components: { + prediction: { + transform: { + insert: 'appl', + deleteLeft: 4, + id: transformId + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + correction: 'appl' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, { sample: { insert: '', deleteLeft: 0, id: transformId }, p: 1}); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on simple primary input`, () => { const context: Context = { left: 'iphon', right: '', @@ -108,13 +147,9 @@ describe('produceKeep', () => { p: 1 }; - const expectedKeep: CorrectionPredictionTuple = { - correction: { - sample: 'iphone', - p: 1 - }, - prediction: { - sample: { + const expectedKeep: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 'iphone', deleteLeft: 5 @@ -123,10 +158,237 @@ describe('produceKeep', () => { matchesModel: false, tag: 'keep' }, - p: 1 + correction: 'iphone' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on full word after a backspace`, () => { + const context: Context = { + left: 'iphone ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: '', + deleteLeft: 1 + }, + p: 1 + }; + + const expectedKeep: CompositedIntermediatePrediction = { + components: { + prediction: { + transform: { + insert: 'iphone', + deleteLeft: 7 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + correction: 'iphone' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex deletion`, () => { + const context: Context = { + left: 'iphone a', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e', + deleteLeft: 3 + }, + p: 1 + }; + + const expectedKeep: CompositedIntermediatePrediction = { + components: { + prediction: { + transform: { + insert: 'iphone', + deleteLeft: 8 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + correction: 'iphone' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex insertion`, () => { + const context: Context = { + left: 'iphon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'es and', + deleteLeft: 0 + }, + p: 1 + }; + + const expectedKeep: CompositedIntermediatePrediction = { + components: { + prediction: { + transform: { + insert: 'iphones and', + deleteLeft: 5 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + correction: 'and' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex replacement`, () => { + const context: Context = { + left: 'iphone ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 's', + deleteLeft: 1 + }, + p: 1 + }; + + const expectedKeep: CompositedIntermediatePrediction = { + components: { + prediction: { + transform: { + insert: 'iphones', + deleteLeft: 7 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + correction: 'iphones' }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an empty 'exact'-match suggestion after adding a wordbreak`, () => { + const context: Context = { + left: 'iphon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e ', + deleteLeft: 0 + }, + p: 1 + }; + + const expectedKeep: CompositedIntermediatePrediction = { + components: { + prediction: { + transform: { + insert: 'iphone ', + deleteLeft: 5 + }, + displayAs: '<>', + matchesModel: false, + tag: 'keep' + }, + correction: '' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 * 1 + }, + autoSelectable: false, + matchLevel: SuggestionSimilarity.exact + } }; const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts deleted file mode 100644 index 1e5146795fc..00000000000 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-alignment.tests.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { assert } from 'chai'; - -import { LexicalModelTypes } from '@keymanapp/common-types'; -import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; -import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; - -import { ContextState, ContextTransition, determineSuggestionAlignment, models } from "@keymanapp/lm-worker/test-index"; - -import CasingFunction = LexicalModelTypes.CasingFunction; -import Context = LexicalModelTypes.Context; -import TrieModel = models.TrieModel; - -const plainApplyCasing: CasingFunction = function(caseToApply, text) { - switch(caseToApply) { - case 'lower': - return text.toLowerCase(); - case 'upper': - return text.toUpperCase(); - case 'initial': - return plainApplyCasing('upper', text.charAt(0)) . concat(text.substring(1)); - default: - return text; - } -}; - -const plainCasedModel = new TrieModel( - jsonFixture('models/tries/english-1000'), { - languageUsesCasing: true, - applyCasing: plainApplyCasing, - wordBreaker: defaultBreaker, - searchTermToKey: function(text: string) { - // We're dealing with very simple English text; no need to normalize or remove diacritics here. - return plainApplyCasing('lower', text); - } - } -); - -describe('determineSuggestionAlignment', () => { - it('handles standard cases well - same token, no preservationTransforms', () => { - const context: Context = { - left: 'this is techn', - startOfBuffer: true, - endOfBuffer: true - }; - const baseState = new ContextState(context, plainCasedModel); - - const transition = new ContextTransition(baseState, 0); - transition.finalize(transition.base, [{sample: { insert: '', deleteLeft: 0 }, p: 1}]); - - // transition, model - const results = determineSuggestionAlignment(transition, transition.final.tokenization, plainCasedModel); - - assert.deepEqual(results.predictionContext, context); - assert.equal(results.correctionDeleteLeft, "techn".length /* does not include the deleted whitespace */); - assert.equal(results.committedDeleteLeft, 0); - }); - - it('handles extension of prior token after backspace', () => { - const context: Context = { - left: 'this is tech ', - startOfBuffer: true, - endOfBuffer: true - }; - const baseState = new ContextState(context, plainCasedModel); - - const transition = baseState.analyzeTransition(context, [{sample: { insert: '', deleteLeft: 1 }, p: 1}]) - - // transition, model - const results = determineSuggestionAlignment(transition, transition.final.tokenization, plainCasedModel); - - assert.deepEqual(results.predictionContext, { - ...context, - left: context.left.substring(0, context.left.length - 1), - right: '', - casingForm: undefined - }); - assert.equal(results.correctionDeleteLeft, "tech".length /* does not include the deleted whitespace */); - assert.equal(results.committedDeleteLeft, 1 /* for the deleted whitespace */); - }); - - it('handles extension of prior token after complex input with delete-left', () => { - const context: Context = { - left: 'this is tech ', - startOfBuffer: true, - endOfBuffer: true - }; - const baseState = new ContextState(context, plainCasedModel); - - const transition = baseState.analyzeTransition(context, [{sample: { insert: 'n', deleteLeft: 1 }, p: 1}]) - - // transition, model - const results = determineSuggestionAlignment(transition, transition.final.tokenization, plainCasedModel); - - assert.deepEqual(results.predictionContext, { - ...context, - left: context.left.substring(0, context.left.length - 1), - right: '', - casingForm: undefined - }); - assert.equal(results.correctionDeleteLeft, "tech".length /* does not include the deleted whitespace */); - assert.equal(results.committedDeleteLeft, 1 /* for the deleted whitespace */); - }); -}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index e9110215370..2412af3fcde 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -103,12 +103,10 @@ describe('determineContextTransition', () => { assert.isOk(transition); assert.equal(transition, tracker.latest); assert.isFalse(warningEmitterSpy.called); - assert.sameOrderedMembers(transition.final.tokenization.exampleInput, ['this', ' ', 'is', ' ', 'for', ' ', 'techn']); - assert.isOk(transition.final.tokenization.transitionEdits); + assert.sameOrderedMembers(transition.final.displayTokenization.exampleInput, ['this', ' ', 'is', ' ', 'for', ' ', 'techn']); assert.equal(transition.final.context.left, targetContext.left); assert.equal(transition.final.context.right ?? "", targetContext.right ?? ""); assert.sameDeepOrderedMembers(transition.inputDistribution, inputDistribution); - assert.isNotOk(transition.final.tokenization.taillessTrueKeystroke); assert.equal(transition.transitionId, 1); } finally { warningEmitterSpy.restore(); @@ -224,8 +222,8 @@ describe('determineContextTransition', () => { assert.notEqual(extendingTransition, baseTransition); // These values support delayed reversions. - assert.equal(extendingTransition.final.tokenization.tokens[6].appliedTransitionId, pred_testing.transform.id); - assert.equal(extendingTransition.final.tokenization.tokens[7].appliedTransitionId, pred_testing.transform.id); + assert.equal(extendingTransition.final.displayTokenization.tokens[6].appliedTransitionId, pred_testing.transform.id); + assert.equal(extendingTransition.final.displayTokenization.tokens[7].appliedTransitionId, pred_testing.transform.id); // We start a new token here, rather than continue (and/or replace) an old one; // this shouldn't be set here yet. diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts index 3445abaadd9..7bc7bd7208b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts @@ -47,8 +47,6 @@ function buildQuickBrownFixture() { baseTokenization.tokens.slice(0, baseTokenCount-1).concat( new ContextToken(new LegacyQuotientSpur(baseTokenization.tail.searchModule, plainInsertDistrib, plainInsertDistrib[0])) ), - null, - null ); const newTokenInsertDistrib: Distribution = [ @@ -58,8 +56,6 @@ function buildQuickBrownFixture() { baseTokenization.tokens.slice(0, baseTokenCount).concat( new ContextToken(new LegacyQuotientSpur(new LegacyQuotientRoot(plainModel), newTokenInsertDistrib, newTokenInsertDistrib[0])) ), - null, - null ); const charReplaceDistrib: Distribution = [ @@ -69,9 +65,7 @@ function buildQuickBrownFixture() { const charReplaceTokenization = new ContextTokenization( baseTokenization.tokens.slice(0, baseTokenCount - 1).concat( new ContextToken(new LegacyQuotientSpur(baseTokenization.tail.searchModule, charReplaceDistrib, charReplaceDistrib[0])) - ), - null, - null + ) ); const eraseTokenDistrib: Distribution = [ @@ -80,9 +74,7 @@ function buildQuickBrownFixture() { const eraseTokenTokenization = new ContextTokenization( baseTokenization.tokens.slice(0, baseTokenCount - 1).concat( new ContextToken(new LegacyQuotientRoot(plainModel)) - ), - null, - null + ) ); const del5Insert5Distrib: Distribution = [ @@ -91,9 +83,7 @@ function buildQuickBrownFixture() { const del5Insert5Tokenization = new ContextTokenization( baseTokenization.tokens.slice(0, baseTokenCount - 3).concat( new ContextToken(new LegacyQuotientSpur(baseTokenization.tokens[baseTokenCount-2].searchModule, del5Insert5Distrib, del5Insert5Distrib[0])) - ), - null, - null + ) ); const deleteToBoundDistrib: Distribution = [ @@ -102,12 +92,13 @@ function buildQuickBrownFixture() { const deleteToBoundTokenization = new ContextTokenization( baseTokenization.tokens.slice(0, baseTokenCount - 3).concat( new ContextToken(new LegacyQuotientSpur(baseTokenization.tokens[baseTokenCount-2].searchModule, deleteToBoundDistrib, deleteToBoundDistrib[0])) - ), - null, - null + ) ); + const deleteLeftCalc = (tokens: ContextToken[]) => tokens.reduce((accum, curr) => accum + curr.codepointLength, 0); + return { + deleteLeftCalc, baseTokenization, variations: { noChange: { @@ -115,7 +106,8 @@ function buildQuickBrownFixture() { tokenization: baseTokenization, range: { tokensToRemove: [baseTokenization.tail], - tokensToPredict: [baseTokenization.tail] + tokensToPredict: [baseTokenization.tail], + deleteLeft: deleteLeftCalc([baseTokenization.tail]) } }, plainInsert: { @@ -123,7 +115,8 @@ function buildQuickBrownFixture() { tokenization: plainInsertTokenization, range: { tokensToRemove: [baseTokenization.tail], - tokensToPredict: [plainInsertTokenization.tail] + tokensToPredict: [plainInsertTokenization.tail], + deleteLeft: deleteLeftCalc([baseTokenization.tail]) } }, newTokenInsert: { @@ -131,7 +124,8 @@ function buildQuickBrownFixture() { tokenization: newTokenInsertTokenization, range: { tokensToRemove: [] as ContextToken[], - tokensToPredict: [newTokenInsertTokenization.tail] + tokensToPredict: [newTokenInsertTokenization.tail], + deleteLeft: deleteLeftCalc([]) } }, charReplace: { @@ -139,7 +133,8 @@ function buildQuickBrownFixture() { tokenization: charReplaceTokenization, range: { tokensToRemove: [baseTokenization.tail], - tokensToPredict: [charReplaceTokenization.tail] + tokensToPredict: [charReplaceTokenization.tail], + deleteLeft: deleteLeftCalc([baseTokenization.tail]) } }, eraseToken: { @@ -147,7 +142,8 @@ function buildQuickBrownFixture() { tokenization: eraseTokenTokenization, range: { tokensToRemove: [baseTokenization.tail], - tokensToPredict: [eraseTokenTokenization.tail] + tokensToPredict: [eraseTokenTokenization.tail], + deleteLeft: deleteLeftCalc([baseTokenization.tail]) } }, del5Insert5: { @@ -155,7 +151,8 @@ function buildQuickBrownFixture() { tokenization: del5Insert5Tokenization, range: { tokensToRemove: baseTokenization.tokens.slice(baseTokenCount-3), - tokensToPredict: [del5Insert5Tokenization.tail] + tokensToPredict: [del5Insert5Tokenization.tail], + deleteLeft: deleteLeftCalc(baseTokenization.tokens.slice(baseTokenCount-3)) } }, deleteToBound: { @@ -163,86 +160,96 @@ function buildQuickBrownFixture() { tokenization: deleteToBoundTokenization, range: { tokensToRemove: baseTokenization.tokens.slice(baseTokenCount-3), - tokensToPredict: [deleteToBoundTokenization.tail] + tokensToPredict: [deleteToBoundTokenization.tail], + deleteLeft: deleteLeftCalc(baseTokenization.tokens.slice(baseTokenCount-3)) } } } }; } +const tokenEquality = (a: ContextToken, b: ContextToken) => a.spaceId == b.spaceId; + describe('determineSuggestionRange', () => { it('adjusts the final token if no tokenization changes occur', () => { const fixture = buildQuickBrownFixture(); const noChange = fixture.variations.noChange; - const analysis = determineSuggestionRange(fixture.baseTokenization, noChange.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, noChange.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, noChange.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, noChange.range.tokensToPredict); + assert.equal(analysis.deleteLeft, noChange.range.deleteLeft); }); it('adjusts the final token after a simple same-token insert', () => { const fixture = buildQuickBrownFixture(); const plainInsert = fixture.variations.plainInsert; - const analysis = determineSuggestionRange(fixture.baseTokenization, plainInsert.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, plainInsert.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, plainInsert.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, plainInsert.range.tokensToPredict); + assert.equal(analysis.deleteLeft, plainInsert.range.deleteLeft); }); it('adjusts the final token after a simple word-breaking insert', () => { const fixture = buildQuickBrownFixture(); const newTokenInsert = fixture.variations.newTokenInsert; - const analysis = determineSuggestionRange(fixture.baseTokenization, newTokenInsert.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, newTokenInsert.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, newTokenInsert.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, newTokenInsert.range.tokensToPredict); + assert.equal(analysis.deleteLeft, newTokenInsert.range.deleteLeft); }); it('adjusts the final token after a simple same-token character replacement', () => { const fixture = buildQuickBrownFixture(); const charReplace = fixture.variations.charReplace; - const analysis = determineSuggestionRange(fixture.baseTokenization, charReplace.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, charReplace.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, charReplace.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, charReplace.range.tokensToPredict); + assert.equal(analysis.deleteLeft, charReplace.range.deleteLeft); }); it('handles deletion of two tokens + alteration of the token before', () => { const fixture = buildQuickBrownFixture(); const del5Insert5 = fixture.variations.del5Insert5; - const analysis = determineSuggestionRange(fixture.baseTokenization, del5Insert5.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, del5Insert5.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, del5Insert5.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, del5Insert5.range.tokensToPredict); + assert.equal(analysis.deleteLeft, del5Insert5.range.deleteLeft); }); it('handles deletion of chars up to closest whitespace', () => { const fixture = buildQuickBrownFixture(); const eraseToken = fixture.variations.eraseToken; - const analysis = determineSuggestionRange(fixture.baseTokenization, eraseToken.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, eraseToken.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, eraseToken.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, eraseToken.range.tokensToPredict); + assert.equal(analysis.deleteLeft, eraseToken.range.deleteLeft); }); it('handles deletion up to boundary of ancestor non-whitespace token', () => { const fixture = buildQuickBrownFixture(); const deleteToBound = fixture.variations.deleteToBound; - const analysis = determineSuggestionRange(fixture.baseTokenization, deleteToBound.tokenization); + const analysis = determineSuggestionRange(fixture.baseTokenization.tokens, deleteToBound.tokenization.tokens, tokenEquality); assert.sameOrderedMembers(analysis.tokensToRemove, deleteToBound.range.tokensToRemove); assert.sameOrderedMembers(analysis.tokensToPredict, deleteToBound.range.tokensToPredict); + assert.equal(analysis.deleteLeft, deleteToBound.range.deleteLeft); }); it('handles large variation in intermediate tokens', () => { - const originalQuickBrownTokenization = buildQuickBrownFixture().baseTokenization; + const { deleteLeftCalc, baseTokenization: originalQuickBrownTokenization } = buildQuickBrownFixture(); const rawText = ['beyond', ' ', 'the', ' ', 'hungry', ' ', 'green', ' ', 'alligator']; // the quick brown fox jumped | // Final whitespace is immediately before index 10. @@ -250,12 +257,10 @@ describe('determineSuggestionRange', () => { const tokensToAppend = rawText.map((t) => ContextToken.fromRawText(plainModel, t, false)); const foxVsAlligatorTokenization = new ContextTokenization( - originalQuickBrownTokenization.tokens.slice(0, transitionSliceIndex).concat(tokensToAppend), - null, - null + originalQuickBrownTokenization.tokens.slice(0, transitionSliceIndex).concat(tokensToAppend) ) - const analysis = determineSuggestionRange(originalQuickBrownTokenization, foxVsAlligatorTokenization); + const analysis = determineSuggestionRange(originalQuickBrownTokenization.tokens, foxVsAlligatorTokenization.tokens, tokenEquality); assert.sameOrderedMembers( analysis.tokensToRemove, @@ -265,21 +270,21 @@ describe('determineSuggestionRange', () => { analysis.tokensToPredict, tokensToAppend ); + + assert.equal(analysis.deleteLeft, deleteLeftCalc(originalQuickBrownTokenization.tokens.slice(transitionSliceIndex))); }); it('handles insertion of many extra new tokens at once', () => { - const originalQuickBrownTokenization = buildQuickBrownFixture().baseTokenization; + const { deleteLeftCalc, baseTokenization: originalQuickBrownTokenization } = buildQuickBrownFixture(); const originalTokenCount = originalQuickBrownTokenization.tokens.length; const rawText = ['dogs', ' ', 'and', ' ', 'the', ' ', 'sleeping', ' ', 'cat']; const tokensToAppend = rawText.map((t) => ContextToken.fromRawText(plainModel, t, false)); const dogsAndCatTokenization = new ContextTokenization( - originalQuickBrownTokenization.tokens.slice(0, originalTokenCount - 1).concat(tokensToAppend), - null, - null + originalQuickBrownTokenization.tokens.slice(0, originalTokenCount - 1).concat(tokensToAppend) ) - const analysis = determineSuggestionRange(originalQuickBrownTokenization, dogsAndCatTokenization); + const analysis = determineSuggestionRange(originalQuickBrownTokenization.tokens, dogsAndCatTokenization.tokens, tokenEquality); assert.sameOrderedMembers( analysis.tokensToRemove, @@ -289,5 +294,7 @@ describe('determineSuggestionRange', () => { analysis.tokensToPredict, tokensToAppend ); + + assert.equal(analysis.deleteLeft, deleteLeftCalc([originalQuickBrownTokenization.tail])); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts new file mode 100644 index 00000000000..02d42208cef --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts @@ -0,0 +1,406 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-05-19 + * + * This file tests the prediction helper-method responsible for preparing + * corrections for multi-token prediction for our standard models, all of which + * utilize LexiconTraversals and the context-tokenization-caching subsystem. + */ + +import { assert } from 'chai'; + +import { LexicalModelTypes } from "@keymanapp/common-types"; +import * as wordBreakers from '@keymanapp/models-wordbreakers'; +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { + determineTokenizedCorrectionSequence, + models, + ContextState, + ContextToken, + ContextTokenization, + TokenizedIntermediatePrediction, + ModelCompositor, + TokenizationResultMapping +} from "@keymanapp/lm-worker/test-index"; + +import Context = LexicalModelTypes.Context; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; +import TrieModel = models.TrieModel; + +const testModel = new TrieModel( + jsonFixture('models/tries/english-1000'), { + wordBreaker: wordBreakers.default, + } +); + +describe('determineTokenizedCorrectionSequence', () => { + it(`properly analyzes common-case token-extension - adding a letter to an existing word`, () => { + const context: Context = { + left: 'the quick brown f', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'o', + deleteLeft: 0 + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + const transition = state.analyzeTransition(context, [trueInput]); + + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, + new TokenizationResultMapping([{ + matchString: 'fo', + inputSamplingCost: -Math.log(trueInput.p), + inputCount: 2, + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }], null) + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.deepEqual(results.tokens, [ + { + correction: { + sample: { + insert: 'fo', + deleteLeft: 0 + }, + p: trueInput.p + }, + casingRoot: 'fo', + autoSelectable: true + } + ]); + }); + + it(`properly analyzes common-case whitespace - ending a token and adding a new one`, () => { + const context: Context = { + left: 'the quick brown', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + const transition = state.analyzeTransition(context, [trueInput]); + + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, + new TokenizationResultMapping([{ + matchString: ' ', + inputSamplingCost: -Math.log(trueInput.p), + inputCount: 1, + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }], null) + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.equal(results.tokens.length, 1); + assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: results.tokens[0].correction.p + }, + casingRoot: ' ', + autoSelectable: false + }]); + }); + + it(`properly analyzes common-case word-start - beginning a new token`, () => { + const context: Context = { + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + const transition = state.analyzeTransition(context, [trueInput]); + + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, + new TokenizationResultMapping([{ + matchString: 'f', + inputSamplingCost: -Math.log(trueInput.p), + inputCount: 1, + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }], null) + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + + assert.equal(results.tokens.length, 1); + assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: results.tokens[0].correction.p + }, + casingRoot: 'f', + autoSelectable: true + }]); + }); + + it(`properly analyzes post-merge case`, () => { + let context: Context = { + left: 'the quick brown fox ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 't', + deleteLeft: 0 + }, + p: .5 + }; + + const constructingState = new ContextState(context, testModel); + const tokens = constructingState.displayTokenization.tokens; + tokens.push(ContextToken.fromRawText(testModel, 'can')); + tokens.push(ContextToken.fromRawText(testModel, '\'')); + + context = models.applyTransform({insert: 'can\'', deleteLeft: 0}, context); + + const state = new ContextState(context, testModel, new ContextTokenization(tokens)); + const transition = state.analyzeTransition(context, [trueInput]); + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, + new TokenizationResultMapping([{ + matchString: 'can\'t', + inputSamplingCost: -Math.log(trueInput.p), + inputCount: 5, + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }], null) + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown fox ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: 'can\'t', + deleteLeft: 0 + }, + p: trueInput.p + }, + casingRoot: 'can\'t', + autoSelectable: true + }]); + }); + + // Will be handled far better after resolving multi-tokenization handling. + it.skip(`properly analyzes post-split new-wordbreak case`, () => { + const context: Context = { + left: 'the quick brown fox can\'', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + assert.equal(state.displayTokenization.tail.exampleInput, 'can\''); + const transition = state.analyzeTransition(context, [trueInput]); + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, + new TokenizationResultMapping([{ + matchString: ' ', + inputSamplingCost: -Math.log(trueInput.p), + inputCount: 1, + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }], null) + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the quick brown fox ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + + assert.equal(results.tokens.length, 1); + assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: results.tokens[0].correction.p + }, + casingRoot: ' ', + autoSelectable: false + }]); + }); + + it(`properly analyzes complex transition - multi-token replacement`, () => { + const context: Context = { + left: 'the quick brown f', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'fast red d', + deleteLeft: 'quick brown f'.length + }, + p: .5 + }; + + const state = new ContextState(context, testModel); + const transition = state.analyzeTransition(context, [trueInput]); + + const results = determineTokenizedCorrectionSequence( + transition, + transition.final.displayTokenization, + new TokenizationResultMapping([{ + matchString: 'd', + inputSamplingCost: -Math.log(trueInput.p), + inputCount: 1, + knownCost: 0, + totalCost: -Math.log(trueInput.p) + }], null) + ); + + assert.deepEqual({...results.rootContext, casingForm: results.rootContext.casingForm}, { + casingForm: undefined, + left: 'the ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }); + + // Coming up next - actually providing ALL correction elements, not just the final one. + // We're not _quite_ ready for that yet, though. + assert.equal(results.tokens.length, 1); + assert.deepEqual(results.tokens[0].correction.sample, { + insert: 'd', + deleteLeft: 0 + }); + assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(results.tokens, [{ + correction: { + sample: { + insert: 'd', + deleteLeft: 0 + }, + p: results.tokens[0].correction.p + }, + casingRoot: 'd', + autoSelectable: true + }]); + + const dummiedTuple: TokenizedIntermediatePrediction = { + components: [{ + prediction: { + transform: { insert: 'dog', deleteLeft: 0 }, + displayAs: 'dog' + }, + correction: 'd', + casingRoot: 'd' + }], + metadata: { + probabilities: { + prediction: .25, + correction: trueInput.p, + total: .25 * trueInput.p + }, + autoSelectable: true + } + }; + + results.applyInPost(dummiedTuple); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts new file mode 100644 index 00000000000..84a63b61d71 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts @@ -0,0 +1,456 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-05-18 + * + * This file tests the prediction helper-method responsible for preparing + * corrections for multi-token prediction for some custom and all legacy models. + */ + +import { assert } from 'chai'; + +import { LexicalModelTypes } from "@keymanapp/common-types"; +import * as wordBreakers from '@keymanapp/models-wordbreakers'; + +import { determineTraversallessCorrectionSequences, TokenizedIntermediatePrediction, ModelCompositor, models } from "@keymanapp/lm-worker/test-index"; + +import Context = LexicalModelTypes.Context; +import DummyModel = models.DummyModel; +import DummyOptions = models.DummyOptions; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; + + +/* + * This file's tests use these parts of a lexical model: + * - model.wordbreaker + * - model.toKey + * - model.applyCasing + * - model.punctuation + */ + +const DUMMY_MODEL_CONFIG: DummyOptions = { + punctuation: { + quotesForKeepSuggestion: { + open: '<', + close: '>' + }, + insertAfterWord: '\u00a0' // non-breaking space + }, + wordbreaker: wordBreakers.default +}; + +const testModel = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + // No suggestions needed here, so we don't define any. +}); + +describe('determineTraversallessCorrectionSequences', () => { + it(`processes common-case corrections correctly - on context reset with existing text`, () => { + const context = { + left: 'appl', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: '', + deleteLeft: 0 + }, + p: 1 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: 'appl', + deleteLeft: 0 + }, + p: trueInput.p + }, + casingRoot: 'appl', + autoSelectable: true + }]); + }); + + it(`processes standard-case corrections correctly - text appended to existing token`, () => { + const context: Context = { + left: 'I want an iPhon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e', + deleteLeft: 0 + }, + p: 1 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'I want an ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(predictionRootEntries[0].tokens, [ + { + correction: { + sample: { + insert: 'iPhone', + deleteLeft: 0 + }, + p: trueInput.p + }, + casingRoot: 'iPhone', + autoSelectable: true + } + ]); + }); + + it(`properly analyzes common-case token-extension - adding a letter to an existing word`, () => { + const context: Context = { + left: 'the quick brown f', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'o', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokens, [ + { + correction: { + sample: { + insert: 'fo', + deleteLeft: 0 + }, + p: trueInput.p + }, + casingRoot: 'fo', + autoSelectable: true + } + ]); + }); + + it(`properly analyzes common-case whitespace - ending a token and adding a new one`, () => { + const context: Context = { + left: 'the quick brown', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the quick brown', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.equal(entry.tokens.length, 1); + assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: '', + deleteLeft: 0 + }, + p: entry.tokens[0].correction.p + }, + casingRoot: '', + autoSelectable: false + }]); + }); + + + it(`properly analyzes common-case word-start - beginning a new token`, () => { + const context: Context = { + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the quick brown ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.equal(entry.tokens.length, 1); + assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: 'f', + deleteLeft: 0 + }, + p: entry.tokens[0].correction.p + }, + casingRoot: 'f', + autoSelectable: true + }]); + }); + + it(`properly analyzes post-merge case`, () => { + let context: Context = { + left: 'the quick brown fox can\'', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 't', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the quick brown fox ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: 'can\'t', + deleteLeft: 0 + }, + p: trueInput.p + }, + casingRoot: 'can\'t', + autoSelectable: true + }]); + }); + + it(`properly analyzes post-split new-wordbreak case`, () => { + const context: Context = { + left: 'the quick brown fox can\'', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: ' ', + deleteLeft: 0 + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + // assert.deepEqual( + // { + // ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + // }, { + // casingForm: undefined, + // // Proper logic requires full multi-token awareness; predictions are currently + // // based on just the last token. + // left: 'the quick brown fox can\'', + // right: '', + // startOfBuffer: true, + // endOfBuffer: true + // } + // ); + + assert.equal(entry.tokens.length, 1); + assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: '', + deleteLeft: 0 + }, + p: entry.tokens[0].correction.p + }, + casingRoot: '', + autoSelectable: false + }]); + }); + + it(`properly analyzes complex transition - multi-token replacement`, () => { + const context: Context = { + left: 'the quick brown f', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'fast red d', + deleteLeft: 'quick brown f'.length + }, + p: .5 + }; + + const predictionRootEntries = determineTraversallessCorrectionSequences(testModel, [trueInput], context); + assert.equal(predictionRootEntries.length, 1); + const entry = predictionRootEntries[0]; + + assert.deepEqual( + { + ...entry.rootContext, casingForm: entry.rootContext.casingForm ?? undefined + }, { + casingForm: undefined, + left: 'the ', + right: '', + startOfBuffer: true, + endOfBuffer: true + } + ); + + // Coming up next - actually providing ALL correction elements, not just the final one. + // We're not _quite_ ready for that yet, though. + assert.equal(entry.tokens.length, 1); + assert.deepEqual(entry.tokens[0].correction.sample, { + insert: 'd', + deleteLeft: 0 + }); + assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000); + + assert.deepEqual(entry.tokens, [{ + correction: { + sample: { + insert: 'd', + deleteLeft: 0 + }, + p: entry.tokens[0].correction.p + }, + casingRoot: 'd', + autoSelectable: true + }]); + + const dummiedTuple: TokenizedIntermediatePrediction = { + components: [{ + prediction: { + transform: { insert: 'dog', deleteLeft: 0 }, + displayAs: 'dog' + }, + correction: 'd', + casingRoot: 'd' + }], + metadata: { + probabilities: { + prediction: .25, + correction: trueInput.p, + total: .25 * trueInput.p + }, + autoSelectable: true + } + }; + + entry.applyInPost(dummiedTuple); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts new file mode 100644 index 00000000000..6b755be5c09 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -0,0 +1,742 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-04-13 + * + * This file unit tests against the `predictFromCorrectionSequence` + * prediction-helper function, validating construction of predictions based on + * their root correction sequences. + */ + +import { assert } from 'chai'; + +import { deepCopy } from "keyman/common/web-utils"; +import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { EDIT_DISTANCE_COST_SCALE, PredictionParameters, models, predictFromCorrectionSequence, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; + +import CasingFunction = LexicalModelTypes.CasingFunction; +import DummyModel = models.DummyModel; +import Outcome = LexicalModelTypes.Outcome; +import Suggestion = LexicalModelTypes.Suggestion; + +// See: developer/src/kmc-model/model-defaults.ts, defaultApplyCasing +const applyCasing: CasingFunction = (casing, text) => { + switch(casing) { + case 'lower': + return text.toLowerCase(); + case 'upper': + return text.toUpperCase(); + case 'initial': + var headCode = text.charCodeAt(0); + // The length of the first code unit, as measured in code points. + var headUnitLength = 1; + + // Is the first character a high surrogate, indicating possible use of UTF-16 + // surrogate pairs? Also, is the string long enough for there to BE a pair? + if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) { + // It's possible, so now we check for low surrogates. + var lowSurrogateCode = text.charCodeAt(1); + + if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) { + // We have a surrogate pair; this pair is the 'first' character. + headUnitLength++; + } + } + + // Capitalizes the first code unit of the string, leaving the rest intact. + return text.substring(0, headUnitLength).toUpperCase() // head - uppercased + .concat(text.substring(headUnitLength)); // tail - unchanged + } +}; + +/** @type { import("#./models/dummy-model.js").DummyOptions } */ +const DUMMY_MODEL_CONFIG = { + punctuation: { + quotesForKeepSuggestion: { + open: '<', + close: '>' + }, + insertAfterWord: '\u00a0' // non-breaking space + }, + applyCasing: applyCasing, + searchTermToKey: (wordform: string) => { + // See: developer/src/kmc-model/model-defaults.ts, defaultCasedSearchTermToKey + return applyCasing('lower', wordform) + .normalize('NFKD') + // Remove any combining diacritics (if input is in NFKD) + .replace(/[\u0300-\u036F]/g, '') + // Replace directional quotation marks with plain apostrophes + .replace(/[‘’]/g, "'") + // Also double-quote marks. + .replace(/[“”]/g, '"') + // ** Difference from model-defaults here ** + // And finally, erase single-quotation marks. + .replace(/'/, ''); + }, + languageUsesCasing: true +}; + +describe('predictFromCorrectionSequence', () => { + describe('on a single correction', () => { + it('constructs suggestions matching multiple lexical entries directly - no transform ID', () => { + const transitionID = 12345; + + const parameters: PredictionParameters = { + rootContext: { + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'Its', + deleteLeft: 0, + id: transitionID + }, + p: 0.6 + }, + casingRoot: '', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; + + const dummied_suggestions: Outcome[] = [ + { + transform: { + insert: "it's", + deleteLeft: 2 + }, + displayAs: "it's", + p: 0.18 + }, { + transform: { + insert: "its", + deleteLeft: 2 + }, + displayAs: "its", + p: 0.02 + } + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: [ dummied_suggestions ] + }); + + const predictions = predictFromCorrectionSequence(model, parameters); + predictions.forEach((entry) => assert.equal(entry.components[0].correction, 'Its')); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 0.6)); + predictions.sort(tupleDisplayOrderSort); + + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((s) => { + delete s.p; + s.transform.id = transitionID; + return s; + })); + + assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001); + assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001); + }); + + it('constructs suggestions matching multiple lexical entries directly - with transform ID', () => { + const transitionID = 314159; + + const parameters: PredictionParameters = { + rootContext: { + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'Its', + deleteLeft: 0, + id: transitionID + }, + p: 0.6 + }, + casingRoot: '', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; + + const dummied_suggestions: Outcome[] = [ + { + transform: { + insert: "it's", + deleteLeft: 2 + }, + displayAs: "it's", + p: 0.18 + }, { + transform: { + insert: "its", + deleteLeft: 2 + }, + displayAs: "its", + p: 0.02 + } + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: [ dummied_suggestions ] + }); + + const predictions = predictFromCorrectionSequence(model, parameters); + predictions.forEach((entry) => assert.equal(entry.components[0].correction, 'Its')); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 0.6)); + predictions.sort(tupleDisplayOrderSort); + + assert.sameOrderedMembers(predictions.map((entry) => entry.components[0].prediction.displayAs), ["it's", "its"]); + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((entry) => { + entry = deepCopy(entry); + entry.transform.id = transitionID; + return entry; + })); + + assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001); + assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001); + predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transform.id, transitionID)); + }); + + it('constructs suggestions without input (as if after a context reset)', () => { + const transitionID = 271828; + + const parameters: PredictionParameters = { + rootContext: { + left: '', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'appl', + deleteLeft: 0, + id: transitionID + }, + p: 1 + }, + casingRoot: 'appl', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; + + const dummied_suggestions: Outcome[] = [ + { + transform: { + insert: "apple", + deleteLeft: 4 + }, + displayAs: "apple", + p: 0.5 + } + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: [ dummied_suggestions ] + }); + + const predictions = predictFromCorrectionSequence(model, parameters); + predictions.forEach((entry) => assert.deepEqual(entry.components.map((c => c.correction)), ['appl'])); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 1)); + predictions.sort(tupleDisplayOrderSort); + + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), [dummied_suggestions.map((s) => { + delete s.p; + s.transform.id = transitionID; + return s; + })]); + }); + }); + + describe('on a sequence of corrections', () => { + it('returns results even if some correction tokens lack predictions', () => { + const transitionID = 101; + + const parameters: PredictionParameters = { + rootContext: { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'g', + deleteLeft: 0, + id: transitionID + }, + p: 0.1 + }, + casingRoot: 'g', + autoSelectable: true + }, { + correction: { + sample: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: ' ', + autoSelectable: true + }, { + correction: { + sample: { + insert: 'apple', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: 'apple', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; + + const dummied_suggestion_sequences: Outcome[][] = [ + [ + { + transform: { + insert: "g", + deleteLeft: 0 + }, + displayAs: "g", + p: 0.1 + } + ], + [], + [ + { + transform: { + insert: "apple", + deleteLeft: 0 + }, + displayAs: "apple", + p: 0.5 + } + ] + ]; + + const expected_predictions: Suggestion[] = [ + { + transform: { + insert: 'g', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'g' + }, { + transform: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + displayAs: ' ' + }, { + transform: { + insert: 'apple', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'apple' + } + ]; + + const expected_prediction_p = dummied_suggestion_sequences.map((dist) => { + return dist[0] + }).reduce((accum, curr) => { + return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) + }, 1); + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: dummied_suggestion_sequences + }); + + const predictions = predictFromCorrectionSequence(model, parameters); + predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['g', ' ', 'apple'])); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1))); + predictions.sort(tupleDisplayOrderSort); + + assert.sameDeepOrderedMembers(predictions[0].components.map((c) => c.prediction), expected_predictions); + + assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction_p, 0.00001); + }); + + it('returns no results if all correction tokens lack predictions', () => { + const transitionID = 3; + + const parameters: PredictionParameters = { + rootContext: { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'golden', + deleteLeft: 0, + id: transitionID + }, + p: 0.1 + }, + casingRoot: 'golden', + autoSelectable: true + }, { + correction: { + sample: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: ' ', + autoSelectable: true + }, { + correction: { + sample: { + insert: 'app', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: 'app', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; + + const dummied_suggestion_sequences: Outcome[][] = [ + [], + [], + [] + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: dummied_suggestion_sequences + }); + + const predictions = predictFromCorrectionSequence(model, parameters); + assert.deepEqual(predictions, []); + }); + + it('uses only the best suggestion for non-final corrected tokens', () => { + const transitionID = 42; + + const parameters: PredictionParameters = { + rootContext: { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'g', + deleteLeft: 0, + id: transitionID + }, + p: 0.1 + }, + casingRoot: 'g', + autoSelectable: true + }, { + correction: { + sample: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: ' ', + autoSelectable: true + }, { + correction: { + sample: { + insert: 'app', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: 'app', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; + + const dummied_suggestion_sequences: Outcome[][] = [ + [ + { + transform: { + insert: "golden", + deleteLeft: 0 + }, + displayAs: "golden", + p: 0.2 + }, { + transform: { + insert: "green", + deleteLeft: 0 + }, + displayAs: "green", + p: 0.15 + }, { + transform: { + insert: "gray", + deleteLeft: 0 + }, + displayAs: "gray", + p: 0.1 + } + ], + [], + [ + { + transform: { + insert: "apple", + deleteLeft: 0 + }, + displayAs: "apple", + p: 0.5 + } + ] + ]; + + const expected_prediction_p = dummied_suggestion_sequences + .map((dist, i) => { + // There is no valid 'g' entry corresponding to token index 0. + return i == 0 ? null : dist[0] + }).reduce((accum, curr) => { + return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) + }, 1); + + const expected_predictions: Suggestion[] = [ + { + transform: { + insert: 'g', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'g' + }, { + transform: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + displayAs: ' ' + }, { + transform: { + insert: 'apple', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'apple' + } + ]; + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: dummied_suggestion_sequences + }); + + const predictions = predictFromCorrectionSequence(model, parameters); + // There should be no variations with 'green' or 'gray' apples. + assert.equal(predictions.length, 1); + + predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['g', ' ', 'app'])); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1))); + predictions.sort(tupleDisplayOrderSort); + + assert.deepEqual(predictions[0].components.map((c) => c.prediction.transform.insert), ['g', ' ', 'apple']); + assert.sameDeepOrderedMembers(predictions[0].components.map((entry) => entry.prediction), expected_predictions); + + assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction_p, 0.00001); + }); + + it('uses all suggestions generated from context-final correction-tokens', () => { + const transitionID = 13; + + const parameters: PredictionParameters = { + rootContext: { + left: 'i want to eat a ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }, + tokens: [ + { + correction: { + sample: { + insert: 'golden', + deleteLeft: 0, + id: transitionID + }, + p: 0.1 + }, + casingRoot: 'golden', + autoSelectable: true + }, { + correction: { + sample: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: ' ', + autoSelectable: true + }, { + correction: { + sample: { + insert: 'app', + deleteLeft: 0, + id: transitionID + }, + p: 0.2 + }, + casingRoot: 'app', + autoSelectable: true + } + ], + applyInPost: (x) => x, + deleteLeft: 0 + }; + + const dummied_suggestion_sequences: Outcome[][] = [ + [ + { + transform: { + insert: "golden", + deleteLeft: 0 + }, + displayAs: "golden", + p: 0.1 + } + ], + [], + [ + { + transform: { + insert: "apple", + deleteLeft: 0 + }, + displayAs: "apple", + p: 0.5 + }, { + transform: { + insert: "application", + deleteLeft: 0 + }, + displayAs: "application", + p: 0.11 + }, { + transform: { + insert: "appetizer", + deleteLeft: 0 + }, + displayAs: "appetizer", + p: 0.1 + } + ] + ]; + + const tailIndex = dummied_suggestion_sequences.length - 1; + + const expected_prediction_prefix_p = dummied_suggestion_sequences + .slice(0, dummied_suggestion_sequences.length - 1) + .map((dist) => { + return dist[0] + }).reduce((accum, curr) => { + return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE)) + }, 1); + + const expected_prediction_prefix: Suggestion[] = [ + { + transform: { + insert: 'golden', + deleteLeft: 0, + id: transitionID + }, + displayAs: 'golden' + }, { + transform: { + insert: ' ', + deleteLeft: 0, + id: transitionID + }, + displayAs: ' ' + } + ]; + + const expected_prediction_sequences: Suggestion[][] = dummied_suggestion_sequences[tailIndex].map((p) => { + return [...expected_prediction_prefix, p]; + }); + + const expected_prediction_seq_probs: number[] = dummied_suggestion_sequences[tailIndex].map((p) => { + return p.p * expected_prediction_prefix_p; + }); + + const model = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + futureSuggestions: dummied_suggestion_sequences + }); + + const predictions = predictFromCorrectionSequence(model, parameters); + assert.equal(predictions.length, dummied_suggestion_sequences[dummied_suggestion_sequences.length - 1].length); + + predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['golden', ' ', 'app'])); + predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1))); + predictions.sort(tupleDisplayOrderSort); + + assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), expected_prediction_sequences); + + for(let i = 0; i < predictions.length; i++) { + assert.approximately(predictions[i].metadata.probabilities.prediction, expected_prediction_seq_probs[i], 0.00001, `Expected probabilty mismatch at index ${i}`); + } + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts deleted file mode 100644 index 31f5063c15c..00000000000 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts +++ /dev/null @@ -1,260 +0,0 @@ - -import { assert } from 'chai'; - -import { deepCopy } from "keyman/common/web-utils"; -import { LexicalModelTypes } from '@keymanapp/common-types'; - -import { models, predictFromCorrections, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index"; - -import CasingFunction = LexicalModelTypes.CasingFunction; -import Context = LexicalModelTypes.Context; -import Distribution = LexicalModelTypes.Distribution; -import DummyModel = models.DummyModel; -import Outcome = LexicalModelTypes.Outcome; -import Suggestion = LexicalModelTypes.Suggestion; -import Transform = LexicalModelTypes.Transform; - -// See: developer/src/kmc-model/model-defaults.ts, defaultApplyCasing -const applyCasing: CasingFunction = (casing, text) => { - switch(casing) { - case 'lower': - return text.toLowerCase(); - case 'upper': - return text.toUpperCase(); - case 'initial': - var headCode = text.charCodeAt(0); - // The length of the first code unit, as measured in code points. - var headUnitLength = 1; - - // Is the first character a high surrogate, indicating possible use of UTF-16 - // surrogate pairs? Also, is the string long enough for there to BE a pair? - if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) { - // It's possible, so now we check for low surrogates. - var lowSurrogateCode = text.charCodeAt(1); - - if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) { - // We have a surrogate pair; this pair is the 'first' character. - headUnitLength++; - } - } - - // Capitalizes the first code unit of the string, leaving the rest intact. - return text.substring(0, headUnitLength).toUpperCase() // head - uppercased - .concat(text.substring(headUnitLength)); // tail - lowercased - } -}; - -/** @type { import("#./models/dummy-model.js").DummyOptions } */ -const DUMMY_MODEL_CONFIG = { - punctuation: { - quotesForKeepSuggestion: { - open: '<', - close: '>' - }, - insertAfterWord: '\u00a0' // non-breaking space - }, - applyCasing: applyCasing, - searchTermToKey: (wordform: string) => { - // See: developer/src/kmc-model/model-defaults.ts, defaultCasedSearchTermToKey - return applyCasing('lower', wordform) - .normalize('NFKD') - // Remove any combining diacritics (if input is in NFKD) - .replace(/[\u0300-\u036F]/g, '') - // Replace directional quotation marks with plain apostrophes - .replace(/[‘’]/g, "'") - // Also double-quote marks. - .replace(/[“”]/g, '"') - // ** Difference from model-defaults here ** - // And finally, erase single-quotation marks. - .replace(/'/, ''); - }, - languageUsesCasing: true -}; - -describe('predictFromCorrections', () => { - it('handles a single correction prefixing multiple entries - no transform ID', () => { - const context: Context = { - left: 'It', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const correctionDistribution: Distribution = [{ - sample: { - insert: 's', - deleteLeft: 0 - }, - p: 0.6 - } - ]; - - const dummied_suggestions: Outcome[] = [ - { - transform: { - insert: "it's", - deleteLeft: 2 - }, - displayAs: "it's", - p: 0.18 - }, { - transform: { - insert: "its", - deleteLeft: 2 - }, - displayAs: "its", - p: 0.02 - } - ]; - - const model = new DummyModel({ - ...DUMMY_MODEL_CONFIG, - futureSuggestions: [ dummied_suggestions ] - }); - - const predictions = predictFromCorrections(model, correctionDistribution, context); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); - predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); - predictions.sort(tupleDisplayOrderSort); - - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions); - - assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); - assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); - }); - - it('handles a single correction prefixing multiple entries - with transform ID', () => { - const context: Context = { - left: 'It', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const correctionDistribution: Distribution = [{ - sample: { - insert: 's', - deleteLeft: 0, - id: 314159 - }, - p: 0.6 - } - ]; - - const dummied_suggestions: Outcome[] = [ - { - transform: { - insert: "it's", - deleteLeft: 2 - }, - displayAs: "it's", - p: 0.18 - }, { - transform: { - insert: "its", - deleteLeft: 2 - }, - displayAs: "its", - p: 0.02 - } - ]; - - const model = new DummyModel({ - ...DUMMY_MODEL_CONFIG, - futureSuggestions: [ dummied_suggestions ] - }); - - const predictions = predictFromCorrections(model, correctionDistribution, context); - predictions.forEach((entry) => assert.equal(entry.correction.sample, 'Its')); - predictions.forEach((entry) => assert.equal(entry.correction.p, 0.6)); - predictions.sort(tupleDisplayOrderSort); - - assert.sameOrderedMembers(predictions.map((entry) => entry.prediction.sample.displayAs), ["it's", "its"]); - assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((entry) => { - entry = deepCopy(entry); - return entry; - })); - - assert.approximately(predictions[0].totalProb, 0.18 * 0.6, 0.00001); - assert.approximately(predictions[1].totalProb, 0.02 * 0.6, 0.00001); - }); - - it('handles multiple corrections at once', () => { - const context: Context = { - left: 'It', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - // Note: each correction is used in order in a separate model.predict call. - /** @type {Distribution} */ - const correctionDistribution: Distribution = [{ - // postContext: is - sample: { - insert: 's', - deleteLeft: 1 - }, - p: 0.4 - }, { - // postContext: its - sample: { - insert: 's', - deleteLeft: 0 - }, - p: 0.6 - } - ]; - - const dummied_suggestions: Outcome[][] = [ - // postContext: is - [{ - transform: { - insert: "is", - deleteLeft: 2 - }, - displayAs: "is", - p: 0.4 - }, { - transform: { - insert: "isn't", - deleteLeft: 2 - }, - displayAs: "isn't", - p: 0.2 - }], - // postContext: its - [{ - transform: { - insert: "it's", - deleteLeft: 2 - }, - displayAs: "it's", - p: 0.18 - }, { - transform: { - insert: "its", - deleteLeft: 2 - }, - displayAs: "its", - p: 0.02 - }] - ]; - - const model = new DummyModel({ - ...DUMMY_MODEL_CONFIG, - futureSuggestions: dummied_suggestions - }); - - const predictions = predictFromCorrections(model, correctionDistribution, context); - predictions.sort(tupleDisplayOrderSort); - - assert.sameOrderedMembers(predictions.map((entry) => entry.prediction.sample.displayAs), ["is", "it's", "isn't", "its"]); - assert.sameDeepMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.flatMap((entry) => entry)); - - assert.approximately(predictions[0].totalProb, 0.4 * 0.4, 0.00001); - assert.approximately(predictions[1].totalProb, 0.18 * 0.6, 0.00001); - assert.approximately(predictions[2].totalProb, 0.4 * 0.2, 0.00001); - assert.approximately(predictions[3].totalProb, 0.02 * 0.6, 0.00001); - }); -}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts new file mode 100644 index 00000000000..945785ada00 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepare-tokenization-search.tests.ts @@ -0,0 +1,329 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-07-21 + * + * This file unit tests against the `prepareTokenizationSearch` + * prediction-helper function, which uses results from + * `determineSuggestionRange` to build phrase-level correctors for multi-token + * correction. + */ + +import { assert } from 'chai'; + +import { LexicalModelTypes } from '@keymanapp/common-types'; +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { ContextState, ContextTokenization, ContextTransition, models, prepareTokenizationSearch, ContextToken, LegacyQuotientRoot, LegacyQuotientSpur } from "@keymanapp/lm-worker/test-index"; + +import Context = LexicalModelTypes.Context; +import Distribution = LexicalModelTypes.Distribution; +import Transform = LexicalModelTypes.Transform; +import TrieModel = models.TrieModel; + +const testModel = new TrieModel(jsonFixture('models/tries/english-1000')); + +describe('prepareTokenizationSearch', () => { + it('handles simple-case, single tokenization transitions well', () => { + const baseContext: Context = { + left: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'a', + startOfBuffer: true, + endOfBuffer: true + }; + + const targetTokenization = new ContextTokenization([ContextToken.fromRawText(testModel, 'a')]); + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization]); + + transition.finalize(nextState, [ + { sample: { insert: 'a', deleteLeft: 0, id: 1}, p: 1 } + ]); + + const correctorsForSearch = prepareTokenizationSearch(transition, [targetTokenization]); + assert.equal(correctorsForSearch.length, 1); + + const corrector = correctorsForSearch[0]; + assert.deepEqual(corrector.orderedTokens, targetTokenization.tokens); + assert.deepEqual(corrector.correctableTokens, []); + assert.deepEqual(corrector.uncorrectableTokens, []); + assert.deepEqual(corrector.predictableToken, targetTokenization.tail); + + assert.equal(corrector.correctableCodepoints, 1); + assert.isTrue(corrector.modelsCorrectables) + assert.equal(corrector.tokenization, targetTokenization); + }); + + it('handles single-tokenization transitions from whitespace well', () => { + const baseContext: Context = { + left: 'space', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'space ', + startOfBuffer: true, + endOfBuffer: true + }; + + const targetTokenization = new ContextTokenization([ + transition.base.displayTokenization.tail, + ContextToken.fromRawText(testModel, ' '), + ContextToken.fromRawText(testModel, '', true) + ]); + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization]); + + transition.finalize(nextState, [ + { sample: { insert: ' ', deleteLeft: 0, id: 1}, p: 1 } + ]); + + const correctorsForSearch = prepareTokenizationSearch(transition, [targetTokenization]); + assert.equal(correctorsForSearch.length, 1); + + const corrector = correctorsForSearch[0]; + assert.deepEqual(corrector.orderedTokens, targetTokenization.tokens.slice(1)); + assert.deepEqual(corrector.correctableTokens, []); + assert.deepEqual(corrector.uncorrectableTokens, [targetTokenization.tokens[1]]); + assert.deepEqual(corrector.predictableToken, targetTokenization.tail); + + assert.equal(corrector.correctableCodepoints, 0); + assert.isTrue(corrector.modelsCorrectables) + assert.equal(corrector.tokenization, targetTokenization); + }); + + it('handles divergent transitions with possible wordbreaks', () => { + const baseContext: Context = { + left: 'space', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'spaced', + startOfBuffer: true, + endOfBuffer: true + }; + + const distribution: Distribution = [ + { sample: { insert: 'd', deleteLeft: 0, id: 1}, p: .8 }, + { sample: { insert: ' ', deleteLeft: 0, id: 1}, p: .2 } + ]; + + + const targetTokenization1 = new ContextTokenization([ + new ContextToken(new LegacyQuotientSpur(transition.base.displayTokenization.tail.searchModule, [distribution[0]], distribution[0])) + ]); + + const targetTokenization2 = new ContextTokenization([ + transition.base.displayTokenization.tail, + new ContextToken(new LegacyQuotientSpur(new LegacyQuotientRoot(testModel), [distribution[1]], distribution[0])), + ContextToken.fromRawText(testModel, '', true) + ]); + + const tokenizations = [targetTokenization1, targetTokenization2]; + + const nextState = new ContextState(targetContext, testModel, targetTokenization1, tokenizations); + + transition.finalize(nextState, distribution); + + const correctorsForSearch = prepareTokenizationSearch(transition, tokenizations); + assert.equal(correctorsForSearch.length, 2); + + const corrector1 = correctorsForSearch[0]; + assert.deepEqual(corrector1.orderedTokens, targetTokenization1.tokens); + assert.deepEqual(corrector1.correctableTokens, []); + assert.deepEqual(corrector1.uncorrectableTokens, []); + assert.deepEqual(corrector1.predictableToken, targetTokenization1.tail); + + assert.equal(corrector1.correctableCodepoints, 6); + assert.isTrue(corrector1.modelsCorrectables) + assert.equal(corrector1.tokenization, targetTokenization1); + + const corrector2 = correctorsForSearch[1]; + assert.deepEqual(corrector2.orderedTokens, targetTokenization2.tokens); + assert.deepEqual(corrector2.correctableTokens, []); + assert.deepEqual(corrector2.uncorrectableTokens, targetTokenization2.tokens.slice(0, 2)); + assert.deepEqual(corrector2.predictableToken, targetTokenization2.tail); + + assert.equal(corrector2.correctableCodepoints, 0); + assert.isTrue(corrector2.modelsCorrectables) + assert.equal(corrector2.tokenization, targetTokenization2); + }); + + it('handles simple-case, backspace tokenization transitions well', () => { + const baseContext: Context = { + left: 'apples', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'apple', + startOfBuffer: true, + endOfBuffer: true + }; + + const targetTokenization = new ContextTokenization([ContextToken.fromRawText(testModel, 'apple')]); + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization]); + + transition.finalize(nextState, [ + { sample: { insert: '', deleteLeft: 1, id: 1}, p: 1 } + ]); + + const correctorsForSearch = prepareTokenizationSearch(transition, [targetTokenization]); + assert.equal(correctorsForSearch.length, 1); + + const corrector = correctorsForSearch[0]; + assert.deepEqual(corrector.orderedTokens, targetTokenization.tokens); + assert.deepEqual(corrector.correctableTokens, []); + assert.deepEqual(corrector.uncorrectableTokens, []); + assert.deepEqual(corrector.predictableToken, targetTokenization.tail); + + assert.equal(corrector.correctableCodepoints, 5); + assert.isTrue(corrector.modelsCorrectables) + assert.equal(corrector.tokenization, targetTokenization); + }); + + it('handles whitespace-token deletion transitions well', () => { + const baseContext: Context = { + left: 'apples ', + startOfBuffer: true, + endOfBuffer: true + }; + + const transition = new ContextTransition(new ContextState(baseContext, testModel), 0); + + const targetContext: Context = { + left: 'apples', + startOfBuffer: true, + endOfBuffer: true + }; + + const targetTokenization = new ContextTokenization([transition.base.displayTokenization.tokens[0]]); + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization]); + + transition.finalize(nextState, [ + { sample: { insert: '', deleteLeft: 1, id: 1}, p: 1 } + ]); + + const correctorsForSearch = prepareTokenizationSearch(transition, [targetTokenization]); + assert.equal(correctorsForSearch.length, 1); + + const corrector = correctorsForSearch[0]; + assert.deepEqual(corrector.orderedTokens, targetTokenization.tokens); + assert.deepEqual(corrector.correctableTokens, []); + assert.deepEqual(corrector.uncorrectableTokens, []); + assert.deepEqual(corrector.predictableToken, targetTokenization.tail); + + assert.equal(corrector.correctableCodepoints, 6); + assert.isTrue(corrector.modelsCorrectables) + assert.equal(corrector.tokenization, targetTokenization); + }); + + // TODO: a more complex transition set. + it('handles complicated dictionary-style wordbreaking transitions', () => { + const baseContext: Context = { + left: 'myapplesandsour', + startOfBuffer: true, + endOfBuffer: true + }; + + // To be safe, make sure the spaces are constructed in proper left-to-right order, + // no matter the variation. + const my = ContextToken.fromRawText(testModel, 'my') + const apples = ContextToken.fromRawText(testModel, 'apples'); + const apple = ContextToken.fromRawText(testModel, 'apple'); + + const and = ContextToken.fromRawText(testModel, 'and'); + const sand = ContextToken.fromRawText(testModel, 'sand'); + const sands = ContextToken.fromRawText(testModel, 'sands'); + + const sour = ContextToken.fromRawText(testModel, 'sour'); + const our = ContextToken.fromRawText(testModel, 'our'); + + const baseTokenization = new ContextTokenization(([my, apples, and, sour])); + + const baseTokenizationVariants: ContextTokenization[] = [ + new ContextTokenization(([my, apple, sand, sour])), + new ContextTokenization(([my, apple, sands, our])) + ] + const transition = new ContextTransition(new ContextState(baseContext, testModel, baseTokenization, baseTokenizationVariants), 0); + + // ---- + const targetContext: Context = { + left: 'myapplesandsourg', + startOfBuffer: true, + endOfBuffer: true + }; + + const distribution: Distribution = [ + { sample: { insert: 'g', deleteLeft: 0, id: 1}, p: 1 } + ]; + + const g = () => new ContextToken(new LegacyQuotientSpur(new LegacyQuotientRoot(testModel), distribution, distribution[0])); + const ourg = () => new ContextToken(new LegacyQuotientSpur(our.searchModule, distribution, distribution[0])); + const sourg = () => new ContextToken(new LegacyQuotientSpur(sour.searchModule, distribution, distribution[0])); + + const targetTokenization = new ContextTokenization([my, apples, and, sour, g()]); + + const targetTokenizationVariants: ContextTokenization[] = [ + new ContextTokenization([my, apples, and, sourg()]), + new ContextTokenization([my, apple, sand, sour, g()]), + new ContextTokenization([my, apple, sand, sourg()]), + new ContextTokenization([my, apple, sands, our, g()]), + new ContextTokenization([my, apple, sands, ourg()]) + ]; + + const nextState = new ContextState(targetContext, testModel, targetTokenization, [targetTokenization, ...targetTokenizationVariants]); + transition.finalize(nextState, distribution); + + const variationStartIndex: Map = new Map(); + for(let variant of nextState.tokenizations) { + let i = 0; + while(variant.tokens[i]?.exampleInput == baseTokenization.tokens[i]?.exampleInput) { + i++; + } + variationStartIndex.set(variant, i); + } + + const correctorsForSearch = prepareTokenizationSearch(transition, nextState.tokenizations, { + rangeValidator: (index, rs) => index >= rs + }); + assert.equal(correctorsForSearch.length, nextState.tokenizations.length); + + correctorsForSearch.forEach((corrector, index) => { + const tokenization = nextState.tokenizations.find((t) => corrector.tokenization == t); + assert.isOk(tokenization); + + assert.deepEqual(corrector.orderedTokens, tokenization.tokens.slice(1), `Error for variant's ordered tokens at index ${index}`); + const correctables = tokenization.tokens.slice(variationStartIndex.get(tokenization), -1) + assert.deepEqual(corrector.correctableTokens, correctables, `Error for variant's correctable tokens at index ${index}`); + assert.deepEqual(corrector.uncorrectableTokens, tokenization.tokens.slice(1, variationStartIndex.get(tokenization)), `Error for variant's uncorrectable tokens at index ${index}`); + assert.deepEqual(corrector.predictableToken, tokenization.tail, `Error for variant's predictable token at index ${index}`); + + assert.equal( + corrector.correctableCodepoints, + correctables.reduce((accum, curr) => accum + curr.codepointLength, 0) + tokenization.tail.codepointLength, + `Error for variant's correctable-codepoint count at index ${index}` + ); + assert.isTrue(corrector.modelsCorrectables, `Error for variant's 'models correctables' flag at index ${index}`); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts index bfd2eeefafc..d28d7a6343b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts @@ -4,7 +4,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { deepCopy } from 'keyman/common/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { CorrectionPredictionTuple, dedupeSuggestions, models } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, dedupeSuggestions, models } from "@keymanapp/lm-worker/test-index"; import Context = LexicalModelTypes.Context; import DummyModel = models.DummyModel; @@ -24,77 +24,89 @@ const testModel = new DummyModel({ * @returns */ const build_its_is_set = () => { - const its: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const its: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 0 }, displayAs: 'its' }, - p: 0.2 + correction: 'its' }, - totalProb: 0.16 - // matchLevel does not yet exist. + metadata: { + probabilities: { + prediction: .2, + correction: .8, + total: .2 * .8 + }, + autoSelectable: true + // matchLevel does not yet exist. + } }; - const it_is: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const it_is: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: '\'s', deleteLeft: 0 }, displayAs: 'it\'s' }, - p: 0.8 + correction: 'its' }, - totalProb: 0.64 + metadata: { + probabilities: { + prediction: .8, + correction: .8, + total: .8 * .8 + }, + autoSelectable: true + } }; - const is: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 1 }, displayAs: 'is' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; - const is_not: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is_not: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 'sn\'t', deleteLeft: 1 }, displayAs: 'isn\'t' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; return { @@ -145,7 +157,7 @@ describe('dedupeSuggestions', () => { // There's no mathematically safe way to combine the components if the // underlying correction sources differ between duplicated suggestions, // though it's mathematically safe to combine their product. - expected.forEach((entry) => entry.totalProb *= (entry.prediction.sample.transform.insert == '\'s') ? 3 : 2); + expected.forEach((entry) => entry.metadata.probabilities.total *= (entry.components.prediction.transform.insert == '\'s') ? 3 : 2); assert.deepEqual(deduplicated, expected); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts index 387bf6dab93..2bf9a6f96a4 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts @@ -5,7 +5,7 @@ import { deepCopy } from 'keyman/common/web-utils'; import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { CorrectionPredictionTuple, finalizeSuggestions, models } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, finalizeSuggestions, models } from "@keymanapp/lm-worker/test-index"; import DummyModel = models.DummyModel; import Outcome = LexicalModelTypes.Outcome; @@ -39,6 +39,7 @@ const testModelWithoutSpacing = new DummyModel({ } }); + /** * Builds a fresh copy of test values useful for suggestion-similarity * testing. @@ -47,78 +48,89 @@ const testModelWithoutSpacing = new DummyModel({ */ const build_its_is_set = (verbose?: string) => { const verboseFlag = (verbose == 'verbose' ? true : false); - - const its: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const its: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 0 }, displayAs: 'its' }, - p: 0.2 + correction: 'its' }, - totalProb: 0.16 - // matchLevel does not yet exist. + metadata: { + probabilities: { + prediction: .2, + correction: .8, + total: .2 * .8 + }, + autoSelectable: true + // matchLevel does not yet exist. + } }; - const it_is: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const it_is: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: '\'s', deleteLeft: 0 }, displayAs: 'it\'s' }, - p: 0.8 + correction: 'its' }, - totalProb: 0.64 + metadata: { + probabilities: { + prediction: .8, + correction: .8, + total: .8 * .8 + }, + autoSelectable: true + } }; - const is: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 1 }, displayAs: 'is' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; - const is_not: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is_not: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 'sn\'t', deleteLeft: 1 }, displayAs: 'isn\'t' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; const baseDefinitions = { @@ -132,13 +144,13 @@ const build_its_is_set = (verbose?: string) => { const expected = unfinalized.map((entry) => { const mapped: Outcome = { - ...deepCopy(entry.prediction.sample), - p: entry.totalProb + ...deepCopy(entry.components.prediction), + p: entry.metadata.probabilities.total }; if(verboseFlag) { - mapped['correction-p'] = entry.correction.p; - mapped['lexical-p'] = entry.prediction.p; + mapped['correction-p'] = entry.metadata.probabilities.correction; + mapped['lexical-p'] = entry.metadata.probabilities.prediction; } return mapped; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts index 6cfa731429d..5c28e2a486d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts @@ -5,7 +5,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { deepCopy } from 'keyman/common/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { CorrectionPredictionTuple, models, processSimilarity, SuggestionSimilarity, toAnnotatedSuggestion } from "@keymanapp/lm-worker/test-index"; +import { CompositedIntermediatePrediction, models, processSimilarity, SuggestionSimilarity, toAnnotatedSuggestion } from "@keymanapp/lm-worker/test-index"; import CasingFunction = LexicalModelTypes.CasingFunction; import Context = LexicalModelTypes.Context; @@ -77,7 +77,7 @@ const testModelWithoutCasing = new DummyModel({ .replace(/[“”]/g, '"') // ** Difference from model-defaults here ** // And finally, erase single-quotation marks. - .replace(/'/, ''); + .replace(/'/g, ''); } // No suggestions needed here, so we don't define any. }); @@ -109,77 +109,89 @@ const testModelWithCasing = new DummyModel({ * @returns */ const build_its_is_set = () => { - const its: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const its: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 0 }, displayAs: 'its' }, - p: 0.2 + correction: 'its' }, - totalProb: 0.16 - // matchLevel does not yet exist. + metadata: { + probabilities: { + prediction: .2, + correction: .8, + total: .2 * .8 + }, + autoSelectable: true + // matchLevel does not yet exist. + } }; - const it_is: CorrectionPredictionTuple = { - correction: { - sample: 'its', - p: 0.8 - }, - prediction: { - sample: { + const it_is: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: '\'s', deleteLeft: 0 }, displayAs: 'it\'s' }, - p: 0.8 + correction: 'its' }, - totalProb: 0.64 + metadata: { + probabilities: { + prediction: .8, + correction: .8, + total: .8 * .8 + }, + autoSelectable: true + } }; - const is: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 's', deleteLeft: 1 }, displayAs: 'is' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; - const is_not: CorrectionPredictionTuple = { - correction: { - sample: 'is', - p: 0.2 - }, - prediction: { - sample: { + const is_not: CompositedIntermediatePrediction = { + components: { + prediction: { transform: { insert: 'sn\'t', deleteLeft: 1 }, displayAs: 'isn\'t' }, - p: 0.5 + correction: 'is' }, - totalProb: 0.1 + metadata: { + probabilities: { + prediction: .5, + correction: .2, + total: .5 * .2 + }, + autoSelectable: true + } }; return { @@ -210,32 +222,22 @@ describe('processSimilarity', () => { const testSet = build_its_is_set(); const distribution = [...Object.values(testSet)]; - const expectation: CorrectionPredictionTuple[] = [ - { - ...testSet.its, - matchLevel: SuggestionSimilarity.exact - }, { - ...testSet.it_is, - matchLevel: SuggestionSimilarity.sameKey - }, { - ...testSet.is, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is_not, - matchLevel: SuggestionSimilarity.none - } - ]; + const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)]; + expectation[0].metadata.matchLevel = SuggestionSimilarity.exact; // its + expectation[1].metadata.matchLevel = SuggestionSimilarity.sameKey; // it_is + expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is + expectation[3].metadata.matchLevel = SuggestionSimilarity.none; // is_not const its = testSet.its; const original_its = deepCopy(its); - const keep_its = toAnnotatedSuggestion(testModelWithCasing, original_its.prediction.sample, 'keep', QuoteBehavior.noQuotes); + const keep_its = toAnnotatedSuggestion(testModelWithCasing, original_its.components.prediction, 'keep', QuoteBehavior.noQuotes); keep_its.matchesModel = true; - processSimilarity(testModelWithCasing, distribution, context, trueInput); + processSimilarity(testModelWithCasing, distribution, context, models.applyTransform(trueInput.sample, context)); assert.sameDeepMembers(distribution, expectation); - assert.equal(its.prediction.sample.tag, 'keep'); - assert.deepEqual(its.prediction.sample, keep_its); + assert.equal(its.components.prediction.tag, 'keep'); + assert.deepEqual(its.components.prediction, keep_its); }); it(`selects contraction as 'more similar' than same-keyed non-contraction when context is contraction`, () => { @@ -257,32 +259,61 @@ describe('processSimilarity', () => { const testSet = build_its_is_set(); const distribution = [...Object.values(testSet)]; - const expectation: CorrectionPredictionTuple[] = [ - { - ...testSet.its, - matchLevel: SuggestionSimilarity.sameKey - }, { - ...testSet.it_is, - matchLevel: SuggestionSimilarity.exact - }, { - ...testSet.is, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is_not, - matchLevel: SuggestionSimilarity.none - } - ]; + const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)]; + expectation[0].metadata.matchLevel = SuggestionSimilarity.sameKey; // its + expectation[1].metadata.matchLevel = SuggestionSimilarity.exact; // it_is + expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is + expectation[3].metadata.matchLevel = SuggestionSimilarity.none; // is_not const it_is = testSet.it_is; const original_it_is = deepCopy(it_is); - const keep_it_is = toAnnotatedSuggestion(testModelWithCasing, original_it_is.prediction.sample, 'keep', QuoteBehavior.noQuotes); + const keep_it_is = toAnnotatedSuggestion(testModelWithCasing, original_it_is.components.prediction, 'keep', QuoteBehavior.noQuotes); keep_it_is.matchesModel = true; - processSimilarity(testModelWithCasing, distribution, context, trueInput); + processSimilarity(testModelWithCasing, distribution, context, models.applyTransform(trueInput.sample, context)); assert.sameDeepMembers(distribution, expectation); - assert.equal(it_is.prediction.sample.tag, 'keep'); - assert.deepEqual(it_is.prediction.sample, keep_it_is); + assert.equal(it_is.components.prediction.tag, 'keep'); + assert.deepEqual(it_is.components.prediction, keep_it_is); + }); + + it('operates properly when no transition in context occurs', () => { + const transformId = 314159; + + const context: Context = { + left: 'appl', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const distribution: CompositedIntermediatePrediction[] = [ + { + components: { + prediction: { + transform: { + insert: 'apple', + deleteLeft: 4, + id: transformId + }, + displayAs: 'apple' + }, + correction: 'appl' + }, + metadata: { + probabilities: { + prediction: 1, + correction: 1, + total: 1 + }, + autoSelectable: true + } + } + ]; + + const result = processSimilarity(testModelWithCasing, distribution, context, context); + assert.isFalse(result); + assert.equal(distribution[0].metadata.matchLevel, SuggestionSimilarity.none); }); describe('with casing', () => { @@ -314,34 +345,22 @@ describe('processSimilarity', () => { // Have the predictions replace existing context parts with the lowercased equivalents. Object.values(testSet).forEach((entry) => { - const transform = entry.prediction.sample.transform; + const transform = entry.components.prediction.transform; transform.insert = transform.deleteLeft == 0 ? `it${transform.insert}` : `i${transform.insert}`; transform.deleteLeft = 2; }); const distribution = [...Object.values(testSet)]; - const expectation: CorrectionPredictionTuple[] = [ - { - ...testSet.its, - matchLevel: SuggestionSimilarity.sameKey - }, { - ...testSet.it_is, - // case mismatch, detectable because we have access to a lowercasing/uppercasing function. - matchLevel: SuggestionSimilarity.sameText - }, { - ...testSet.is, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is_not, - matchLevel: SuggestionSimilarity.none - } - ]; - - processSimilarity(testModelWithCasing, distribution, context, trueInput); + const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)]; + expectation[0].metadata.matchLevel = SuggestionSimilarity.sameKey; // its + expectation[1].metadata.matchLevel = SuggestionSimilarity.sameText; // it_is + expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is + expectation[3].metadata.matchLevel = SuggestionSimilarity.none; // is_not + processSimilarity(testModelWithCasing, distribution, context, models.applyTransform(trueInput.sample, context)); // Because we mucked with the casing here, there is no perfect 'keep' match. - const keep = distribution.find((entry) => entry.prediction.sample.tag == 'keep'); + const keep = distribution.find((entry) => entry.components.prediction.tag == 'keep'); assert.isNotOk(keep); assert.sameDeepMembers(distribution, expectation); }); @@ -368,34 +387,20 @@ describe('processSimilarity', () => { // Have the predictions replace existing context parts with the lowercased equivalents. Object.values(testSet).forEach((entry) => { - const transform = entry.prediction.sample.transform; + const transform = entry.components.prediction.transform; transform.insert = transform.deleteLeft == 0 ? `it${transform.insert}` : `i${transform.insert}`; transform.deleteLeft = 2; }); const distribution = [...Object.values(testSet)]; - const expectation: CorrectionPredictionTuple[] = [ - { - ...testSet.its, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.it_is, - // case mismatch, detectable because we have access to a lowercasing/uppercasing function. - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is, - matchLevel: SuggestionSimilarity.none - }, { - ...testSet.is_not, - matchLevel: SuggestionSimilarity.none - } - ]; + const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)]; - processSimilarity(testModelWithoutCasing, distribution, context, trueInput); + expectation.forEach((entry) => entry.metadata.matchLevel = SuggestionSimilarity.none); + processSimilarity(testModelWithoutCasing, distribution, context, models.applyTransform(trueInput.sample, context)); // Because we mucked with the casing here, there is no perfect 'keep' match. - const keep = distribution.find((entry) => entry.prediction.sample.tag == 'keep'); + const keep = distribution.find((entry) => entry.components.prediction.tag == 'keep'); assert.isNotOk(keep); assert.sameDeepMembers(distribution, expectation); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/suggestion-casing.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/suggestion-casing.tests.ts index dd586eab646..b54b688614a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/suggestion-casing.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/suggestion-casing.tests.ts @@ -13,7 +13,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { applySuggestionCasing, models } from '@keymanapp/lm-worker/test-index'; +import { TokenizedPredictionData, applySuggestionCasing, models } from '@keymanapp/lm-worker/test-index'; import CasingFunction = LexicalModelTypes.CasingFunction; import TrieModel = models.TrieModel; @@ -45,117 +45,137 @@ describe('applySuggestionCasing', function() { ); it('properly cases suggestions with no suggestion root', function() { - let suggestion = { - transform: { - insert: 'the', - deleteLeft: 0 + let suggestion: TokenizedPredictionData[] = [{ + prediction: { + transform: { + insert: 'the', + deleteLeft: 0 + }, + displayAs: 'the' }, - displayAs: 'the' - }; - - applySuggestionCasing(suggestion, '', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'The'); - assert.equal(suggestion.transform.insert, 'The'); - - suggestion = { - transform: { - insert: 'thE', - deleteLeft: 0 - }, - displayAs: 'thE' - }; - - applySuggestionCasing(suggestion, '', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'ThE'); - assert.equal(suggestion.transform.insert, 'ThE'); - - suggestion = { - transform: { - insert: 'the', - deleteLeft: 0 + correction: '', + casingRoot: 'th' + }]; + + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'the'); + assert.equal(suggestion[0].prediction.transform.insert, 'the'); + + suggestion = [{ + prediction: { + transform: { + insert: 'ThE', + deleteLeft: 0 + }, + displayAs: 'ThE' }, - displayAs: 'the' - }; + correction: '', + casingRoot: 'Th' + }]; - applySuggestionCasing(suggestion, '', plainCasedModel, 'upper'); - assert.equal(suggestion.displayAs, 'THE'); - assert.equal(suggestion.transform.insert, 'THE'); + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'ThE'); + assert.equal(suggestion[0].prediction.transform.insert, 'ThE'); }); it('properly cases suggestions that fully replace the suggestion root', function() { - let suggestion = { - transform: { - insert: 'therefore', - deleteLeft: 3 + let suggestion: TokenizedPredictionData[] = [{ + prediction: { + transform: { + insert: 'therefore', + deleteLeft: 3 + }, + displayAs: 'therefore' }, - displayAs: 'therefore' - }; - - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'Therefore'); - assert.equal(suggestion.transform.insert, 'Therefore'); - - suggestion = { - transform: { - insert: 'thereFore', - deleteLeft: 3 + correction: 'The', + casingRoot: 'Th' + }]; + + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'Therefore'); + assert.equal(suggestion[0].prediction.transform.insert, 'Therefore'); + + suggestion = [{ + prediction: { + transform: { + insert: 'thereFore', + deleteLeft: 3 + }, + displayAs: 'thereFore' }, - displayAs: 'thereFore' - }; - - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'ThereFore'); - assert.equal(suggestion.transform.insert, 'ThereFore'); - - suggestion = { - transform: { - insert: 'therefore', - deleteLeft: 3 + correction: 'The', + casingRoot: 'Th' + }]; + + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'ThereFore'); + assert.equal(suggestion[0].prediction.transform.insert, 'ThereFore'); + + suggestion = [{ + prediction: { + transform: { + insert: 'therefore', + deleteLeft: 3 + }, + displayAs: 'therefore' }, - displayAs: 'therefore' - }; + correction: 'THE', + casingRoot: 'TH' + }]; - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'upper'); - assert.equal(suggestion.displayAs, 'THEREFORE'); - assert.equal(suggestion.transform.insert, 'THEREFORE'); + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'THEREFORE'); + assert.equal(suggestion[0].prediction.transform.insert, 'THEREFORE'); }); it('properly cases suggestions that do not fully replace the suggestion root', function() { - let suggestion = { - transform: { - insert: 'erefore', - deleteLeft: 1 + let suggestion: TokenizedPredictionData[] = [{ + prediction: { + transform: { + insert: 'therefore', + deleteLeft: 3 + }, + displayAs: 'therefore' }, - displayAs: 'therefore' - }; + correction: 'The', + casingRoot: 'Th' + }]; // When integrated, the 'the' string comes from a wordbreak operation on the current context. - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'Therefore'); - assert.equal(suggestion.transform.insert, 'Therefore'); - - suggestion = { - transform: { - insert: 'ereFore', - deleteLeft: 1 + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'Therefore'); + assert.equal(suggestion[0].prediction.transform.insert, 'Therefore'); + + suggestion = [{ + prediction: { + transform: { + insert: 'ThereFore', + deleteLeft: 3 + }, + displayAs: 'thereFore' }, - displayAs: 'thereFore' - }; - - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial'); - assert.equal(suggestion.displayAs, 'ThereFore'); - assert.equal(suggestion.transform.insert, 'ThereFore'); - - suggestion = { - transform: { - insert: 'erefore', - deleteLeft: 1 + correction: 'The', + casingRoot: 'Th' + }]; + + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'ThereFore'); + assert.equal(suggestion[0].prediction.transform.insert, 'ThereFore'); + + suggestion = [{ + prediction: { + transform: { + insert: 'therefore', + deleteLeft: 3 + }, + displayAs: 'therefore' }, - displayAs: 'therefore' - }; + correction: 'THE', + casingRoot: 'TH' + }]; - applySuggestionCasing(suggestion, 'the', plainCasedModel, 'upper'); - assert.equal(suggestion.displayAs, 'THEREFORE'); - assert.equal(suggestion.transform.insert, 'THEREFORE'); + applySuggestionCasing(suggestion[0], plainCasedModel); + assert.equal(suggestion[0].prediction.displayAs, 'THEREFORE'); + assert.equal(suggestion[0].prediction.transform.insert, 'THEREFORE'); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts index 9b9ab2c3121..01f1a6ac96a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts @@ -81,6 +81,37 @@ describe('Custom Punctuation', function () { open: "'", close: "'" } + }, + // Some of the suggestions above actually wordbreak differently from + // what might be expected. So, we override the wordbreaker to ensure + // the tests run smoothly. + wordbreaker: (text) => { + const textLen = text.length; + if(text.charAt(0) == "᚛") { // ensure the prior token component (the '᚛') wordbreaks. + if(text.charAt(textLen - 1) == " ") { // ensure the insert-after component word-breaks. + return [ + {text: text.substring(0, 1), start: 0, end: 1, length: 1}, + {text: text.substring(1, textLen-2), start: 1, end: textLen-1, length: textLen-2}, + {text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1} + ]; + } else { + return [ + {text: text.substring(0, 1), start: 0, end: 1, length: 1}, + {text: text.substring(1), start: 1, end: textLen, length: textLen-1} + ]; + } + } else { + if(text.charAt(textLen - 1) == " ") { + return [ + {text: text.substring(0, textLen-2), start: 0, end: textLen-1, length: textLen-1}, + {text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1} + ]; + } else { + return [ + {text: text, start: 0, end: textLen, length: textLen} + ]; + } + } } }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts index f01cafbac68..0ac51dd107b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts @@ -949,6 +949,9 @@ describe('ModelCompositor', function() { id: baseSuggestion.transform.id } + // Future adjustment: add the 'baseSuggestion' to DummyModel so that it actually + // returns the suggestion again. + // `new models.DummyModel(..., futureSuggestions: [[baseSuggestion]])` let model = new models.DummyModel({punctuation: englishPunctuation}); let compositor = new ModelCompositor(model, true); @@ -963,6 +966,7 @@ describe('ModelCompositor', function() { // As this test is a bit... 'hard-wired', we only get the 'keep' suggestion. // It should still be accurate, though. + // Can be fixed via the "Future adjustment" noted above. assert.equal(suggestions.length, 1); let expectedTransform = {