epic: boundary-correction 🔪

docs/epics.md

@@ -0,0 +1,18 @@
+# Epics
+
+This document lists epics in the Keyman project.
+
+ Name                  | Lead      | Target release | Spec
+-----------------------|-----------|----------------|--------------------
+ epic/embed-osk-in-kmx | @mcdurdin | 19.0           | [embed-osk-in-kmx]
+ epic/autocorrect      | @jahorton | 19.0           |
+ epic/boundary-correction | @jahorton | 20.0        |
+
+
+Note: when establishing a new epic branch, add the details of the epic to this
+file, as a clean way to start an epic without unreviewed code changes in the
+base epic PR.
+
+---
+
+[embed-osk-in-kmx]: https://docs.google.com/document/d/15EjtSH7NAsGrdapfB3E7SlS6CqI-2zPy3PNyfUiAv54/edit?tab=t.0

web/src/engine/predictive-text/templates/src/common.ts

@@ -60,7 +60,7 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans
     deleteLeft: first.deleteLeft + mergedSecondDelete
   }
 
-  if(first.id && first.id == second.id) {
+  if(first.id !== undefined && first.id == second.id) {
     returnedObj.id = first.id;
   }
 

web/src/engine/predictive-text/templates/src/tokenization.ts

@@ -95,6 +95,10 @@ export function tokenize(
     currentIndex = nextIndex;
   }
 
+  if(tokenization.left.length == 0) {
+    tokenization.left.push({text: '', isWhitespace: false});
+  }
+
   // New step 2: handle any rejoins needed.
 
   // Handle any desired special handling for directly-pre-caret scenarios - where for this

web/src/engine/predictive-text/worker-thread/build.sh

@@ -21,6 +21,7 @@ SRCMAP_CLEANER="${KEYMAN_ROOT}/web/build/tools/building/sourcemap-root/index.js"
 ################################ Main script ################################
 
 SUBPROJECT_NAME=engine/predictive-text/worker-thread
+SUBPROJECT_HELPERS=engine/predictive-text/helpers
 
 builder_describe \
   "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \
@@ -100,6 +101,7 @@ function do_test() {
     WTR_INSPECT=" --manual"
   fi
 
+  test-headless-typescript $SUBPROJECT_HELPERS
   test-headless-typescript $SUBPROJECT_NAME
 
   web-test-runner --config ./src/tests/test-runner/web-test-runner${WTR_CONFIG}.config.mjs ${WTR_INSPECT}

web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts

@@ -46,7 +46,13 @@ export class ContextState {
   /**
    * Denotes the possible tokenization(s) for the represented Context.
    */
-  tokenization: ContextTokenization;
+  _tokenizations: ContextTokenization[];
+
+  /**
+   * Tracks the tokenization pattern best matching the word boundary
+   * patterns in the actual context.
+   */
+  _displayTokenization: ContextTokenization;
 
   /**
    * Denotes the keystroke-sourced Transform that was last applied to a
@@ -97,7 +103,14 @@ export class ContextState {
    * is visible to the user.
    */
   get displayTokenization(): ContextTokenization {
-    return this.tokenization;
+    return this._displayTokenization;
+  }
+
+  /**
+   * Denotes the possible tokenization(s) for the represented Context.
+   */
+  get tokenizations(): ContextTokenization[] {
+    return this._tokenizations;
   }
 
   /**
@@ -127,13 +140,16 @@ export class ContextState {
    * @param tokenization Precomputed tokenization for the context, leveraging previous
    * correction-search progress and results
    */
-  constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization);
-  constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization) {
+  constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization, tokenizations?: ContextTokenization[]);
+  constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization, tokenizations?: ContextTokenization[]) {
     if(!(param1 instanceof ContextState)) {
       this.context = param1;
       this.model = model;
       if(tokenization) {
-        this.tokenization = tokenization;
+        this._tokenizations = tokenizations ? tokenizations : [tokenization];
+        this._displayTokenization = tokenization;
+
+        this.inputTransforms = new Map();
       } else {
         this.initFromReset();
       }
@@ -142,7 +158,8 @@ export class ContextState {
 
       Object.assign(this, stateToClone);
       this.inputTransforms = new Map(stateToClone.inputTransforms);
-      this.tokenization = new ContextTokenization(stateToClone.tokenization);
+      this._displayTokenization = new ContextTokenization(stateToClone._displayTokenization);
+      this._tokenizations = stateToClone.tokenizations.map((t) => new ContextTokenization(t));
 
       // A shallow copy of the array is fine, but we'd be best off
       // not aliasing the array itself.
@@ -173,10 +190,35 @@ export class ContextState {
     if(baseTokens.length == 0) {
       baseTokens.push(ContextToken.fromRawText(this.model, ''));
     }
-    this.tokenization = new ContextTokenization(baseTokens);
+    this._displayTokenization = new ContextTokenization(baseTokens);
+    this._tokenizations = [this._displayTokenization];
     this.inputTransforms = new Map();
   }
 
+  /**
+   * Builds a variant of the ContextState with no input metadata, reiterating the resulting
+   * state of a recent transition but without any associated input mutation data.
+   * @param stateToClone
+   * @param context
+   * @returns
+   */
+  transitionContextWindow(context: Context) {
+    // The context may have slid since the last ContextState observation.  We check for slide effects
+    // and apply them here as well.
+    const slideUpdateTransform = determineContextSlideTransform(this.context, context);
+
+    const model = this.model;
+
+    // Should a context-reset or similar occur, there is no input to replace, nor is there
+    // pre-tail token transform data to preserve.  It's important that part, in particular,
+    // be cleared so that Suggestions in this state are not adversely affected.
+    const displayTokenization = this._displayTokenization.applyContextSlide(model, slideUpdateTransform);
+    const tokenizations = this.tokenizations.map((t) => t.applyContextSlide(model, slideUpdateTransform));
+
+    const state = new ContextState(context, this.model, displayTokenization, tokenizations);
+    return state;
+  }
+
   /**
    * As written, this method attempts to determine the context state and tokenization(s)
    * that result from applying an incoming transform distribution to the incoming context
@@ -209,8 +251,7 @@ export class ContextState {
 
     const slideUpdateTransform = determineContextSlideTransform(this.context, context);
 
-    // Goal:  allow multiple base tokenizations.
-    const startTokenizations = [this.tokenization].map((t) => {
+    const startTokenizations = this.tokenizations.map((t) => {
       return t.applyContextSlide(lexicalModel, slideUpdateTransform);
     });
 
@@ -222,14 +263,15 @@ export class ContextState {
       // If the tokenizations match, clone the ContextState; we want to preserve a post-application
       // context separately from pre-application contexts for predictions based on empty roots.
       const state = new ContextState(this);
-      state.tokenization = [...startTokenizations.values()][0];
+      state._tokenizations = [...startTokenizations.values()];
+      state._displayTokenization = this._displayTokenization.applyContextSlide(lexicalModel, slideUpdateTransform);
       transition.finalize(state, transformDistribution);
       return transition;
     }
 
     const { subsets, keyMatchingUserContext: trueInputSubsetKey } = precomputeTransitions(startTokenizations, transformDistribution);
-    const resultTokenization = transitionTokenizations(subsets, transformDistribution).get(trueInputSubsetKey);
-
+    const possibleTokenizations = transitionTokenizations(subsets, transformDistribution);
+    const resultTokenization = possibleTokenizations.get(trueInputSubsetKey);
     // ------------
 
     // So, if we have a suggestion transition ID at the end and didn't just apply...
@@ -240,10 +282,11 @@ export class ContextState {
     // 'any'.)
 
     const state = new ContextState(applyTransform(trueInput, context), lexicalModel);
-    state.tokenization = resultTokenization;
+    state._tokenizations = [resultTokenization]; // TODO:  [...possibleTokenizations.values()];
+    state._displayTokenization = resultTokenization;
     state.appliedInput = transformDistribution?.[0].sample;
     transition.finalize(state, transformDistribution);
-    transition.revertableTransitionId = state.tokenization.tail.appliedTransitionId;
+    transition.revertableTransitionId = state._displayTokenization.tail.appliedTransitionId;
     return transition;
   }
 }

web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts

@@ -15,7 +15,6 @@ import { LegacyQuotientSpur } from "./legacy-quotient-spur.js";
 import { LegacyQuotientRoot } from "./legacy-quotient-root.js";
 import { generateSubsetId } from './tokenization-subsets.js';
 
-import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
 import ProbabilityMass = LexicalModelTypes.ProbabilityMass;
 import Transform = LexicalModelTypes.Transform;
@@ -37,11 +36,42 @@ function textToCharTransforms(text: string, transitionId?: number): Transform[]
     [...text].map(insert => ({insert, deleteLeft: 0}));
 }
 
+
+/**
+ * Defines an interface compatible with ContextToken that is useful for handling
+ * cases that should not be considered correctable.
+ */
+export interface ContextTokenLike {
+  /**
+   * Generates text corresponding to the net effects of the most likely inputs
+   * received that can correspond to the represented token.
+   */
+  exampleInput: string;
+
+  /**
+   * Reports the length in codepoints of corrected text represented by the
+   * current token.
+   */
+  codepointLength: number;
+
+  /**
+   * Whether or not the token is likely still being edited by the user (due to
+   * adjacency of the caret)
+   */
+  isPartial?: boolean;
+
+  /**
+   * Gets a compact string-based representation of `inputRange` that
+   * maps compatible token source ranges to each other.
+   */
+  sourceRangeKey?: string;
+}
+
 /**
  * Represents cached data about one token (either a word or a unit of whitespace)
  * in the context and associated correction-search progress and results.
  */
-export class ContextToken {
+export class ContextToken implements ContextTokenLike {
   /**
    * Indicates whether or not the token is considered whitespace.
    */
@@ -56,6 +86,10 @@ export class ContextToken {
   }
   private _searchModule: SearchQuotientNode;
 
+  /**
+   * Whether or not the token is likely still being edited by the user (due to
+   * adjacency of the caret)
+   */
   isPartial: boolean;
 
   /**
@@ -125,11 +159,11 @@ export class ContextToken {
   }
 
   /**
-   * Call this to record the original keystroke Transforms for the context range
-   * corresponding to this token.
+   * Reports the length in codepoints of corrected text represented by the
+   * current token.
    */
-  addInput(inputSource: PathInputProperties, distribution: Distribution<Transform>) {
-    this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource);
+  get codepointLength() {
+    return this._searchModule.codepointLength;
   }
 
   get inputCount() {
@@ -169,7 +203,7 @@ export class ContextToken {
 
   /**
    * Generates text corresponding to the net effects of the most likely inputs
-   * received that can correspond to the current instance.
+   * received that can correspond to the represented token.
    */
   get exampleInput(): string {
     return this.searchModule.bestExample.text;