Editor: right-to-left lines align right and move by what is on screen (#134)

packages/app-core/src/components/Editor.tsx

@@ -11,6 +11,7 @@ import { useCallback, useEffect, useMemo, useRef } from "react";
 import type { EditorView } from "@codemirror/view";
 import { Vim, getCM } from "@replit/codemirror-vim";
 import { registerDisplayLineMotion } from "../lib/cm-vim-display-line";
+import { registerVisualCharMotion } from "../lib/cm-bidi";
 import {
   HALF_PAGE_MOTION,
   halfPageMotionArgs,
@@ -571,6 +572,8 @@ function registerVimCommands(): void {
   registerDisplayLineMotion(
     () => useStore.getState().vimWrappedLineMotions,
   );
+  // #134: h/l and the arrow keys move by what is on screen in RTL lines.
+  registerVisualCharMotion();
   registerHeadingMotion();
   registerHalfPageMotion();
   registerReflowOperator();

packages/app-core/src/components/EditorPane.tsx

@@ -63,6 +63,7 @@ import {
   listIndentWidth,
   markdownListIndentPlugin
 } from '../lib/cm-markdown-list-indent'
+import { bidiExtension } from '../lib/cm-bidi'
 import { forwardOnCheckboxArrow } from '../lib/cm-forward-task'
 import { markerHopCommands } from '../lib/cm-marker-hop'
 import { isInMarkdownCode } from '../lib/cm-auto-pairs'
@@ -438,6 +439,7 @@ function markdownEditingExtensions(showHeadingLevelLabels = false): Extension[]
     markdownLinkExtension,
     vimAwareMarkdownKeymap,
     markdownListIndentPlugin,
+    bidiExtension,
     frontmatterTagExtension,
     orderedListRenumber,
     forwardOnCheckboxArrow,

packages/app-core/src/components/ExternalFileApp.tsx

@@ -23,6 +23,7 @@ import { customCodeFenceHighlightExtension } from '../lib/cm-custom-code-languag
 import { vimHalfPageKeymap } from '../lib/vim-half-page-keymap'
 import { applyVimInsertEscape } from '../lib/vim-insert-escape'
 import { markdownListIndentPlugin } from '../lib/cm-markdown-list-indent'
+import { bidiExtension, registerVisualCharMotion } from '../lib/cm-bidi'
 import { appMarkdownSnippetExtension } from '../lib/markdown-snippets-config'
 import { syntaxHighlighting, defaultHighlightStyle } from '@codemirror/language'
 import type { ExternalFileContent } from '@shared/ipc'
@@ -173,6 +174,7 @@ export function ExternalFileApp(): JSX.Element {
           customCodeFenceHighlightExtension,
           vimAwareMarkdownKeymap,
           markdownListIndentPlugin,
+          bidiExtension,
           headingFolding({ showLevelLabels: prefs.showHeadingLevelLabels }),
           syntaxHighlighting(paperHighlight),
           syntaxHighlighting(defaultHighlightStyle, { fallback: true }),
@@ -424,6 +426,7 @@ function registerExternalFileVimCommands(): void {
 
   registerHalfPageMotion()
   mapDefaultHalfPageKeys()
+  registerVisualCharMotion()
 
   Vim.defineEx('write', 'w', () => {
     void externalFileHandlers.persist?.()

packages/app-core/src/components/FloatingNoteApp.tsx

@@ -40,6 +40,7 @@ import { registerReflowOperator } from '../lib/cm-vim-reflow'
 import { vimHalfPageKeymap } from '../lib/vim-half-page-keymap'
 import { isTouchPrimaryDevice, vimImeGuard } from '../lib/cm-vim-ime-guard'
 import { markdownListIndentPlugin } from '../lib/cm-markdown-list-indent'
+import { bidiExtension, registerVisualCharMotion } from '../lib/cm-bidi'
 import { appMarkdownSnippetExtension } from '../lib/markdown-snippets-config'
 import { syntaxHighlighting, HighlightStyle, defaultHighlightStyle } from '@codemirror/language'
 import { tags as t } from '@lezer/highlight'
@@ -347,6 +348,7 @@ export function FloatingNoteApp({ notePath }: { notePath: string }): JSX.Element
           markdownLinkExtension,
           vimAwareMarkdownKeymap,
           markdownListIndentPlugin,
+          bidiExtension,
           headingFolding({ showLevelLabels: prefs.showHeadingLevelLabels }),
           syntaxHighlighting(paperHighlight),
           syntaxHighlighting(defaultHighlightStyle, { fallback: true }),
@@ -565,6 +567,7 @@ function registerFloatingVimCommands(
   getWrappedLineMotionMode: () => VimWrappedLineMotionMode
 ): void {
   registerDisplayLineMotion(getWrappedLineMotionMode)
+  registerVisualCharMotion()
   if (floatingVimRegistered) return
   floatingVimRegistered = true
 

packages/app-core/src/components/PinnedReferencePane.tsx

@@ -40,6 +40,7 @@ import {
   listIndentWidth,
   markdownListIndentPlugin
 } from '../lib/cm-markdown-list-indent'
+import { bidiExtension } from '../lib/cm-bidi'
 import { syntaxHighlighting, HighlightStyle, defaultHighlightStyle } from '@codemirror/language'
 import { tags as t } from '@lezer/highlight'
 import { autocompletion } from '@codemirror/autocomplete'
@@ -252,6 +253,7 @@ export function PinnedReferencePane(): JSX.Element | null {
           markdownLinkExtension,
           vimAwareMarkdownKeymap,
           markdownListIndentPlugin,
+          bidiExtension,
           frontmatterStyle,
           frontmatterTagExtension,
           headingCompartment.of(

packages/app-core/src/components/QuickCaptureApp.tsx

@@ -54,6 +54,7 @@ import { noteMarkdown } from '../lib/cm-markdown-language'
 import { customCodeFenceHighlightExtension } from '../lib/cm-custom-code-languages'
 import { markdownLinkExtension } from '../lib/cm-markdown-links'
 import { markdownListIndentPlugin } from '../lib/cm-markdown-list-indent'
+import { bidiExtension, registerVisualCharMotion } from '../lib/cm-bidi'
 import { appMarkdownSnippetExtension } from '../lib/markdown-snippets-config'
 import { syntaxHighlighting, HighlightStyle, defaultHighlightStyle } from '@codemirror/language'
 import { tags as t } from '@lezer/highlight'
@@ -217,6 +218,7 @@ function registerCaptureVimCommands(
   getWrappedLineMotionMode: () => VimWrappedLineMotionMode
 ): void {
   registerDisplayLineMotion(getWrappedLineMotionMode)
+  registerVisualCharMotion()
   if (vimRegistered) return
   vimRegistered = true
 
@@ -484,6 +486,7 @@ export function QuickCaptureApp(): JSX.Element {
           markdownLinkExtension,
           vimAwareMarkdownKeymap,
           markdownListIndentPlugin,
+          bidiExtension,
           headingFolding({ showLevelLabels: prefs.showHeadingLevelLabels }),
           syntaxHighlighting(captureHighlight),
           syntaxHighlighting(defaultHighlightStyle, { fallback: true }),

packages/app-core/src/lib/cm-bidi.test.ts

@@ -0,0 +1,120 @@
+// @vitest-environment jsdom
+import { EditorSelection, EditorState } from '@codemirror/state'
+import { EditorView } from '@codemirror/view'
+import { Vim, getCM, vim } from '@replit/codemirror-vim'
+import { afterEach, beforeAll, describe, expect, it } from 'vitest'
+import { bidiExtension, lineIsRtl, registerVisualCharMotion } from './cm-bidi'
+
+describe('lineIsRtl', () => {
+  it.each([
+    ['مرحبا بالعالم', true],
+    ['שלום עולם', true],
+    ['Hello world', false],
+    ['', false],
+    ['123 שלום', true],
+    ['Hello مرحبا', false],
+    ['مرحبا Hello', true],
+    ['**مرحبا**', true],
+    ['# عنوان', true],
+    ['## Title عنوان', false],
+    ['> اقتباس', true],
+    ['- عنصر', true],
+    ['1. خطوة', true],
+    ['- [x] مهمة', true],
+    ['- [ ] task', false],
+    ['- [x]', false],
+    ['‏abc', true],
+    ['‎مرحبا', false]
+  ])('%j reads right to left: %s', (text, rtl) => {
+    expect(lineIsRtl(text)).toBe(rtl)
+  })
+})
+
+const views: EditorView[] = []
+
+afterEach(() => {
+  views.splice(0).forEach((view) => view.destroy())
+})
+
+function editor(doc: string, cursor = 0, withVim = false): EditorView {
+  const view = new EditorView({
+    parent: document.body,
+    state: EditorState.create({
+      doc,
+      selection: EditorSelection.cursor(cursor),
+      extensions: [...(withVim ? [vim()] : []), bidiExtension]
+    })
+  })
+  views.push(view)
+  return view
+}
+
+describe('bidiExtension', () => {
+  it('marks right-to-left lines with dir="rtl" and leaves the others alone', () => {
+    const view = editor('Hello\nمرحبا\n- [x] مهمة\n- [ ] task\nשלום')
+    const dirs = [...view.contentDOM.querySelectorAll('.cm-line')].map((line) =>
+      line.getAttribute('dir')
+    )
+    expect(dirs).toEqual([null, 'rtl', 'rtl', null, 'rtl'])
+  })
+
+  it('follows edits', () => {
+    const view = editor('Hello')
+    view.dispatch({ changes: { from: 0, to: 5, insert: 'مرحبا' } })
+    expect(view.contentDOM.querySelector('.cm-line')?.getAttribute('dir')).toBe('rtl')
+  })
+})
+
+describe('registerVisualCharMotion', () => {
+  beforeAll(() => {
+    registerVisualCharMotion()
+    registerVisualCharMotion() // idempotent
+  })
+
+  function press(view: EditorView, keys: string[]): void {
+    const cm = getCM(view)!
+    for (const key of keys) Vim.handleKey(cm, key, 'user')
+  }
+
+  it('keeps h/l logical in a left-to-right line', () => {
+    const view = editor('abcdef', 2, true)
+    press(view, ['l'])
+    expect(view.state.selection.main.head).toBe(3)
+    press(view, ['h', 'h'])
+    expect(view.state.selection.main.head).toBe(1)
+  })
+
+  it('moves by what is on screen in a right-to-left line', () => {
+    const view = editor('אבגדהו', 2, true)
+    press(view, ['l'])
+    expect(view.state.selection.main.head).toBe(1)
+    press(view, ['h', 'h'])
+    expect(view.state.selection.main.head).toBe(3)
+  })
+
+  it('takes a count and drives <Left>/<Right>', () => {
+    const view = editor('أبجدهوز', 5, true)
+    press(view, ['2', 'l'])
+    expect(view.state.selection.main.head).toBe(3)
+    press(view, ['<Left>'])
+    expect(view.state.selection.main.head).toBe(4)
+    press(view, ['<Right>'])
+    expect(view.state.selection.main.head).toBe(3)
+  })
+
+  it('decides per line', () => {
+    const view = editor('abc\nאבג', 1, true)
+    press(view, ['l'])
+    expect(view.state.selection.main.head).toBe(2)
+    const second = view.state.doc.line(2).from
+    view.dispatch({ selection: EditorSelection.cursor(second + 1) })
+    press(view, ['l'])
+    expect(view.state.selection.main.head).toBe(second)
+  })
+
+  it('deletes toward the visual right with dl in a right-to-left line', () => {
+    const view = editor('אבג', 1, true)
+    press(view, ['d', 'l'])
+    expect(view.state.doc.toString()).toBe('בג')
+  })
+})

packages/app-core/src/lib/cm-bidi.ts

@@ -0,0 +1,128 @@
+import { CodeMirror, Vim } from '@replit/codemirror-vim'
+import { RangeSetBuilder, type Extension } from '@codemirror/state'
+import {
+  Decoration,
+  type DecorationSet,
+  EditorView,
+  ViewPlugin,
+  type ViewUpdate
+} from '@codemirror/view'
+
+/**
+ * Right-to-left lines (#134). Each line takes the direction of its first
+ * strong letter, so a Hebrew or Arabic line aligns right and the arrow keys
+ * and `h`/`l` move the way they point on screen.
+ *
+ * The direction is computed here rather than left to `dir="auto"`, because
+ * the browser would decide on the first letter of the Markdown source: the
+ * `x` of `- [x] مهمة` makes a task line left-to-right. Leading list markers,
+ * task boxes, heading hashes and quote markers are skipped first, and the
+ * same answer feeds the rendered `dir`, CodeMirror's arrow keys (through
+ * `perLineTextDirection`) and the Vim motion, so the three never disagree.
+ */
+
+// Letters of the scripts written right to left. Marks and digits are not
+// strong, so `123 שלום` is still a right-to-left line.
+const RTL_LETTER =
+  /[\p{Script=Arabic}\p{Script=Hebrew}\p{Script=Syriac}\p{Script=Thaana}\p{Script=Nko}\p{Script=Samaritan}\p{Script=Mandaic}\p{Script=Adlam}\p{Script=Hanifi_Rohingya}]/u
+// The first strong character: any letter, or an explicit LRM / RLM / ALM.
+const FIRST_STRONG = /[\p{L}‎‏؜]/u
+const MARKDOWN_PREFIX =
+  /^[ \t]*(?:>[ \t]*)*(?:#{1,6}[ \t]+|(?:[-+*]|\d{1,9}[.)])[ \t]+(?:\[[ xX>/-]\][ \t]+)?)?/
+
+function firstStrongIsRtl(text: string): boolean | null {
+  const match = FIRST_STRONG.exec(text)
+  if (!match) return null
+  const ch = match[0]
+  if (ch === '‎') return false
+  if (ch === '‏' || ch === '؜') return true
+  return RTL_LETTER.test(ch)
+}
+
+/** Whether a line of Markdown source reads right to left. */
+export function lineIsRtl(text: string): boolean {
+  const body = text.slice(MARKDOWN_PREFIX.exec(text)?.[0].length ?? 0)
+  return firstStrongIsRtl(body) ?? firstStrongIsRtl(text) ?? false
+}
+
+const rtlLine = Decoration.line({ attributes: { dir: 'rtl' } })
+
+function rtlLineDecorations(view: EditorView): DecorationSet {
+  const builder = new RangeSetBuilder<Decoration>()
+  const { doc } = view.state
+  let lastLine = -1
+  for (const { from, to } of view.visibleRanges) {
+    for (let pos = from; pos <= to; ) {
+      const line = doc.lineAt(pos)
+      if (line.number !== lastLine && lineIsRtl(line.text)) {
+        builder.add(line.from, line.from, rtlLine)
+      }
+      lastLine = line.number
+      pos = line.to + 1
+    }
+  }
+  return builder.finish()
+}
+
+const rtlLinePlugin = ViewPlugin.fromClass(
+  class {
+    decorations: DecorationSet
+    constructor(view: EditorView) {
+      this.decorations = rtlLineDecorations(view)
+    }
+    update(update: ViewUpdate): void {
+      if (update.docChanged || update.viewportChanged) {
+        this.decorations = rtlLineDecorations(update.view)
+      }
+    }
+  },
+  { decorations: (plugin) => plugin.decorations }
+)
+
+/**
+ * Per-line text direction for an editor: right-to-left lines get `dir="rtl"`
+ * (so they align right, `text-align` being `start`), and CodeMirror reads the
+ * direction of each line, which makes the arrow keys move visually.
+ */
+export const bidiExtension: Extension = [EditorView.perLineTextDirection.of(true), rtlLinePlugin]
+
+type VimCharCm = { getLine: (line: number) => string }
+type VimCharMotionArgs = { forward?: boolean; repeat?: number }
+
+/** `h`/`l` by what is on screen: in a right-to-left line `l` steps back. */
+function zenMoveByVisualCharacters(
+  cm: VimCharCm,
+  head: { line: number; ch: number },
+  motionArgs: VimCharMotionArgs
+): { line: number; ch: number } {
+  const repeat = motionArgs.repeat ?? 1
+  const forward = Boolean(motionArgs.forward) !== lineIsRtl(cm.getLine(head.line))
+  return new CodeMirror.Pos(head.line, forward ? head.ch + repeat : head.ch - repeat)
+}
+
+let visualCharMotionRegistered = false
+
+/**
+ * Register the visual `h`/`l` motion on the (per-window) global Vim (#134).
+ * Like Vim's 'rightleft', it applies in every context, so `dl` in a
+ * right-to-left line deletes the character to the right on screen. `x`, `<BS>`
+ * and `<Space>` keep their logical meaning. `<Left>`/`<Right>` are mapped as
+ * well: Vim forwards them to its built-in `h`/`l` without remapping, so they
+ * would otherwise still move backwards in a right-to-left line. Idempotent.
+ */
+export function registerVisualCharMotion(): void {
+  if (visualCharMotionRegistered) return
+  visualCharMotionRegistered = true
+  Vim.defineMotion(
+    'zenMoveByVisualCharacters',
+    zenMoveByVisualCharacters as unknown as Parameters<typeof Vim.defineMotion>[1]
+  )
+  for (const context of ['normal', 'visual', 'operatorPending'] as const) {
+    for (const key of ['h', '<Left>']) {
+      Vim.mapCommand(key, 'motion', 'zenMoveByVisualCharacters', { forward: false }, { context })
+    }
+    for (const key of ['l', '<Right>']) {
+      Vim.mapCommand(key, 'motion', 'zenMoveByVisualCharacters', { forward: true }, { context })
+    }
+  }
+}