diff --git a/.changeset/markdown-unicode-punctuation-flanking.md b/.changeset/markdown-unicode-punctuation-flanking.md new file mode 100644 index 00000000000..a22b458a477 --- /dev/null +++ b/.changeset/markdown-unicode-punctuation-flanking.md @@ -0,0 +1,13 @@ +--- +'@lexical/markdown': patch +--- + +[fix] Follow CommonMark's Unicode punctuation and whitespace when deciding whether `*`, `_`, `~~` and `==` markers open or close formatting + +Markdown import and the typing shortcuts now count every character in the Unicode punctuation (P) and symbol (S) categories as punctuation, so an en dash, a curly quote, a currency sign or a symbol such as โœ“ or ๐Ÿ˜€ next to a marker no longer reads as a letter, and `See โ€“**[here](https://example.com)**` imports as a bold link. On import, the backslash, which was missing from its ASCII list, counts too. Whitespace is the Zs category plus tab, line feed, form feed and carriage return, so U+000B, U+2028, U+2029 and U+FEFF no longer count as whitespace beside a marker, on import or for the typing shortcuts. + +Import also applies emphasis that closes before a later code span ahead of that span, as in `` **a.**x**`c`** ``, so its markers no longer split apart and stay as literal text. A link or other text match that ends before a code span is applied ahead of it, so `[a](b) and [`c`](d)` keeps both links, and one no longer takes part of any code span in the text, not only the first. A delimiter run that an earlier match cuts through now reads as it did in the whole text instead of as one at the edge of a line, so `*)**a*` keeps its literal asterisks. An escaped backtick right before more backticks no longer keeps the rest of that run from opening a code span, so ```x\``bb`x``` imports with code `bb`, as CommonMark reads it. Long lines import faster, since their code spans, emphasis, links and the matches of other text-match transformers whose patterns do not look behind are found once per line rather than again after each match. + +Export of formatting that starts or ends with punctuation right beside a letter, such as plain `foo` then bold `โ€œbarโ€`, now writes that letter as a numeric character reference (`foo**โ€œbarโ€**baz`), so that the formatting survives a round trip through a CommonMark reader. Those references are chosen so that micromark-based readers such as `@lexical/mdast`, which read an emoji or another character outside the Basic Multilingual Plane as a letter, format it the same way, so plain `๐Ÿ˜€` then bold `โ€œbโ€` exports as `😀**โ€œbโ€**`. Characters that are neither punctuation nor whitespace count as letters here, including the variation selector that ends an emoji such as โค๏ธ. A raw backslash that ends text right before a marker or such a reference (text exported with `shouldPreserveNewLines`, or decorator text) now goes out as `\`, so that it does not escape it. With `shouldPreserveNewLines`, backslashes right before a `*`, `_`, `` ` `` or `~` of the text are escaped too, so that they no longer cancel that character's escape. A bare URL, `www.` host or email that GFM readers would link, and that would hold such a reference or take in the markers or code span after it, now has a character of its prefix written as a reference too, so that no reader links it and the formatting and text come through: bold `www.x.co` then `a` exports as `**www.x.co**a`, and plain `see https://x.com/a` then bold `โ€œxโ€` as `see https://x.com/a**โ€œxโ€**`, where a GFM reader used to take the markers into the link. `@lexical/mdast` still links the decoded text. + +Verify: Markdown with a marker between a letter and Unicode punctuation, such as `**โ€œbarโ€**baz`, `**ไฝ ๅฅฝใ€‚**ไธ–็•Œ` or `**ๆณจๆ„๏ผš**่ฏทๅ…ˆ็™ปๅฝ•`, now imports with literal asterisks, as CommonMark readers show it, where it used to import as bold. The same goes for formatting that starts or ends with an escaped ASCII character right beside a letter, such as `a**\_x**`, since the backslash is punctuation. Lexical exported these shapes for such formatting before, so check stored Markdown that has them. Exported Markdown for such formatting, and for formatting beside a bare URL, now holds the character references described above, so check export snapshots too. diff --git a/packages/lexical-markdown/README.md b/packages/lexical-markdown/README.md index b14b76dc6bd..b2b76f870b5 100644 --- a/packages/lexical-markdown/README.md +++ b/packages/lexical-markdown/README.md @@ -99,6 +99,18 @@ reference definition or a check list item, so text such as `[1]` and `array[0]` exports as typed. Escaped brackets and guarded `<`/`&` use numeric references so escaping does not change adjacent emphasis flanking. +Import follows CommonMark's Unicode punctuation and whitespace when deciding +whether `*`, `_`, `~~` and `==` open or close formatting. When formatted text +starts or ends with punctuation right beside a letter, export writes that +letter as a numeric reference (`foo**โ€œbarโ€**`) so that the markers still +open and close, for micromark-based readers such as `@lexical/mdast` too. This +holds with `shouldPreserveNewLines` as well, where a raw backslash right before +such a marker or reference goes out as `\`. Where a bare URL, `www.` host +or email that GFM readers would link holds such a reference, or would take in +the markers or code span after it, export writes a character of its prefix as +a reference too (`www.x.co*a*b`), so that no reader links it and the +formatting comes through. + There're three types of transformers: - **Element transformer** handles top level elements (lists, headings, quotes, tables or code blocks) diff --git a/packages/lexical-markdown/src/MarkdownExport.ts b/packages/lexical-markdown/src/MarkdownExport.ts index 2c19f3782f3..857131f21ce 100644 --- a/packages/lexical-markdown/src/MarkdownExport.ts +++ b/packages/lexical-markdown/src/MarkdownExport.ts @@ -34,7 +34,17 @@ import { type TextMatchTransformer, type Transformer, } from './MarkdownTransformers'; -import {isEmptyParagraph, transformersByType} from './utils'; +import { + characterAt, + characterBefore, + getPortableEmphasisSides, + isEmptyParagraph, + isEscaped, + isLetterForSomeReader, + isPortablyFlankedBy, + toCharacterReference, + transformersByType, +} from './utils'; /** * Renders string from markdown. The selection is moved to the start after the operation. @@ -178,13 +188,15 @@ function $processNodeForSelection( const result = transformer.export( node, node_ => - $exportChildrenForSelection( - node_, - selection, - textFormatTransformers, - textMatchTransformers, - shouldPreserveNewLines, - ).markdown, + joinPieces( + $exportChildrenForSelection( + node_, + selection, + textFormatTransformers, + textMatchTransformers, + shouldPreserveNewLines, + ).pieces, + ), selection, ); @@ -217,7 +229,7 @@ function $processNodeForSelection( shouldPreserveNewLines, ); return { - markdown: childResult.markdown, + markdown: joinPieces(childResult.pieces), shouldInclude: shouldInclude || childResult.shouldInclude, }; } else if ($isDecoratorNode(node)) { @@ -242,8 +254,8 @@ function $exportChildrenForSelection( shouldPreserveNewLines: boolean, unclosedTags?: OpenTag[], unclosableTags?: OpenTag[], -): {shouldInclude: boolean; markdown: string} { - const output = []; + output: MarkdownPieces = createPieces(), +): {shouldInclude: boolean; pieces: MarkdownPieces} { const children = node.getChildren(); let anyChildIncluded = false; @@ -266,24 +278,28 @@ function $exportChildrenForSelection( const result = transformer.export( child, parentNode => - $exportChildrenForSelection( - parentNode, - selection, - textFormatTransformers, - textMatchTransformers, - shouldPreserveNewLines, - unclosedTags, - [...unclosableTags, ...unclosedTags], - ).markdown, + joinPieces( + $exportChildrenForSelection( + parentNode, + selection, + textFormatTransformers, + textMatchTransformers, + shouldPreserveNewLines, + unclosedTags, + [...unclosableTags, ...unclosedTags], + ).pieces, + ), textNode => - $exportTextFormat( - textNode, - $exportedText(textNode, selection), - textFormatTransformers, - unclosedTags, - unclosableTags, - shouldPreserveNewLines, - selection, + joinPieces( + $exportTextFormat( + textNode, + $exportedText(textNode, selection), + textFormatTransformers, + unclosedTags, + unclosableTags, + shouldPreserveNewLines, + selection, + ), ), ); @@ -298,7 +314,7 @@ function $exportChildrenForSelection( childIncluded = true; } if (childIncluded) { - output.push(result); + pushPiece(output, RAW, result); anyChildIncluded = true; } continue mainLoop; @@ -307,21 +323,20 @@ function $exportChildrenForSelection( if ($isLineBreakNode(child)) { if (childIncluded) { - output.push($exportLineBreak(child)); + pushPiece(output, RAW, $exportLineBreak(child)); anyChildIncluded = true; } } else if ($isTextNode(child)) { if (childIncluded) { - output.push( - $exportTextFormat( - child, - $exportedText(child, selection), - textFormatTransformers, - unclosedTags, - unclosableTags, - shouldPreserveNewLines, - selection, - ), + $exportTextFormat( + child, + $exportedText(child, selection), + textFormatTransformers, + unclosedTags, + unclosableTags, + shouldPreserveNewLines, + selection, + output, ); anyChildIncluded = true; } @@ -334,6 +349,7 @@ function $exportChildrenForSelection( shouldPreserveNewLines, unclosedTags, unclosableTags, + output, ); // extractWithChild: if child has selected descendants, ask parent if it should be included @@ -345,19 +361,19 @@ function $exportChildrenForSelection( childIncluded = true; } + // An element child writes pieces only when it has included ones. if (childIncluded || childResult.shouldInclude) { - output.push(childResult.markdown); anyChildIncluded = true; } } else if ($isDecoratorNode(child)) { if (childIncluded) { - output.push(child.getTextContent()); + pushPiece(output, TEXT, child.getTextContent()); anyChildIncluded = true; } } } - return {markdown: output.join(''), shouldInclude: anyChildIncluded}; + return {pieces: output, shouldInclude: anyChildIncluded}; } function $exportTopLevelElements( @@ -411,7 +427,27 @@ function $exportChildren( unclosableTags?: OpenTag[], shouldPreserveNewLines: boolean = false, ): string { - const output = []; + return joinPieces( + $exportChildPieces( + node, + textTransformersIndex, + textMatchTransformers, + unclosedTags, + unclosableTags, + shouldPreserveNewLines, + ), + ); +} + +function $exportChildPieces( + node: ElementNode, + textTransformersIndex: TextFormatTransformer[], + textMatchTransformers: TextMatchTransformer[], + unclosedTags?: OpenTag[], + unclosableTags?: OpenTag[], + shouldPreserveNewLines: boolean = false, + output: MarkdownPieces = createPieces(), +): MarkdownPieces { const children = node.getChildren(); // keep track of unclosed tags from the very beginning if (!unclosedTags) { @@ -444,55 +480,55 @@ function $exportChildren( shouldPreserveNewLines, ), (textNode, textContent) => - $exportTextFormat( - textNode, - textContent, - textTransformersIndex, - unclosedTags, - unclosableTags, - shouldPreserveNewLines, - null, + joinPieces( + $exportTextFormat( + textNode, + textContent, + textTransformersIndex, + unclosedTags, + unclosableTags, + shouldPreserveNewLines, + null, + ), ), ); if (result != null) { - output.push(result); + pushPiece(output, RAW, result); continue mainLoop; } } if ($isLineBreakNode(child)) { - output.push($exportLineBreak(child)); + pushPiece(output, RAW, $exportLineBreak(child)); } else if ($isTextNode(child)) { - output.push( - $exportTextFormat( - child, - child.getTextContent(), - textTransformersIndex, - unclosedTags, - unclosableTags, - shouldPreserveNewLines, - null, - ), + $exportTextFormat( + child, + child.getTextContent(), + textTransformersIndex, + unclosedTags, + unclosableTags, + shouldPreserveNewLines, + null, + output, ); } else if ($isElementNode(child)) { // empty paragraph returns "" - output.push( - $exportChildren( - child, - textTransformersIndex, - textMatchTransformers, - unclosedTags, - unclosableTags, - shouldPreserveNewLines, - ), + $exportChildPieces( + child, + textTransformersIndex, + textMatchTransformers, + unclosedTags, + unclosableTags, + shouldPreserveNewLines, + output, ); } else if ($isDecoratorNode(child)) { - output.push(child.getTextContent()); + pushPiece(output, TEXT, child.getTextContent()); } } - return output.join(''); + return output; } function $exportLineBreak(node: LineBreakNode): string { @@ -509,7 +545,9 @@ function $exportTextFormat( shouldPreserveNewLines: boolean, // the selection being exported, which ends the text a tag can extend over selection: BaseSelection | null, -): string { + // where the exported pieces go + pieces: MarkdownPieces = createPieces(), +): MarkdownPieces { // This function handles the case of a string looking like this: " foo " // Where it would be invalid markdown to generate: "** foo **" // so the tags go inside the whitespace instead: " **foo** ". @@ -518,19 +556,31 @@ function $exportTextFormat( let output = textContent; if (!isCode) { - // Preserve literal backslashes when preserving source newlines. + // Preserve literal backslashes when preserving source newlines, except + // right before a character escaped here, where they would escape the + // escape instead. // Default mode guards numeric references; named references keep legacy output. // Numeric escapes avoid changing emphasis flanking beside formatting. // Guard ambiguous '<' and '&' at node ends for constructs split across nodes. if (shouldPreserveNewLines) { - output = output.replace(/([*_`~])/g, '\\$1'); + // A whole run of backslashes is matched, so that one before no marker + // is not tried again from each of its backslashes. + output = output.replace( + /\\+|[*_`~]/g, + (escaped, offset: number, text: string) => + escaped[0] !== '\\' + ? `\\${escaped}` + : /[*_`~]/.test(text[offset + escaped.length]) + ? escaped + escaped + : escaped, + ); } else { output = $escapeLiteralBrackets( node, output.replace(/([*_`~\\]|<(?=\S|$)|&(?=#|$))/g, character => /[*_`~\\]/.test(character) ? `\\${character}` - : `&#${character.charCodeAt(0)};`, + : toCharacterReference(character), ), selection, ); @@ -620,27 +670,568 @@ function $exportTextFormat( } else if (!holdsTag) { // If the node is entirely whitespace, we don't apply opening/closing tags // around it, but it must still output any closing tags. - return closingTagsBefore + closingTagsAfter + output; + pushPiece(pieces, CLOSER, closingTagsBefore + closingTagsAfter); + pushPiece(pieces, TEXT, output); + return pieces; } // Flanking Compliance: CommonMark flanking rules require formatting tags to be // adjacent to non-whitespace characters, so openingTags and closingTagsAfter // are placed INSIDE the whitespace boundaries! + // Tags stay inside JavaScript whitespace, which holds CommonMark's and the + // other characters some readers count as whitespace. // (Trimming rather than a regex keeps this linear in long whitespace runs.) const trimmedStart = output.trimStart(); const trimmedOutput = trimmedStart.trimEnd(); const leadingSpace = output.slice(0, output.length - trimmedStart.length); const trailingSpace = trimmedStart.slice(trimmedOutput.length); + pushPiece(pieces, CLOSER, closingTagsBefore); + pushPiece(pieces, TEXT, leadingSpace); + pushPiece(pieces, OPENER, openingTags); + // Inline code is a literal span: a character reference would show as is. + pushPiece(pieces, isCode ? RAW : TEXT, trimmedOutput); + pushPiece(pieces, CLOSER, closingTagsAfter); + pushPiece(pieces, TEXT, trailingSpace); + return pieces; +} + +/** + * Exported inline Markdown, with the format markers kept apart from the text + * until `joinPieces` can see the characters on both sides of each delimiter + * run: `values[i]` is a piece of the output and `kinds[i]` what it is. A + * character reference can stand in for a character of `TEXT`, which is text + * and decorator content; `RAW` is anything else, such as code spans, line + * breaks and the output of transformers, whose syntax a reference could + * break. `OPENER` and `CLOSER` are format markers. + */ +type PieceKind = typeof TEXT | typeof RAW | typeof OPENER | typeof CLOSER; + +interface MarkdownPieces { + kinds: PieceKind[]; + values: string[]; +} + +const TEXT = 0; +const RAW = 1; +const OPENER = 2; +const CLOSER = 3; + +function createPieces(): MarkdownPieces { + return {kinds: [], values: []}; +} + +// Adjacent text (and adjacent raw output) is merged, so that each run of +// markers sits between the pieces whose characters a reader sees beside it. +function pushPiece(pieces: MarkdownPieces, kind: PieceKind, value: string) { + const {kinds, values} = pieces; + const last = kinds.length - 1; + if (value === '') { + return; + } else if (last >= 0 && kinds[last] === kind && !isMarker(kind)) { + values[last] += value; + } else { + kinds.push(kind); + values.push(value); + } +} + +function isMarker(kind: PieceKind | undefined): boolean { + return kind === OPENER || kind === CLOSER; +} + +// Characters a URL or email can hold: GFM ends an autolink at +// whitespace or `<`, and a URL's host and path may hold any other character. +// This takes in the backslash that escapes a marker too. +const LINK_END = /[\s\p{Cc}<]/u; +function isLinkCharacter(character: string): boolean { + return !LINK_END.test(character); +} +/** Where the run of URL characters that ends `text` starts. */ +function linkRunStart(text: string): number { + let start = text.length; + while (start > 0 && isLinkCharacter(text[start - 1])) { + start--; + } + return start; +} +// What starts a URL: `http://`, `https://` (and cmark-gfm's `ftp://`) with +// a host after it, which may start with an emoji but not with other ASCII +// punctuation than `-` and `_`, and `www.`, which micromark links with +// anything after it. GFM readers start `www.` only at the start of a run or +// after `*_~(`, and a scheme after anything but a letter. +const LINK_PREFIX = /(?:https?|ftp):\/\/(?![\s\p{Cc}!-,./:-@[-^`{-~])|www\./giu; +function startsLink(prefix: string, before: string): boolean { + return /^w/i.test(prefix) ? '*_~('.includes(before) : !/[a-z]/i.test(before); +} +// Characters an email can hold (word characters, `.+@-`, and the backslash +// that escapes one), and what makes one: GFM needs a dot in its domain. An +// email takes in no format, and cmark-gfm finds it in the text with its +// references decoded, but micromark only where its `@` is written as is. +const ADDRESS_CHARACTER = /[\w.+@\\-]/; +const ADDRESS = /[\w.+-]@[\w-]+\.[\w-]/; +const LINKISH = /www\.|:\/\/|@/i; +// Characters that GFM readers leave out at the end of a URL. +const LINK_TRAIL = '?!.,:*_~'; + +// Characters that a reference does not stand in for everywhere: CommonMark +// readers and HTML decode references to control characters, unpaired +// surrogates and noncharacters as U+FFFD (or remap them). +const UNREFERENCEABLE = /[\p{Cc}\p{Cs}\p{Noncharacter_Code_Point}]/u; + +/** Whether a character reference can stand in for `character` beside a marker. */ +function isReferenceable(character: string | undefined): boolean { return ( - closingTagsBefore + - leadingSpace + - openingTags + - trimmedOutput + - closingTagsAfter + - trailingSpace + character !== undefined && + isLetterForSomeReader(character) && + !UNREFERENCEABLE.test(character) ); } +/** + * Adjacent markers between the pieces at `before` and `after`: whether the + * first delimiter run opens a tag, the marker character after it within the + * markers (if any), whether the last delimiter run closes a tag, and the + * marker character before it. + */ +type MarkerRun = [ + before: number, + after: number, + opens: boolean, + afterFirst: string | undefined, + closes: boolean, + beforeLast: string | undefined, +]; + +/** + * Joins exported pieces into Markdown. A delimiter run between a word + * character and punctuation can only open or close on the punctuation side + * (CommonMark flanking), so plain `a` before a bold `*` cannot go out as + * `a**\***`. The word character beside such a run goes out as a numeric + * reference instead, whose `&` or `;` is punctuation: `a**\***`. + */ +function joinPieces({kinds, values}: MarkdownPieces): string { + const text = values.join(''); + if (kinds.every(kind => kind === TEXT)) { + return text; + } + const [encodeStart, encodeEnd] = decideReferences(kinds, values); + // The code units of `text` that go out as references. + const isReference = new Uint8Array(text.length); + for (let index = 0, base = 0; index < kinds.length; index++) { + const value = values[index]; + const start = encodeStart[index] ? characterAt(value, 0)!.length : 0; + let end = value.length; + if (encodeEnd[index] && end > start) { + end -= characterBefore(value, end)!.length; + } + // A backslash left unescaped (when preserving newlines, or in decorator + // text) would escape the reference, marker or raw punctuation written + // after it, or end the line with a hard break. + if ( + kinds[index] === TEXT && + (isMarker(kinds[index + 1]) || + (kinds[index + 1] === RAW && ESCAPABLE.test(values[index + 1]))) && + end > start && + isEscaped(value, end) + ) { + end--; + } + isReference.fill(1, base, base + start); + isReference.fill(1, base + end, (base += value.length)); + } + spoilLinks(kinds, values, isReference); + let markdown = ''; + let from = 0; + for ( + let at = isReference.indexOf(1); + at !== -1; + at = isReference.indexOf(1, from) + ) { + const character = characterAt(text, at)!; + markdown += text.slice(from, at) + toCharacterReference(character); + from = at + character.length; + } + return markdown + text.slice(from); +} + +// What a backslash before it escapes: ASCII punctuation or a line ending. +const ESCAPABLE = /^[!-/:-@[-`{-~\n\r]/; + +/** + * Raw output as the bare-URL scan sees it. It keeps whitespace and `<`, which + * end a URL, `*_~(`, after which `www.` starts one, and ASCII letters, before + * which no scheme starts one, as `x`, and is `#` elsewhere. So no URL starts + * inside it, but one before it runs on into it, as GFM readers run a URL on + * through a code span or a link. + */ +function maskRaw(value: string): string { + return value.replace(/[a-z]/gi, 'x').replace(/[^\sx<*_~(]/g, '#'); +} + +/** + * Finds the bare URLs and `www.` hosts that GFM readers would link in the + * joined pieces, and that a reference would land in or that would take in + * the markers, code span or other raw output after them: the reader would + * keep the reference's characters in the link, or the markers and code as + * text. Such a URL is spoiled instead, by writing the `.` of `www.` or the + * `:` after its scheme as a reference, so that no reader links it and its + * text and formats come through. So is a URL that a reference before it + * would start. `isReference` marks the code units of the joined text that go + * out as references, and the ones that spoil a link are marked in it too. + */ +function spoilLinks( + kinds: PieceKind[], + values: string[], + isReference: Uint8Array, +): void { + // Every URL and email starts with one of these. With no reference written, + // only a URL that runs on to the end of its text, into the markers or raw + // output after it, is spoiled. + const hasReferences = isReference.includes(1); + if ( + !values.some( + (value, i) => + kinds[i] === TEXT && + LINKISH.test(value) && + (hasReferences || LINKISH.test(value.slice(linkRunStart(value)))), + ) + ) { + return; + } + let text = ''; + // Whether each code unit is not text, as `1`. + let isFormat = ''; + for (let i = 0; i < values.length; i++) { + text += kinds[i] === RAW ? maskRaw(values[i]) : values[i]; + isFormat += (kinds[i] === TEXT ? '0' : '1').repeat(values[i].length); + } + // Running counts of referenced code units and of code units that are not + // text, so that whether any lie in a span is a subtraction. + const references = new Uint32Array(text.length + 1); + const formats = new Uint32Array(text.length + 1); + for (let i = 0; i < text.length; i++) { + references[i + 1] = references[i] + isReference[i]; + formats[i + 1] = formats[i] + Number(isFormat[i] === '1'); + } + const has = (sums: Uint32Array, from: number, to: number) => + from < to && sums[to] > sums[from]; + // A reference in an email would cut a micromark reader's link short, so + // each `@` of one that holds a reference is spoiled. + const spoilEmails = (from: number, to: number) => { + for (let start = from; start < to; ) { + let end = start; + while (end < to && ADDRESS_CHARACTER.test(text[end])) { + end++; + } + if ( + has(references, start, end) && + ADDRESS.test(text.slice(start, end).replace(/\\/g, '')) + ) { + for (let i = start; i < end; i++) { + // A raw backslash before an `@` would escape the reference's `&`. + if ( + text[i] === '@' && + !isReference[i] && + isFormat[i] === '0' && + !isEscaped(text, i) + ) { + isReference[i] = 1; + } + } + } + start = Math.max(end, start + 1); + } + }; + for (let start = 0; start < text.length; ) { + let end = start; + while ( + end < text.length && + (isLinkCharacter(text[end]) || isReference[end] === 1) + ) { + end++; + } + if (end === start) { + start++; + continue; + } + // A URL runs to the end of the run, less the punctuation at its end. It + // takes in a backslash before that punctuation, which then no longer + // escapes it, so the punctuation can join the markers after it. + let trimmed = end; + while (trimmed > start && LINK_TRAIL.includes(text[trimmed - 1])) { + trimmed--; + } + if (isEscaped(text, trimmed) && has(formats, trimmed, end)) { + trimmed = end; + } + // Where the text that is not part of a URL starts, which may hold an + // email, and where the next URL may start. + let rest = start; + for (const prefix of text.slice(start, end).matchAll(LINK_PREFIX)) { + const at = start + prefix.index; + if (at < rest) { + continue; + } + // A reference before the prefix, or a spoiled `.` or `:`, puts `;` + // there. + const before = at > start ? text[at - 1] : ''; + const wasLink = at === start || startsLink(prefix[0], before); + if ( + at > start && + !startsLink(prefix[0], isReference[at - 1] ? ';' : before) + ) { + continue; + } + spoilEmails(rest, at); + if (wasLink && !has(references, at, end) && !has(formats, at, trimmed)) { + rest = end; + break; + } + // A reference in the prefix spoils it already. A URL whose `.` or `:` + // is a custom format's marker is left as it is. + const spoil = at + prefix[0].search(/[.:]/); + if (!has(references, at, at + prefix[0].length)) { + if (isFormat[spoil] === '1') { + rest = end; + break; + } + isReference[spoil] = 1; + } + rest = at + prefix[0].length; + } + spoilEmails(rest, end); + start = end; + } +} + +/** + * Walks the delimiter run at the edge of the markers from `from` to `to`, + * forward when `step` is 1 and backward when it is -1. Returns whether a + * marker of `kind` is part of that run, and the character past it within the + * markers, if any. + */ +function scanDelimiterRun( + kinds: PieceKind[], + values: string[], + from: number, + to: number, + step: 1 | -1, + kind: PieceKind, +): [boolean, string | undefined] { + const edge = values[from]; + const delimiter = edge[step === 1 ? 0 : edge.length - 1]; + let hasKind = false; + for (let i = from; i !== to + step; i += step) { + const value = values[i]; + for ( + let c = step === 1 ? 0 : value.length - 1; + c >= 0 && c < value.length; + c += step + ) { + if (value[c] !== delimiter) { + return [hasKind, value[c]]; + } + hasKind ||= kinds[i] === kind; + } + } + return [hasKind, undefined]; +} + +/** + * Marks the text pieces whose first (in the first array) or last character + * has to go out as a reference for the marker run beside it to open or close. + */ +function decideReferences( + kinds: PieceKind[], + values: string[], +): [Uint8Array, Uint8Array] { + const count = kinds.length; + const encodeStart = new Uint8Array(count); + const encodeEnd = new Uint8Array(count); + const runs: MarkerRun[] = []; + // The run that follows and the run that precedes each piece, or -1. (A + // typed array ignores a write out of its bounds.) + const runAfter = new Int32Array(count).fill(-1); + const runBefore = new Int32Array(count).fill(-1); + for (let start = 0; start < count; start++) { + if (isMarker(kinds[start])) { + let end = start; + while (isMarker(kinds[end + 1])) { + end++; + } + runAfter[start - 1] = runBefore[end + 1] = runs.length; + // Markers of one character form one delimiter run, and only the first + // and the last delimiter run touch text. + runs.push([ + start - 1, + end + 1, + ...scanDelimiterRun(kinds, values, start, end, 1, OPENER), + ...scanDelimiterRun(kinds, values, end, start, -1, CLOSER), + ]); + start = end; + } + } + const firstChar = (index: number) => characterAt(values[index], 0)!; + const lastChar = (index: number) => + characterBefore(values[index], values[index].length)!; + // A one-character piece written as a reference reads as punctuation at + // both ends: `a`. + const isOneCharacter = (index: number) => + firstChar(index).length === values[index].length; + const isEncoded = (index: number) => + (encodeStart[index] || encodeEnd[index]) && isOneCharacter(index); + const startChar = (index: number) => + index >= count + ? undefined + : encodeStart[index] || isEncoded(index) + ? '&' + : firstChar(index); + const endChar = (index: number) => + index < 0 + ? undefined + : encodeEnd[index] || isEncoded(index) + ? ';' + : lastChar(index); + // Whether the last (or first) character of the piece at `index` can go out + // as a reference. + const canReference = (index: number, atEnd: boolean): boolean => + kinds[index] === TEXT && + isReferenceable(atEnd ? endChar(index) : startChar(index)); + // Writing a one-character piece as a reference makes its other edge + // punctuation too. That can stop the delimiter run touching that edge from + // flanking unless the piece past it is written as a reference as well, and + // so on along a chain of one-character pieces. Where the chain ends at + // something that cannot be (raw output, say), writing the first reference + // would only trade one lost format for another, so it is not written. + // Each piece's answer is remembered for each direction, so each piece is + // walked at most twice. Raw output and unreferenceable characters stay + // what they are. A remembered block could only lift if a piece on its + // chain were later written as a reference from the other side, which takes + // a run of one marker character that both closes and opens; then a + // reference that would have kept a format is not written. + // 0 while unknown, then 1 when it spreads and 2 when it is blocked. + const spreads = [new Uint8Array(count), new Uint8Array(count)]; + const canSpread = (index: number, leftward: boolean): boolean => { + const known = spreads[Number(leftward)]; + const visited: number[] = []; + let result = true; + for (let piece = index; ; ) { + if (known[piece] !== 0) { + result = known[piece] === 1; + break; + } + visited.push(piece); + const runIndex = (leftward ? runBefore : runAfter)[piece]; + if (runIndex === -1) { + break; + } + const [before, after, opens, afterFirst, closes, beforeLast] = + runs[runIndex]; + const next = leftward ? before : after; + // Only a run of a single delimiter character touches both pieces. + if ( + !(leftward + ? opens && afterFirst === undefined + : closes && beforeLast === undefined) || + isPortablyFlankedBy( + leftward ? '&' : ';', + leftward ? endChar(next) : startChar(next), + ) + ) { + break; + } + if (!canReference(next, leftward)) { + result = false; + break; + } + if (!isOneCharacter(next)) { + break; + } + piece = next; + } + for (const piece of visited) { + known[piece] = result ? 1 : 2; + } + return result; + }; + // Whether a reference beside the run, before it when `opening` and after it + // otherwise, lets its first delimiter run open (or its last close) where it + // could not, and whether that reference can be written. `_` between two + // letters cannot open either way. + const isHelpedBy = ( + [before, after, opens, afterFirst, closes, beforeLast]: MarkerRun, + opening: boolean, + ): boolean => { + const piece = opening ? before : after; + if (!(opening ? opens : closes) || !canReference(piece, opening)) { + return false; + } + const side = opening ? 'canOpen' : 'canClose'; + const marker = opening + ? values[before + 1][0] + : values[after - 1][values[after - 1].length - 1]; + const previous = (!opening && beforeLast) || endChar(before); + const next = (opening && afterFirst) || startChar(after); + return ( + !getPortableEmphasisSides(marker, previous, next)[side] && + getPortableEmphasisSides( + marker, + opening ? ';' : previous, + opening ? next : '&', + )[side] && + (!isOneCharacter(piece) || canSpread(piece, opening)) + ); + }; + // Text does not escape every tag character (the `=` of `==`, say), and one + // right beside a marker of the same character would join its delimiter + // run, so it goes out as a reference. + for (let i = 0; i < count; i++) { + const value = values[i]; + if (kinds[i] === TEXT) { + if ( + isMarker(kinds[i + 1]) && + value[value.length - 1] === values[i + 1][0] && + !isEscaped(value, value.length - 1) + ) { + encodeEnd[i] = 1; + } + if ( + isMarker(kinds[i - 1]) && + value[0] === values[i - 1][values[i - 1].length - 1] + ) { + encodeStart[i] = 1; + } + } + } + const pending = runs.map((_run, index) => index); + // Writing a one-character piece as a reference changes what the run on its + // other side sees, so that run is decided again. Each piece changes at most + // twice, so this stays linear. + while (pending.length > 0) { + const index = pending.pop()!; + const run = runs[index]; + const [before, after, opens, afterFirst] = run; + if (isHelpedBy(run, true)) { + encodeEnd[before] = 1; + if (isEncoded(before) && runBefore[before] !== -1) { + pending.push(runBefore[before]); + } + } + if (isHelpedBy(run, false)) { + encodeStart[after] = 1; + // An opener that sees the same character, of a run that both closes + // and opens, may now need its own reference. + if (opens && afterFirst === undefined) { + pending.push(index); + } + if (isEncoded(after) && runAfter[after] !== -1) { + pending.push(runAfter[after]); + } + } + } + return [encodeStart, encodeEnd]; +} + /** * Escape only the literal brackets that would otherwise turn into Markdown * syntax, and leave every other bracket as typed, so `[1]` citations and diff --git a/packages/lexical-markdown/src/MarkdownShortcuts.ts b/packages/lexical-markdown/src/MarkdownShortcuts.ts index 8b40fbc0b39..64e498cac42 100644 --- a/packages/lexical-markdown/src/MarkdownShortcuts.ts +++ b/packages/lexical-markdown/src/MarkdownShortcuts.ts @@ -42,7 +42,13 @@ import { type Transformer, TRANSFORMERS, } from './MarkdownTransformers'; -import {indexBy, PUNCTUATION_OR_SPACE, transformersByType} from './utils'; +import { + characterAt, + characterBefore, + indexBy, + isPunctuationOrSpace, + transformersByType, +} from './utils'; function runElementTransformers( parentNode: ElementNode, @@ -248,12 +254,12 @@ function $runTextFormatTransformers( } // Some tags can not be used within words, hence should have newline/space/punctuation after it - const afterCloseTagChar = textContent[closeTagEndIndex + 1]; + const afterCloseTagChar = characterAt(textContent, closeTagEndIndex + 1); if ( matcher.intraword === false && afterCloseTagChar && - !PUNCTUATION_OR_SPACE.test(afterCloseTagChar) + !isPunctuationOrSpace(afterCloseTagChar) ) { continue; } @@ -313,12 +319,15 @@ function $runTextFormatTransformers( } // Some tags can not be used within words, hence should have newline/space/punctuation before it - const beforeOpenTagChar = prevOpenNodeText[openTagStartIndex - 1]; + const beforeOpenTagChar = characterBefore( + prevOpenNodeText, + openTagStartIndex, + ); if ( matcher.intraword === false && beforeOpenTagChar && - !PUNCTUATION_OR_SPACE.test(beforeOpenTagChar) + !isPunctuationOrSpace(beforeOpenTagChar) ) { continue; } diff --git a/packages/lexical-markdown/src/MarkdownTransformers.ts b/packages/lexical-markdown/src/MarkdownTransformers.ts index 7ad38353526..3a38ce92fd9 100644 --- a/packages/lexical-markdown/src/MarkdownTransformers.ts +++ b/packages/lexical-markdown/src/MarkdownTransformers.ts @@ -50,7 +50,12 @@ import { } from 'lexical'; import {scanCodeSpans} from './importTextFormatTransformer'; -import {isEmptyParagraph, isEscaped, unescapeText} from './utils'; +import { + isEmptyParagraph, + isEscaped, + toCharacterReference, + unescapeText, +} from './utils'; export type Transformer = | ElementTransformer @@ -1195,7 +1200,7 @@ function escapeLinkPart(value: string, syntax: RegExp): string { .replace(/&(?=#\d+;|#[Xx][\dA-Fa-f]+;|[A-Za-z][\dA-Za-z]*;)/g, '&') .replace(syntax, '\\$&') .replace(/`/g, '\\`') - .replace(/[\r\n]/g, lineEnding => `&#${lineEnding.charCodeAt(0)};`); + .replace(/[\r\n]/g, toCharacterReference); } // Mask escaped/code brackets without changing source offsets, then find an inner @@ -1336,6 +1341,8 @@ export const LINK: TextMatchTransformer = { const linkTitle = rawLinkTitle != null ? unescapeText(rawLinkTitle) : undefined; const linkNode = $createLinkNode(linkUrl, {title: linkTitle}); + // Import passes the label from its real opener, so this finds none, but + // a custom caller may pass a match that starts at an earlier `[`. const openBracketIndex = getLinkLabelOpener(linkText); if (openBracketIndex === null) return; let parsedLinkText = linkText; diff --git a/packages/lexical-markdown/src/__tests__/unit/LexicalMarkdown.test.ts b/packages/lexical-markdown/src/__tests__/unit/LexicalMarkdown.test.ts index 22703b2ac57..90d0778401c 100644 --- a/packages/lexical-markdown/src/__tests__/unit/LexicalMarkdown.test.ts +++ b/packages/lexical-markdown/src/__tests__/unit/LexicalMarkdown.test.ts @@ -6,6 +6,7 @@ * */ import {$createCodeNode, CodeNode} from '@lexical/code-core'; +import {buildEditorFromExtensions} from '@lexical/extension'; import {createHeadlessEditor} from '@lexical/headless'; import {$generateHtmlFromNodes, $generateNodesFromDOM} from '@lexical/html'; import invariant from '@lexical/internal/invariant'; @@ -26,13 +27,19 @@ import { CODE, type ElementTransformer, HEADING, + ITALIC_STAR, LINK, type MultilineElementTransformer, registerMarkdownShortcuts, + type TextFormatTransformer, type TextMatchTransformer, type Transformer, TRANSFORMERS, } from '@lexical/markdown'; +import { + $convertFromMarkdownString as $convertFromMdastMarkdownString, + MdastGfmExtension, +} from '@lexical/mdast'; import { $createHeadingNode, $createQuoteNode, @@ -63,6 +70,7 @@ import { $setState, COMPOSITION_END_TAG, type CreateEditorArgs, + defineExtension, type ElementNode, KEY_ENTER_COMMAND, type LexicalEditor, @@ -566,6 +574,25 @@ describe('Markdown', () => { html: '

$a $b

', md: '`$a` `$b`', }, + { + // Emphasis before a code span is applied first, and a link that takes + // part of the span still loses to it. + html: '

[x a y](z)

', + md: '[x *a* `y](z)`', + skipExport: true, + }, + { + html: '

a [b c](d)

', + md: '*a [b* `c](d)`', + skipExport: true, + }, + { + // A delimiter left over beside emphasis that ends before a code span + // stays literal. + html: '

a*x*

', + md: '*a**`x`*', + skipExport: true, + }, { // Inline code containing a backtick must use a longer fence (CommonMark // code spans) and pad with spaces so it round-trips losslessly. @@ -960,8 +987,9 @@ describe('Markdown', () => { { html: '

[hello]](https://lexical.dev)world

', md: '[hello]](https://lexical.dev)[world](https://lexical.dev)', + // GFM readers would run the URL on into the link after it. mdAfterExport: - '[hello]](https://lexical.dev)[world](https://lexical.dev)', + '[hello]](https://lexical.dev)[world](https://lexical.dev)', }, { html: '

[helloworld

', @@ -1640,6 +1668,17 @@ describe('Markdown', () => { ); }); + it('round-trips italic then bold italic between punctuation', () => { + expectRoundTrip( + [ + [')', ITALIC], + ['(', BOLD], + [')', BOLD | ITALIC], + ], + '*)***(*)***', + ); + }); + it('round-trips strikethrough overlapping bold', () => { expectRoundTrip( [ @@ -4611,6 +4650,1875 @@ describe('link import', () => { } }); +describe('emphasis next to Unicode punctuation and symbols', () => { + // CommonMark counts every character in the Unicode P (punctuation) and S + // (symbol) general categories as punctuation when it decides whether a + // delimiter run is left- or right-flanking, not only the ASCII ones. + // https://spec.commonmark.org/0.31.2/#unicode-punctuation-character + type Run = [text: string, format: string]; + + const EN_DASH = '\u2013'; // Pd + const LEFT_DOUBLE_QUOTE = '\u201C'; // Pi + const RIGHT_DOUBLE_QUOTE = '\u201D'; // Pf + const LOW_DOUBLE_QUOTE = '\u201E'; // Ps + const EURO_SIGN = '\u20AC'; // Sc + const LATIN_E_ACUTE = '\u00E9'; // Ll, a letter + const CYRILLIC_BE = '\u0431'; // Ll, a letter + const HERE_LINK = '[here](https://example.com)'; + + function importRuns(md: string): Run[] { + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(md, TRANSFORMERS), { + discrete: true, + }); + return editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map((node): Run => { + const formats = (['bold', 'italic'] as const).filter(format => + node.hasFormat(format), + ); + return [node.getTextContent(), formats.join(' ') || 'plain']; + }), + ); + } + + // Each expected result matches what micromark, a CommonMark parser, produces + // for the same input. + const CASES: [md: string, runs: Run[]][] = [ + [ + `See ${EN_DASH}**${HERE_LINK}** now.`, + [ + [`See ${EN_DASH}`, 'plain'], + ['here', 'bold'], + [' now.', 'plain'], + ], + ], + [ + `See ${LEFT_DOUBLE_QUOTE}**${HERE_LINK}**${RIGHT_DOUBLE_QUOTE} now.`, + [ + [`See ${LEFT_DOUBLE_QUOTE}`, 'plain'], + ['here', 'bold'], + [`${RIGHT_DOUBLE_QUOTE} now.`, 'plain'], + ], + ], + [ + `See ${LOW_DOUBLE_QUOTE}**${HERE_LINK}**${LEFT_DOUBLE_QUOTE} now.`, + [ + [`See ${LOW_DOUBLE_QUOTE}`, 'plain'], + ['here', 'bold'], + [`${LEFT_DOUBLE_QUOTE} now.`, 'plain'], + ], + ], + [ + `See ${EURO_SIGN}**${HERE_LINK}** now.`, + [ + [`See ${EURO_SIGN}`, 'plain'], + ['here', 'bold'], + [' now.', 'plain'], + ], + ], + [ + // Only the closing run touches a Unicode punctuation character. + `See **${HERE_LINK}**${EN_DASH}now.`, + [ + ['See ', 'plain'], + ['here', 'bold'], + [`${EN_DASH}now.`, 'plain'], + ], + ], + [ + `See ${EN_DASH}*${HERE_LINK}* now.`, + [ + [`See ${EN_DASH}`, 'plain'], + ['here', 'italic'], + [' now.', 'plain'], + ], + ], + [ + `See ${EN_DASH}_${HERE_LINK}_ now.`, + [ + [`See ${EN_DASH}`, 'plain'], + ['here', 'italic'], + [' now.', 'plain'], + ], + ], + [ + `See ${EN_DASH}__${HERE_LINK}__ now.`, + [ + [`See ${EN_DASH}`, 'plain'], + ['here', 'bold'], + [' now.', 'plain'], + ], + ], + // Unchanged: ASCII punctuation and whitespace still work. + [ + `See -**${HERE_LINK}** now.`, + [ + ['See -', 'plain'], + ['here', 'bold'], + [' now.', 'plain'], + ], + ], + [ + `See **${HERE_LINK}** now.`, + [ + ['See ', 'plain'], + ['here', 'bold'], + [' now.', 'plain'], + ], + ], + // Unchanged: a delimiter run inside a word. + [ + 'a**b**', + [ + ['a', 'plain'], + ['b', 'bold'], + ], + ], + [ + '**a**b', + [ + ['a', 'bold'], + ['b', 'plain'], + ], + ], + ['snake_case_word', [['snake_case_word', 'plain']]], + // Unchanged: a letter is not punctuation, whatever the script. + [ + `${LATIN_E_ACUTE}**${HERE_LINK}**`, + [ + [`${LATIN_E_ACUTE}**`, 'plain'], + ['here', 'plain'], + ['**', 'plain'], + ], + ], + [ + `${CYRILLIC_BE}**${HERE_LINK}**`, + [ + [`${CYRILLIC_BE}**`, 'plain'], + ['here', 'plain'], + ['**', 'plain'], + ], + ], + ]; + + for (const [md, runs] of CASES) { + it(`imports ${JSON.stringify(md)}`, () => { + expect(importRuns(md)).toEqual(runs); + }); + } + + // Importing the bold link as text used to leave the asterisks in the text, + // so the export escaped them. The exporter writes the bold inside the link. + const EXPORT_CASES: [md: string, exported: string][] = [ + [ + `See ${EN_DASH}**${HERE_LINK}** now.`, + `See ${EN_DASH}[**here**](https://example.com) now.`, + ], + [ + `See ${LEFT_DOUBLE_QUOTE}**${HERE_LINK}**${RIGHT_DOUBLE_QUOTE} now.`, + `See ${LEFT_DOUBLE_QUOTE}[**here**](https://example.com)${RIGHT_DOUBLE_QUOTE} now.`, + ], + ]; + + for (const [md, exported] of EXPORT_CASES) { + it(`exports ${JSON.stringify(md)} without escaped asterisks`, () => { + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(md, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => $convertToMarkdownString(TRANSFORMERS)), + ).toBe(exported); + }); + } + + // Characters outside the Basic Multilingual Plane are read whole: an emoji + // is a symbol (So), so it is punctuation too. markdown-it agrees; micromark + // reads UTF-16 code units here. + const GRINNING_FACE = '\u{1F600}'; + it.each([ + [`a**${GRINNING_FACE}**`, [[`a**${GRINNING_FACE}**`, 'plain']]], + [`**${GRINNING_FACE}**a`, [[`**${GRINNING_FACE}**a`, 'plain']]], + [ + `a **${GRINNING_FACE}**`, + [ + ['a ', 'plain'], + [GRINNING_FACE, 'bold'], + ], + ], + // U+FEFF is a format character (Cf), not CommonMark whitespace. + // markdown-it agrees; micromark uses JavaScript's `\s`. + [ + `x\uFEFF**${LEFT_DOUBLE_QUOTE}a${RIGHT_DOUBLE_QUOTE}**`, + [[`x\uFEFF**${LEFT_DOUBLE_QUOTE}a${RIGHT_DOUBLE_QUOTE}**`, 'plain']], + ], + ] as [string, Run[]][])('imports %j', (md, runs) => { + expect(importRuns(md)).toEqual(runs); + }); + + // A tag between a word character and punctuation cannot open or close, so + // the word character goes out as a numeric reference, whose `&` or `;` is + // punctuation. + it.each([ + [ + [ + ['foo', []], + [`${LEFT_DOUBLE_QUOTE}bar${RIGHT_DOUBLE_QUOTE}`, ['bold']], + ['baz', []], + ], + `foo**${LEFT_DOUBLE_QUOTE}bar${RIGHT_DOUBLE_QUOTE}**baz`, + ], + [ + [ + ['a', []], + [GRINNING_FACE, ['italic']], + ['b', []], + ], + `a*${GRINNING_FACE}*b`, + ], + [ + [ + ['a', []], + ['x', ['bold', 'code']], + ['b', []], + ], + 'a**`x`**b', + ], + [ + [ + ['x.', ['bold']], + ['b', []], + ['(y)', ['bold']], + ], + '**x.**b**(y)**', + ], + // Italic closes and reopens around the bold's end. + [ + [ + ['b', ['bold', 'italic']], + [`${LEFT_DOUBLE_QUOTE}y${RIGHT_DOUBLE_QUOTE}`, ['italic']], + ], + '***b****โ€œyโ€*', + ], + // Markers of different characters are separate delimiter runs. + [ + [ + ['a', ['bold']], + ['!', ['strikethrough']], + ], + '**a**~~!~~', + ], + // A one-character reference reads as punctuation on its other side too, + // so the run there is decided again. + [ + [ + ['x', []], + ['a', ['bold']], + [`${LEFT_DOUBLE_QUOTE}b${RIGHT_DOUBLE_QUOTE}`, ['italic']], + ], + 'x**a***โ€œbโ€*', + ], + [ + [ + [`${LEFT_DOUBLE_QUOTE}a${RIGHT_DOUBLE_QUOTE}`, ['bold', 'italic']], + ['b', ['italic']], + ['c', []], + ], + '***โ€œaโ€****b*c', + ], + [ + [ + ['x', []], + ['a', ['bold']], + [`${LEFT_DOUBLE_QUOTE}b${RIGHT_DOUBLE_QUOTE}`, ['bold', 'italic']], + ], + 'x**a*โ€œbโ€***', + ], + // An `@` that is not part of a link where the text ends does not stop + // the reference. + [ + [ + ['่”็ณปๆˆ‘@ๅพฎๅšไฝ ๅฅฝ', []], + [`${LEFT_DOUBLE_QUOTE}้‡่ฆ${RIGHT_DOUBLE_QUOTE}`, ['bold']], + ], + `่”็ณปๆˆ‘@ๅพฎๅšไฝ 好**${LEFT_DOUBLE_QUOTE}้‡่ฆ${RIGHT_DOUBLE_QUOTE}**`, + ], + [ + [ + ['(a@b.co)and', []], + [`${LEFT_DOUBLE_QUOTE}x${RIGHT_DOUBLE_QUOTE}`, ['bold']], + ], + `(a@b.co)and**${LEFT_DOUBLE_QUOTE}x${RIGHT_DOUBLE_QUOTE}**`, + ], + // A word character beside a word character needs no reference. + [ + [ + ['a', []], + ['b', ['bold']], + ['c', []], + ], + 'a**b**c', + ], + // A tag character that text does not escape is kept out of the marker. + [ + [ + ['a=', []], + [`${LEFT_DOUBLE_QUOTE}b${RIGHT_DOUBLE_QUOTE}`, ['highlight']], + ['=c', []], + ], + 'a===โ€œbโ€===c', + ], + // Emphasis that ends before a code span is read before the span. + [ + [ + ['a.', ['bold']], + ['x', []], + ['c', ['bold', 'code']], + ], + '**a.**x**`c`**', + ], + [ + [ + [LEFT_DOUBLE_QUOTE, ['bold']], + ['.b ', ['italic']], + ['c', ['italic', 'code']], + ], + '**โ€œ***.b `c`*', + ], + ] as [[string, TextFormatType[]][], string][])( + 'exports and reimports %j as %j', + (runs, exported) => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + ...runs.map(([text, formats]) => { + const node = $createTextNode(text); + formats.forEach(format => node.toggleFormat(format)); + return node; + }), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS), + ); + expect(markdown).toBe(exported); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [ + node.getTextContent(), + ( + [ + 'bold', + 'italic', + 'strikethrough', + 'highlight', + 'code', + ] as const + ).filter(format => node.hasFormat(format)), + ]), + ), + ).toEqual(runs); + }, + ); + + it('guards a word character after a raw backslash when preserving newlines', () => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('x\\b'), + $createTextNode( + `${LEFT_DOUBLE_QUOTE}y${RIGHT_DOUBLE_QUOTE}`, + ).toggleFormat('bold'), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS, undefined, true), + ); + expect(markdown).toBe( + `x\b**${LEFT_DOUBLE_QUOTE}y${RIGHT_DOUBLE_QUOTE}**`, + ); + editor.update( + () => $convertFromMarkdownString(markdown, TRANSFORMERS, undefined, true), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.hasFormat('bold')]), + ), + ).toEqual([ + ['x\\b', false], + [`${LEFT_DOUBLE_QUOTE}y${RIGHT_DOUBLE_QUOTE}`, true], + ]); + }); + + it.each([ + ['bold', 'a\**b**'], + ['italic', 'a\*b*'], + ] as [TextFormatType, string][])( + 'keeps a raw backslash from escaping a %s marker when preserving newlines', + (format, exported) => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('a\\'), + $createTextNode('b').toggleFormat(format), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS, undefined, true), + ); + expect(markdown).toBe(exported); + editor.update( + () => + $convertFromMarkdownString(markdown, TRANSFORMERS, undefined, true), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.hasFormat(format)]), + ), + ).toEqual([ + ['a\\', false], + ['b', true], + ]); + }, + ); + + it('keeps raw backslashes from escaping an escape when preserving newlines', () => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('a\\\\_b_'), + $createTextNode('c').toggleFormat('bold'), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS, undefined, true), + ); + expect(markdown).toBe('a\\\\\\\\\\_b\\_**c**'); + editor.update( + () => $convertFromMarkdownString(markdown, TRANSFORMERS, undefined, true), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.hasFormat('bold')]), + ), + ).toEqual([ + ['a\\\\_b_', false], + ['c', true], + ]); + }); + + it('writes no reference that would break a run beside raw output', () => { + // Writing `a` as a reference for the italic would leave the bold's `**` + // between the raw `x` and the reference's `&`, unable to open. + const MENTION: TextMatchTransformer = { + dependencies: [], + export: node => + $isTextNode(node) && node.getTextContent() === '@x' ? '@x' : null, + regExp: /@x$/, + replace: () => {}, + type: 'text-match', + }; + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('@x'), + $createTextNode('a').toggleFormat('bold'), + $createTextNode(`${LEFT_DOUBLE_QUOTE}y${RIGHT_DOUBLE_QUOTE}`) + .toggleFormat('bold') + .toggleFormat('italic'), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString([MENTION, ...TRANSFORMERS]), + ); + expect(markdown).not.toContain('a'); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .some( + node => + node.getTextContent().startsWith('a') && node.hasFormat('bold'), + ), + ), + ).toBe(true); + }); + + // GFM readers would link a URL here, keeping a reference inside as text or + // taking markers or a code span into it, so export writes the `.` or `:` of + // its prefix (or the `@` of an email) as a reference too, and no reader + // links it as written (@lexical/mdast links the decoded text, which keeps + // the formats). + it.each<[string, [string, number][], string]>([ + [ + 'a `www.` host', + [ + ['see www.example.com', 0], + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see www.example.com**\u201cx\u201d**', + ], + [ + 'an email', + [ + ['see me@example.com', 0], + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see me@example.com**\u201cx\u201d**', + ], + [ + 'a URL', + [ + ['see https://example.com/a', 0], + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see https://example.com/a**\u201cx\u201d**', + ], + [ + 'the resource of an `xmpp:` address', + [ + ['see xmpp:a@b.co/res', 0], + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see xmpp:a@b.co/res**\u201cx\u201d**', + ], + [ + 'a `www.` host after `_`', + [ + ['see x_www.example.co', 0], + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see x\\_www.example.co**\u201cx\u201d**', + ], + [ + 'a URL with a Unicode host', + [ + ['see https://\u4f8b\u3048.jp/a', 0], + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see https://\u4f8b\u3048.jp/a**\u201cx\u201d**', + ], + [ + 'a URL that holds an emoji', + [ + ['see https://example.com/a\u{1f600}b', 0], + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see https://example.com/a\u{1f600}b**\u201cx\u201d**', + ], + [ + 'the start of a `www.` host, which its reference spoils', + [ + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ['www.example.com', 0], + ], + '**\u201cx\u201d**www.example.com', + ], + [ + 'the start of a URL, which its reference spoils', + [ + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ['https://example.com', 0], + ], + '**\u201cx\u201d**https://example.com', + ], + [ + 'the start of `mailto:`, which its reference spoils', + [ + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ['mailto:me@example.com', 0], + ], + '**\u201cx\u201d**mailto:me@example.com', + ], + [ + 'the start of an email', + [ + ['\u201cx\u201d', TEXT_TYPE_TO_FORMAT.bold], + ['me@example.com', 0], + ], + '**\u201cx\u201d**me@example.com', + ], + [ + 'a URL that a reference before it would start', + [ + ['.', TEXT_TYPE_TO_FORMAT.bold], + ['ahttp://x.co', 0], + ], + '**.**ahttp://x.co', + ], + [ + '`mailto:`, which a reference before it does not change', + [ + ['.', TEXT_TYPE_TO_FORMAT.bold], + ['bmailto:a@b.co', 0], + ], + '**.**bmailto:a@b.co', + ], + [ + 'a URL right after a code span', + [ + ['x ', 0], + ['a', TEXT_TYPE_TO_FORMAT.code], + ['http://x.co/b', 0], + ['\u201cq\u201d', TEXT_TYPE_TO_FORMAT.bold], + [' y', 0], + ], + 'x `a`http://x.co/b**\u201cq\u201d** y', + ], + [ + 'an email before a URL', + [ + ['x a@b.co', 0], + ['\u201cq\u201d', TEXT_TYPE_TO_FORMAT.bold], + [',http://x.co/ y', 0], + ], + 'x a@b.co**\u201cq\u201d**,http://x.co/ y', + ], + [ + 'a URL that would take in a code span', + [ + ['see https://a.com/', 0], + ['v2', TEXT_TYPE_TO_FORMAT.code], + ['s', 0], + ['(beta)', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see https://a.com/`v2`s**(beta)**', + ], + [ + 'a URL that runs on through formats', + [ + ['https://x.com/', TEXT_TYPE_TO_FORMAT.bold], + ['ab', 0], + ['\u201cc\u201d', TEXT_TYPE_TO_FORMAT.italic], + ], + '**https://x.com/**ab*\u201cc\u201d*', + ], + [ + 'a `www.` host that would take in markers', + [ + ['www.x.co', TEXT_TYPE_TO_FORMAT.bold], + ['a', 0], + ], + '**www.x.co**a', + ], + [ + 'a `www.` host that would take in italic', + [ + ['www.x.co', 0], + ['a', TEXT_TYPE_TO_FORMAT.italic], + ['b', 0], + ], + 'www.x.co*a*b', + ], + [ + 'a `www.` host that would take in markers after `:`', + [ + ['www.x.co:', 0], + ['a', TEXT_TYPE_TO_FORMAT.bold], + [' z', 0], + ], + 'www.x.co:**a** z', + ], + [ + 'a URL before a character that is a letter only for CommonMark', + [ + ['see ', 0], + ['http://x.co/', TEXT_TYPE_TO_FORMAT.bold], + ['\u2028x', 0], + ], + 'see **http://x.co/**
x', + ], + [ + 'a `www.` host that would take in a code span', + [ + ['www.x.co', 0], + ['v', TEXT_TYPE_TO_FORMAT.code], + ['s', 0], + ], + 'www.x.co`v`s', + ], + [ + 'a URL that would take in strikethrough', + [ + ['see https://x.com/a', 0], + ['b', TEXT_TYPE_TO_FORMAT.strikethrough], + ], + 'see https://x.com/a~~b~~', + ], + [ + 'a URL that holds a `www.` host', + [ + ['http://a.co/www.x.co', 0], + ['!', TEXT_TYPE_TO_FORMAT.bold], + ], + 'http://a.co/www.x.co**!**', + ], + [ + 'a URL that would take in the backslash of an escape at its end', + [ + ['see ', 0], + ['https://a*', TEXT_TYPE_TO_FORMAT.italic], + ], + 'see *https://a\\**', + ], + [ + 'a `www.` host that leaves out the markers after it', + [ + ['www.x.co', TEXT_TYPE_TO_FORMAT.bold], + [' a', 0], + ], + '**www.x.co** a', + ], + [ + 'a `www.` host after `/`, which starts no link', + [ + ['/www.x.co', 0], + ['a', TEXT_TYPE_TO_FORMAT.italic], + ['b', 0], + ], + '/www.x.co*a*b', + ], + [ + 'a URL that holds an email', + [ + ['http://a.co/x@y.co', 0], + ['!', TEXT_TYPE_TO_FORMAT.bold], + ], + 'http://a.co/x@y.co**!**', + ], + ])( + 'exports formatting beside %s, which both importers read back', + (_name, runs, markdown) => { + // Links split the text, and @lexical/mdast keeps its own links. + const $runs = () => { + const merged: [string, number][] = []; + for (const node of $getRoot().getAllTextNodes()) { + const last = merged[merged.length - 1]; + if (last !== undefined && last[1] === node.getFormat()) { + last[0] += node.getTextContent(); + } else { + merged.push([node.getTextContent(), node.getFormat()]); + } + } + return merged; + }; + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + ...runs.map(([text, format]) => + $createTextNode(text).setFormat(format), + ), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => $convertToMarkdownString(TRANSFORMERS)), + ).toBe(markdown); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect(editor.read('latest', $runs)).toEqual(runs); + const mdastEditor = buildEditorFromExtensions( + defineExtension({ + dependencies: [MdastGfmExtension], + name: '[root]', + }), + ); + onTestFinished(() => mdastEditor.dispose()); + mdastEditor.update(() => $convertFromMdastMarkdownString(markdown), { + discrete: true, + }); + expect(mdastEditor.read('latest', $runs)).toEqual(runs); + }, + ); + + // GFM links none of these: a mention is not Markdown, an email needs a dot + // in its domain, and a URL needs `http`, `https` or `ftp` for its scheme. + it.each([ + ['@john', '@john'], + ['foo@bar', 'foo@bar'], + ['a://b', 'a://b'], + ['xhttp://b', 'xhttp://b'], + ])( + 'writes a reference after %j, which readers do not autolink', + (text, markdown) => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode(`see ${text}`), + $createTextNode( + `${LEFT_DOUBLE_QUOTE}x${RIGHT_DOUBLE_QUOTE}`, + ).toggleFormat('bold'), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => $convertToMarkdownString(TRANSFORMERS)), + ).toBe(`see ${markdown}**${LEFT_DOUBLE_QUOTE}x${RIGHT_DOUBLE_QUOTE}**`); + }, + ); + + it.each([ + ['after', 'user:bob**(admin)**', ['(admin)', 1]], + ['before', '**(admin)**user:bob', ['(admin)', 1]], + ['in italic after', 'user:bob*\u201ca\u201d*', ['\u201ca\u201d', 2]], + ['nested after', 'user:bob**(text *more*)**', ['more', 3]], + [ + 'far after', + 'user:bob**(' + 'x '.repeat(40) + 'y)**', + ['(' + 'x '.repeat(40) + 'y)', 1], + ], + ] as const)( + 'reads formatting %s a text match whose syntax has a letter beside it', + (_where, markdown, formatted) => { + // A mention-style transformer whose syntax starts and ends in a letter. + const MENTION: TextMatchTransformer = { + dependencies: [LinkNode], + export: node => + $isLinkNode(node) && node.getURL().startsWith('mention:') + ? 'user:' + node.getURL().slice('mention:'.length) + : null, + importRegExp: /user:(\w+)/, + regExp: /user:(\w+)$/, + replace: (textNode, match) => { + const link = $createLinkNode('mention:' + match[1]); + link.append($createTextNode(match[1])); + textNode.replace(link); + }, + type: 'text-match', + }; + const transformers = [MENTION, ...TRANSFORMERS]; + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(markdown, transformers), { + discrete: true, + }); + const read = () => + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]), + ); + const imported = read(); + expect(imported).toContainEqual(formatted); + expect( + editor.read('latest', () => $convertToMarkdownString(transformers)), + ).toBe(markdown); + }, + ); + + it.each([ + ['a short run', '#'], + ['a long run', '#' + 'x'.repeat(80)], + ])( + 'reads a run after a link the same whatever follows it on the line, with %s', + (_name, inner) => { + // As on main, the run reads the link's end as the edge of a line, so + // it opens without closing, and the rule of 3 does not keep `*` from + // closing it. + const editor = createTestEditor(); + for (const tail of ['', ' [b](v)', ' ' + 'y'.repeat(80)]) { + editor.update( + () => + $convertFromMarkdownString( + `[a](u)**${inner}*${tail}`, + TRANSFORMERS, + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .slice(0, 3) + .map(node => [node.getTextContent(), node.getFormat()]), + ), + ).toEqual([ + ['a', 0], + ['*', 0], + [inner, TEXT_TYPE_TO_FORMAT.italic], + ]); + } + }, + ); + + it('gives a text match found earlier in the line its named groups where they are', () => { + const ranges: string[] = []; + const MENTION: TextMatchTransformer = { + dependencies: [], + importRegExp: new RegExp('@(?\\w+)', 'd'), + regExp: new RegExp('@(?\\w+)$', 'd'), + replace: (textNode, match) => { + const {indices, input = ''} = match; + const name = indices?.groups?.name; + assert(name !== undefined && indices?.[1] !== undefined); + const [start, end] = name; + ranges.push(input.slice(start, end), input.slice(...indices[1])); + textNode.setTextContent(input.slice(start, end)); + }, + type: 'text-match', + }; + const editor = createTestEditor(); + editor.update( + () => + $convertFromMarkdownString('**a** @bob', [MENTION, ...TRANSFORMERS]), + {discrete: true}, + ); + expect(ranges).toEqual(['bob', 'bob']); + }); + + it("writes no reference for the `.` of `www.` that is a format's marker", () => { + const DOT: TextFormatTransformer = { + format: ['italic'], + tag: '.', + type: 'text-format', + }; + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('www'), + $createTextNode('x.co').toggleFormat('italic'), + $createTextNode(' or www.y.co'), + ), + ), + {discrete: true}, + ); + expect(editor.read('latest', () => $convertToMarkdownString([DOT]))).toBe( + 'www.x.co. or www.y.co', + ); + }); + + it("writes no reference for an email's `@` after a raw backslash", () => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('x\\@b.co'), + $createTextNode('\u201cy\u201d').toggleFormat('bold'), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS, undefined, true), + ), + ).toBe('x\\@b.co**\u201cy\u201d**'); + }); + + it('keeps an email that holds a reference whole for @lexical/mdast', () => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('a@b.co'), + $createTextNode('\u201cx\u201d').toggleFormat('bold'), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS), + ); + expect(markdown).toBe('a@b.co**\u201cx\u201d**'); + const mdastEditor = buildEditorFromExtensions( + defineExtension({ + dependencies: [MdastGfmExtension], + name: '[root]', + }), + ); + onTestFinished(() => mdastEditor.dispose()); + mdastEditor.update(() => $convertFromMdastMarkdownString(markdown), { + discrete: true, + }); + expect( + mdastEditor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => { + const parent = node.getParent(); + return [ + node.getTextContent(), + $isLinkNode(parent) ? parent.getURL() : null, + ]; + }), + ), + ).toEqual([ + ['a@b.co', 'mailto:a@b.co'], + ['\u201cx\u201d', null], + ]); + }); + + it.each<[string, [string, number][], string]>([ + [ + 'a backslash before a marker', + [ + ['see http://x.com/a\\', 0], + ['b', TEXT_TYPE_TO_FORMAT.bold], + ], + 'see http://x.com/a\**b**', + ], + [ + 'a backslash before a code span', + [ + ['see http://x.com/a\\', 0], + ['b', TEXT_TYPE_TO_FORMAT.code], + ], + 'see http://x.com/a\`b`', + ], + [ + 'a tag character before its marker', + [ + ['see https://x.com/?a=', 0], + ['b', TEXT_TYPE_TO_FORMAT.highlight], + ], + 'see https://x.com/?a===b==', + ], + ])('spoils a URL for the reference to %s', (_name, runs, markdown) => { + const $runs = () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]); + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + ...runs.map(([text, format]) => + $createTextNode(text).setFormat(format), + ), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS, undefined, true), + ), + ).toBe(markdown); + editor.update( + () => $convertFromMarkdownString(markdown, TRANSFORMERS, undefined, true), + {discrete: true}, + ); + expect(editor.read('latest', $runs)).toEqual(runs); + }); + + it('writes a reference after CJK punctuation that ends bold before a CJK character', () => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('\u4f60\u597d\u3002').toggleFormat('bold'), + $createTextNode('\u4e16\u754c'), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS), + ); + expect(markdown).toBe('**\u4f60\u597d\u3002**世\u754c'); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.hasFormat('bold')]), + ), + ).toEqual([ + ['\u4f60\u597d\u3002', true], + ['\u4e16\u754c', false], + ]); + }); + + // micromark, which @lexical/mdast imports with, reads a character outside + // the Basic Multilingual Plane as a letter, so export writes references + // that make the markers open and close for it too. + it.each<[runs: [string, number][], markdown: string]>([ + [ + [ + ['\u{1f600}', 0], + ['\u201cb\u201d', TEXT_TYPE_TO_FORMAT.bold], + ], + '😀**\u201cb\u201d**', + ], + [ + [ + ['a', TEXT_TYPE_TO_FORMAT.bold], + ['\u{1f600}', TEXT_TYPE_TO_FORMAT.italic], + ], + '**a***😀*', + ], + [ + [ + ['\u6ce8\u610f\uff1a', TEXT_TYPE_TO_FORMAT.bold], + ['\u8bf7\u5148\u767b\u5f55', 0], + ], + '**\u6ce8\u610f\uff1a**请\u5148\u767b\u5f55', + ], + // A scheme or `mailto:` with no host or address after it starts no URL. + [ + [ + ['\u4f60.w mailto:', TEXT_TYPE_TO_FORMAT.italic], + ['1', TEXT_TYPE_TO_FORMAT.bold], + ], + '*\u4f60.w mailto:***1**', + ], + [ + [ + [':http://', TEXT_TYPE_TO_FORMAT.bold], + ['\u4f60', TEXT_TYPE_TO_FORMAT.italic], + ], + '**:http://***你*', + ], + ])('exports %j so that both importers read it back', (runs, markdown) => { + const $runs = () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]); + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + ...runs.map(([text, format]) => + $createTextNode(text).setFormat(format), + ), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => $convertToMarkdownString(TRANSFORMERS)), + ).toBe(markdown); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect(editor.read('latest', $runs)).toEqual(runs); + const mdastEditor = buildEditorFromExtensions( + defineExtension({ + dependencies: [MdastGfmExtension], + name: '[root]', + }), + ); + onTestFinished(() => mdastEditor.dispose()); + mdastEditor.update(() => $convertFromMdastMarkdownString(markdown), { + discrete: true, + }); + expect(mdastEditor.read('latest', $runs)).toEqual(runs); + }); + + it('exports bold italic text followed by italic text so that it re-imports (#9443)', () => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('a').toggleFormat('bold').toggleFormat('italic'), + $createTextNode(': b').toggleFormat('italic'), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString(TRANSFORMERS), + ); + expect(markdown).toBe('***a****: b*'); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]), + ), + ).toEqual([ + ['a', TEXT_TYPE_TO_FORMAT.bold | TEXT_TYPE_TO_FORMAT.italic], + [': b', TEXT_TYPE_TO_FORMAT.italic], + ]); + }); + + it('writes references around an underscore italic between letters', () => { + const transformers = TRANSFORMERS.filter( + transformer => transformer !== ITALIC_STAR, + ); + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('a'), + $createTextNode('b').toggleFormat('italic'), + $createTextNode('c'), + ), + ), + {discrete: true}, + ); + const markdown = editor.read('latest', () => + $convertToMarkdownString(transformers), + ); + expect(markdown).toBe('a_b_c'); + editor.update(() => $convertFromMarkdownString(markdown, transformers), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.hasFormat('italic')]), + ), + ).toEqual([ + ['a', false], + ['b', true], + ['c', false], + ]); + }); + + it.each<[string, [string, number][], string]>([ + [ + 'a URL inside a code span', + [ + ['a', 0], + ['\u201cb\u201d', TEXT_TYPE_TO_FORMAT.bold], + ['http://y', TEXT_TYPE_TO_FORMAT.code], + ], + 'a**\u201cb\u201d**`http://y`', + ], + [ + 'a URL that starts later in the run', + [ + ['foo', 0], + ['\u201cbar\u201d', TEXT_TYPE_TO_FORMAT.bold], + ['(www.x.com)', 0], + ], + 'foo**\u201cbar\u201d**(www.x.com)', + ], + ])('writes a reference outside %s', (_name, runs, markdown) => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + ...runs.map(([text, format]) => + $createTextNode(text).setFormat(format), + ), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => $convertToMarkdownString(TRANSFORMERS)), + ).toBe(markdown); + }); + + // Not as micromark reads them: it also lets a run open before, and close + // after, another `*`, `_` or `~` marker. + it.each<[string, [string, number][]]>([ + [ + '**bold***~~gone~~*', + [ + ['bold', TEXT_TYPE_TO_FORMAT.bold], + ['*', 0], + ['gone', TEXT_TYPE_TO_FORMAT.strikethrough], + ['*', 0], + ], + ], + [ + '**a***_b_*', + [ + ['a', TEXT_TYPE_TO_FORMAT.bold], + ['*', 0], + ['b', TEXT_TYPE_TO_FORMAT.italic], + ['*', 0], + ], + ], + ['**~~**a', [['**~~**a', 0]]], + ['~~a*~~b', [['~~a*~~b', 0]]], + [ + '[a](u)*`c`)**', + [ + ['a', 0], + ['*', 0], + ['c', TEXT_TYPE_TO_FORMAT.code], + [')**', 0], + ], + ], + [ + '**)*(.`*b!****_', + [ + [')', TEXT_TYPE_TO_FORMAT.bold], + ['(.`b!', TEXT_TYPE_TO_FORMAT.bold | TEXT_TYPE_TO_FORMAT.italic], + ['_', 0], + ], + ], + ])( + 'reads a run beside another marker as CommonMark does in %j', + (markdown, runs) => { + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]), + ), + ).toEqual(runs); + }, + ); + + it.each<[string, [string, number][]]>([ + [ + '**==`====**', + [ + ['`', TEXT_TYPE_TO_FORMAT.bold | TEXT_TYPE_TO_FORMAT.highlight], + ['==', TEXT_TYPE_TO_FORMAT.bold], + ], + ], + ['**~~`~~~~**', [['~~`~~~~', TEXT_TYPE_TO_FORMAT.bold]]], + ])( + "reads a highlight inside a format as a line of its own, and GFM's markers as GFM does, in %j", + (markdown, runs) => { + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]), + ), + ).toEqual(runs); + }, + ); + + it.each<[string, [string, number][]]>([ + [ + 'x\\``bb`x', + [ + ['x`', 0], + ['bb', TEXT_TYPE_TO_FORMAT.code], + ['x', 0], + ], + ], + [ + '\\``a` `b`', + [ + ['`', 0], + ['a', TEXT_TYPE_TO_FORMAT.code], + [' ', 0], + ['b', TEXT_TYPE_TO_FORMAT.code], + ], + ], + ['`a\\`', [['a\\', TEXT_TYPE_TO_FORMAT.code]]], + ])('opens a code span after an escaped backtick in %j', (markdown, runs) => { + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]), + ), + ).toEqual(runs); + }); + + it.each<[string, [string, number][]]>([ + [ + ':a**b*', + [ + ['X', 0], + ['b', TEXT_TYPE_TO_FORMAT.italic], + ], + ], + [':a***b*', [['X**b*', 0]]], + ])( + 'reads the rest of a run that a text match cuts in %j on its own', + (markdown, runs) => { + const TAG: TextMatchTransformer = { + dependencies: [], + importRegExp: /:[a-z]+\*/, + regExp: /:[a-z]+\*$/, + replace: textNode => { + textNode.setTextContent('X'); + }, + type: 'text-match', + }; + const editor = createTestEditor(); + editor.update( + () => $convertFromMarkdownString(markdown, [TAG, ...TRANSFORMERS]), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]), + ), + ).toEqual(runs); + }, + ); + + // A `;` before it starts no URL: `www.` needs whitespace or `*_~(` before + // it. + it.each(['bwww.x.co'])( + 'writes a reference before %j, which readers do not autolink', + text => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('.').toggleFormat('bold'), + $createTextNode(text), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => $convertToMarkdownString(TRANSFORMERS)), + ).toBe(`**.**b${text.slice(1)}`); + }, + ); + + it('searches a line once for a text match that it does not hold', () => { + // The characters searched: each text left after a match was searched in + // full again before. + let searched = 0; + class CountingRegExp extends RegExp { + exec(text: string) { + searched += text.length; + return super.exec(text); + } + } + const NEVER: TextMatchTransformer = { + dependencies: [], + importRegExp: new CountingRegExp('\\$\\$[^$]+\\$\\$'), + regExp: /\$\$[^$]+\$\$$/, + replace: () => {}, + type: 'text-match', + }; + const markdown = '**a** [b](c) '.repeat(20); + const editor = createTestEditor(); + editor.update( + () => $convertFromMarkdownString(markdown, [NEVER, ...TRANSFORMERS]), + {discrete: true}, + ); + expect( + editor.read('latest', () => $getRoot().getAllTextNodes().length), + ).toBe(80); + expect(searched).toBeLessThan(2 * markdown.length); + }); + + it.each([ + ['escaped openers', '\\['.repeat(100) + 'a](u)'], + ['code spans before a link', '[`a`'.repeat(100) + '](u)'], + ])('searches the line for a link a few times with %s', (_name, markdown) => { + let searches = 0; + const exec = RegExp.prototype.exec; + const spy = vi.spyOn(RegExp.prototype, 'exec').mockImplementation(function ( + this: RegExp, + text: string, + ) { + if (this.source === LINK.importRegExp!.source) { + searches++; + } + return exec.call(this, text); + }); + onTestFinished(() => spy.mockRestore()); + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + spy.mockRestore(); + expect(editor.read('latest', () => $getRoot().getTextContent())).toBe( + markdown.startsWith('\\') + ? markdown.replace(/\\/g, '') + : '[a'.repeat(99) + 'a', + ); + expect(searches).toBeLessThan(5); + }); + + it('scans the text after a match again when its transformer changes it', () => { + const MENTION: TextMatchTransformer = { + dependencies: [], + importRegExp: /@(\w+)/, + regExp: /@(\w+)$/, + replace: textNode => { + const next = textNode.getNextSibling(); + if ($isTextNode(next)) { + next.setTextContent(next.getTextContent().trimStart()); + } + textNode.toggleFormat('code'); + }, + type: 'text-match', + }; + const editor = createTestEditor(); + editor.update( + () => + $convertFromMarkdownString('@bob *a* *b*', [MENTION, ...TRANSFORMERS]), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]), + ), + ).toEqual([ + ['@bob', TEXT_TYPE_TO_FORMAT.code], + ['a', TEXT_TYPE_TO_FORMAT.italic], + [' ', 0], + ['b', TEXT_TYPE_TO_FORMAT.italic], + ]); + }); + + it('reads the text after a match on its own when its transformer changes it', () => { + const MENTION: TextMatchTransformer = { + dependencies: [], + importRegExp: /@(\w+)/, + regExp: /@(\w+)$/, + replace: textNode => { + const next = textNode.getNextSibling(); + if ($isTextNode(next)) { + next.setTextContent(next.getTextContent().trimStart()); + } + textNode.toggleFormat('code'); + }, + type: 'text-match', + }; + const editor = createTestEditor(); + editor.update( + () => + $convertFromMarkdownString('@bob _a_ _b_', [MENTION, ...TRANSFORMERS]), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.getFormat()]), + ), + ).toEqual([ + ['@bob', TEXT_TYPE_TO_FORMAT.code], + ['a', TEXT_TYPE_TO_FORMAT.italic], + [' ', 0], + ['b', TEXT_TYPE_TO_FORMAT.italic], + ]); + }); + + it('imports the text after a match before the text inside it', () => { + const seen: string[] = []; + const TAG: TextMatchTransformer = { + dependencies: [], + importRegExp: /@x/, + regExp: /@x$/, + replace: textNode => { + seen.push(textNode.getParentOrThrow().getTextContent()); + }, + type: 'text-match', + }; + const editor = createTestEditor(); + editor.update( + () => $convertFromMarkdownString('**_a_** @x', [TAG, ...TRANSFORMERS]), + {discrete: true}, + ); + expect(seen).toEqual(['_a_ @x']); + }); + + it("guards a letter right after an inline link's destination", () => { + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createLinkNode('https://e.com').append($createTextNode('API')), + $createTextNode('s'), + $createTextNode('(beta)').toggleFormat('bold'), + $createTextNode(' end'), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => $convertToMarkdownString(TRANSFORMERS)), + ).toBe('[API](https://e.com)s**(beta)** end'); + }); + + it.each([ + [ + '*)**a*', + [ + ['*)*', false], + ['a', true], + ], + ], + [ + 'โ€œ_)__a(โ€œ[)_', + [ + ['โ€œ_)_', false], + ['a(โ€œ[)', true], + ], + ], + [ + '*a**`x`*', + [ + ['a', true], + ['*', false], + ['x', false], + ['*', false], + ], + ], + ] as [string, [string, boolean][]][])( + 'reads a delimiter run cut by an earlier match as it was in %j', + (markdown, runs) => { + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.hasFormat('italic')]), + ), + ).toEqual(runs); + }, + ); + + it.each<[string, [string, boolean, boolean][]]>([ + [ + 'Use [this](u) or the [`--flag` from [docs](u)', + [ + ['Use ', false, false], + ['this', false, true], + [' or the [', false, false], + ['--flag', true, false], + [' from ', false, false], + ['docs', false, true], + ], + ], + [ + '[a](b) [`x`[c](d)', + [ + ['a', false, true], + [' [', false, false], + ['x', true, false], + ['c', false, true], + ], + ], + ])( + 'keeps a code span after a stray `[` before a link in %j', + (markdown, nodes) => { + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [ + node.getTextContent(), + node.hasFormat('code'), + $isLinkNode(node.getParent()), + ]), + ), + ).toEqual(nodes); + }, + ); + + it('applies a link before a later link that holds a code span', () => { + const editor = createTestEditor(); + editor.update( + () => + $convertFromMarkdownString( + '[a](https://u) and [`b`](https://v)', + TRANSFORMERS, + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [ + node.getTextContent(), + node.hasFormat('code'), + $isLinkNode(node.getParent()), + ]), + ), + ).toEqual([ + ['a', false, true], + [' and ', false, false], + ['b', true, true], + ]); + }); + + it.each([ + [ + '`c` *[a](b)*', + [ + ['c', false], + [' ', false], + ['a', true], + ], + ], + [ + '`c` x *[a](b) `d`*', + [ + ['c', false], + [' x ', false], + ['a', true], + [' ', true], + ['d', true], + ], + ], + ] as [string, [string, boolean][]][])( + 'keeps emphasis around a link after a code span in %j', + (markdown, runs) => { + const editor = createTestEditor(); + editor.update(() => $convertFromMarkdownString(markdown, TRANSFORMERS), { + discrete: true, + }); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [node.getTextContent(), node.hasFormat('italic')]), + ), + ).toEqual(runs); + }, + ); + + it('keeps a link from taking part of a later code span', () => { + const editor = createTestEditor(); + editor.update( + () => $convertFromMarkdownString('**`a` [b** `c](d)`', TRANSFORMERS), + {discrete: true}, + ); + expect( + editor.read('latest', () => + $getRoot() + .getAllTextNodes() + .map(node => [ + node.getTextContent(), + node.hasFormat('bold'), + node.hasFormat('code'), + ]), + ), + ).toEqual([ + ['a', true, true], + [' [b', true, false], + [' ', false, false], + ['c](d)', false, true], + ]); + }); + + it('guards word characters in a selection export', () => { + const editor = createTestEditor(); + editor.update( + () => { + const bold = $createTextNode( + `${LEFT_DOUBLE_QUOTE}bar${RIGHT_DOUBLE_QUOTE}`, + ).toggleFormat('bold'); + const after = $createTextNode('baz'); + $getRoot().append( + $createParagraphNode().append($createTextNode('foo'), bold, after), + ); + bold.select(0, 0).focus.set(after.getKey(), 2, 'text'); + }, + {discrete: true}, + ); + expect( + editor.read('latest', () => + $convertSelectionToMarkdownString(TRANSFORMERS, $getSelection()), + ), + ).toBe(`**${LEFT_DOUBLE_QUOTE}bar${RIGHT_DOUBLE_QUOTE}**ba`); + }); + + it('guards word characters across an inline element without a transformer', () => { + const transformers = TRANSFORMERS.filter( + transformer => transformer !== LINK, + ); + const editor = createTestEditor(); + editor.update( + () => + $getRoot().append( + $createParagraphNode().append( + $createTextNode('foo'), + $createLinkNode('https://example.com').append( + $createTextNode( + `${LEFT_DOUBLE_QUOTE}bar${RIGHT_DOUBLE_QUOTE}`, + ).toggleFormat('bold'), + ), + $createTextNode('baz'), + ), + ), + {discrete: true}, + ); + expect( + editor.read('latest', () => $convertToMarkdownString(transformers)), + ).toBe(`foo**${LEFT_DOUBLE_QUOTE}bar${RIGHT_DOUBLE_QUOTE}**baz`); + }); +}); + describe('decimal character references that name no character', () => { // `String.fromCodePoint` throws on a number past U+10FFFF, so one of these // in the text or a link used to abort the whole import. CommonMark reads at diff --git a/packages/lexical-markdown/src/__tests__/unit/MarkdownTransformers.test.ts b/packages/lexical-markdown/src/__tests__/unit/MarkdownTransformers.test.ts index 6d7c6bcff06..dc89bd36bc3 100644 --- a/packages/lexical-markdown/src/__tests__/unit/MarkdownTransformers.test.ts +++ b/packages/lexical-markdown/src/__tests__/unit/MarkdownTransformers.test.ts @@ -287,6 +287,53 @@ describe('CODE_SPAN_PRECEDENCE', () => { }); }); +describe('UNDERSCORE_AFTER_UNICODE_PUNCTUATION', () => { + // Underscore formats do not apply inside a word, so the character before the + // opening tag has to be whitespace or punctuation. CommonMark counts every + // Unicode punctuation (P) and symbol (S) character as punctuation, not only + // the ASCII ones. + // https://spec.commonmark.org/0.31.2/#unicode-punctuation-character + const EN_DASH = '\u2013'; + const LEFT_DOUBLE_QUOTE = '\u201C'; + + test.each([ + [`See ${EN_DASH}_here_`, `See ${EN_DASH}`], + [`See ${LEFT_DOUBLE_QUOTE}_here_`, `See ${LEFT_DOUBLE_QUOTE}`], + // An emoji (So) outside the Basic Multilingual Plane, read whole. + ['See \u{1F600}_here_', 'See \u{1F600}'], + ])('typing %j formats the word between the underscores', (typed, before) => { + using editor = buildEditorFromExtensions([MarkdownShortcutTestExtension]); + typeMarkdown(editor, typed); + editor.read(() => { + const paragraph = $getRoot().getFirstChildOrThrow(); + assert($isParagraphNode(paragraph), 'Root child must be a paragraph'); + const textNodes = paragraph.getChildren().filter($isTextNode); + expect(textNodes.map(node => node.getTextContent())).toEqual([ + before, + 'here', + ]); + expect(textNodes.map(node => node.hasFormat('italic'))).toEqual([ + false, + true, + ]); + }); + }); + + test('typing an underscore format right after a letter still does not format it', () => { + using editor = buildEditorFromExtensions([MarkdownShortcutTestExtension]); + typeMarkdown(editor, 'See \u00E9_here_'); + editor.read(() => { + const paragraph = $getRoot().getFirstChildOrThrow(); + assert($isParagraphNode(paragraph), 'Root child must be a paragraph'); + const textNodes = paragraph.getChildren().filter($isTextNode); + expect(textNodes.map(node => node.getTextContent())).toEqual([ + 'See \u00E9_here_', + ]); + expect(textNodes[0].hasFormat('italic')).toBe(false); + }); + }); +}); + describe('WRAPPING_PRESERVES_FORMAT', () => { test('**...** around already-bold text preserves bold', () => { // https://github.com/meta/lexical/issues/8727 diff --git a/packages/lexical-markdown/src/importTextFormatTransformer.ts b/packages/lexical-markdown/src/importTextFormatTransformer.ts index 7c159bc3010..a7a223d637d 100644 --- a/packages/lexical-markdown/src/importTextFormatTransformer.ts +++ b/packages/lexical-markdown/src/importTextFormatTransformer.ts @@ -10,7 +10,13 @@ import type {TextFormatTransformersIndex} from './MarkdownImport'; import type {TextFormatTransformer} from './MarkdownTransformers'; import type {TextNode} from 'lexical'; -import {PUNCTUATION, WHITESPACE} from './utils'; +import { + characterAt, + characterBefore, + getDelimiterRunEdge, + getEmphasisSides, + isEscaped, +} from './utils'; interface Delimiter { index: number; @@ -21,122 +27,283 @@ interface Delimiter { active: boolean; } -export function findOutermostTextFormatTransformer( - textNode: TextNode, - textFormatTransformersIndex: TextFormatTransformersIndex, -): { +interface FormatMatch { startIndex: number; endIndex: number; - transformer: TextFormatTransformer; - match: RegExpMatchArray; - // True when the outermost format is a code span. Code spans are atomic: their - // content is never reprocessed and they bind tighter than text-match - // transformers, so this is tagged here rather than re-derived by callers. - isCodeSpan: boolean; -} | null { - const textContent = textNode.getTextContent(); + tag: string; + content: string; +} + +/** + * The code spans and outermost emphasis of a text, found once and read again + * for what is left of the text after each match, at `offset`. A match there + * starts with the code span at `nextCodeSpan` or the emphasis at + * `nextEmphasis`. + */ +export interface FormatScan { + text: string; + /** The characters that start an emphasis tag. */ + delimiters: Set; + codeSpans: FormatMatch[]; + emphasis: FormatMatch[]; + offset: number; + nextCodeSpan: number; + nextEmphasis: number; + /** An emphasis that starts at `offset`, found after the scan. */ + pending: FormatMatch | undefined; +} +export function scanTextFormats( + text: string, + textFormatTransformersIndex: TextFormatTransformersIndex, + before: string | undefined, + after: string | undefined, +): FormatScan { // Find code spans first. Emphasis delimiters inside inline elements (e.g., code spans) // should not be processed. Currently only code spans are handled; other inline elements // (e.g., links, raw HTML) may need similar treatment in the future. - const codeTransformer = textFormatTransformersIndex.transformersByTag['`']; - - const excludeRanges: {start: number; end: number}[] = []; - let codeMatch = null; - if (codeTransformer) { - const codeSpans = scanCodeSpans(textContent); - - for (const span of codeSpans) { - if (!codeMatch) { - codeMatch = { - content: span.content, - endIndex: span.endIndex, - startIndex: span.startIndex, - tag: '`', - }; - } + const codeSpans = textFormatTransformersIndex.transformersByTag['`'] + ? scanCodeSpans(text).map(span => ({...span, tag: '`'})) + : []; + const delimiterChars = new Set( + Object.keys(textFormatTransformersIndex.transformersByTag) + .filter(tag => tag[0] !== '`') + .map(tag => tag[0]), + ); + const delimiters = scanDelimiters( + text, + delimiterChars, + codeSpans, + before, + after, + ); + return { + codeSpans, + delimiters: delimiterChars, + emphasis: + delimiters.length > 0 + ? processEmphasis(text, delimiters, textFormatTransformersIndex) + : [], + nextCodeSpan: 0, + nextEmphasis: 0, + offset: 0, + pending: undefined, + text, + }; +} - excludeRanges.push({ - end: span.endIndex, - start: span.startIndex, - }); +/** + * Moves `scan` on to the text after `offset`, where a match ends. What it + * found there holds for that text on its own as long as no code span or + * emphasis runs on from before `offset` (a pair cannot reach past the end of + * the outermost one), no backslash before it escapes it, and no run of a + * delimiter or backtick goes on across it; returns false otherwise. A + * closing run that the end of a format match cuts is the exception: the scan + * measured it by what was left of it, as a scan of the rest does. + */ +export function advanceFormatScan( + scan: FormatScan, + offset: number, + isFormatEnd: boolean, +): boolean { + const {codeSpans, emphasis, pending, text} = scan; + scan.offset = offset; + scan.pending = undefined; + if (pending !== undefined && pending.endIndex > offset) { + return false; + } + while ( + scan.nextCodeSpan < codeSpans.length && + codeSpans[scan.nextCodeSpan].startIndex < offset + ) { + if (codeSpans[scan.nextCodeSpan++].endIndex > offset) { + return false; } } - - const delimiters = scanDelimiters( - textContent, - textFormatTransformersIndex, - excludeRanges, + while ( + scan.nextEmphasis < emphasis.length && + emphasis[scan.nextEmphasis].startIndex < offset + ) { + if (emphasis[scan.nextEmphasis++].endIndex > offset) { + return false; + } + } + const cut = text[offset - 1]; + return ( + cut !== '\\' && + // A pair found after the scan cut a run that the scan measured whole. + ((isFormatEnd && pending === undefined) || + cut !== text[offset] || + (cut !== '`' && !scan.delimiters.has(cut))) ); - const emphasisMatch = - delimiters.length > 0 - ? processEmphasis(textContent, delimiters, textFormatTransformersIndex) - : null; +} - let resultMatch = null; - let resultTransformer = null; +/** + * Lets `scan`, moved on to `rest` by {@link advanceFormatScan}, read the + * delimiter run that starts `rest` beside the edge of a line rather than + * beside what came before, where it opened or closed otherwise. Nothing in + * `rest` comes before the run, so the rest reads the same either way but for + * the pair the run opens, which nothing in `rest` can hold, and whatever + * pairs across its end, which `advanceFormatScan` turns down there. The pair + * ends before a code span that the scan found next, which is imported first + * as it is with the match before it, and it is found in ever longer prefixes + * of `rest`, so that finding it costs about as much as the text it holds. + * Returns false when the rest has to be scanned again. + */ +export function findPairAtEdge( + scan: FormatScan, + rest: string, + textFormatTransformersIndex: TextFormatTransformersIndex, +): boolean { + const {codeSpans, offset} = scan; + let span = scan.nextCodeSpan; + const next = scan.emphasis[scan.nextEmphasis]; + const end = + span < codeSpans.length && + (!next || codeSpans[span].startIndex < next.startIndex) + ? codeSpans[span].startIndex - offset + : rest.length; + for (let length = 64; ; length *= 2) { + // Cut neither a code span nor a run of a delimiter or backtick. + while ( + span < codeSpans.length && + codeSpans[span].endIndex <= offset + length + ) { + span++; + } + if ( + span < codeSpans.length && + codeSpans[span].startIndex < offset + length + ) { + length = codeSpans[span].endIndex - offset; + } + if ( + length < end && + (rest[length - 1] === '`' || scan.delimiters.has(rest[length - 1])) + ) { + length = getDelimiterRunEdge(rest, length, 1); + } + if (length >= end) { + if (end === rest.length) { + return false; + } + length = end; + } + const pair = scanTextFormats( + rest.slice(0, length), + textFormatTransformersIndex, + undefined, + characterAt(rest, length), + ).emphasis[0]; + if (pair !== undefined && pair.startIndex === 0) { + scan.pending = { + ...pair, + endIndex: pair.endIndex + offset, + startIndex: offset, + }; + return true; + } + if (length === end) { + return true; + } + } +} - if (codeMatch && emphasisMatch) { +/** + * Whether the text from `startIndex` to `endIndex` after `scan`'s offset + * holds part of a code span but not all of it. + */ +export function partlyHoldsCodeSpan( + scan: FormatScan, + startIndex: number, + endIndex: number, +): boolean { + const {codeSpans, offset} = scan; + for (let i = scan.nextCodeSpan; i < codeSpans.length; i++) { + const spanStart = codeSpans[i].startIndex - offset; + const spanEnd = codeSpans[i].endIndex - offset; + if (spanStart >= endIndex) { + break; + } if ( - emphasisMatch.startIndex <= codeMatch.startIndex && - emphasisMatch.endIndex >= codeMatch.endIndex + spanEnd > startIndex && + (spanStart < startIndex || spanEnd > endIndex) ) { - resultMatch = emphasisMatch; - resultTransformer = - textFormatTransformersIndex.transformersByTag[emphasisMatch.tag]; - } else { - resultMatch = codeMatch; - resultTransformer = codeTransformer; + return true; } - } else if (codeMatch) { - resultMatch = codeMatch; - resultTransformer = codeTransformer; - } else if (emphasisMatch) { - resultMatch = emphasisMatch; - resultTransformer = - textFormatTransformersIndex.transformersByTag[emphasisMatch.tag]; } + return false; +} - if (!resultMatch || !resultTransformer) { +/** + * The outermost format match in `text`, what is left after `scan`'s offset of + * the text it scanned. + */ +export function findOutermostTextFormatTransformer( + text: string, + textFormatTransformersIndex: TextFormatTransformersIndex, + scan: FormatScan, +): { + startIndex: number; + endIndex: number; + transformer: TextFormatTransformer; + match: RegExpMatchArray; + // True when the outermost format is a code span. Code spans are atomic: their + // content is never reprocessed and they bind tighter than text-match + // transformers, so this is tagged here rather than re-derived by callers. + isCodeSpan: boolean; +} | null { + const codeMatch: FormatMatch | undefined = scan.codeSpans[scan.nextCodeSpan]; + const emphasisMatch: FormatMatch | undefined = + scan.pending ?? scan.emphasis[scan.nextEmphasis]; + // Delimiters inside a code span were excluded, so an emphasis match that + // starts first either holds the whole span or ends before it, and either + // way it is applied first. + const resultMatch = + emphasisMatch !== undefined && + (codeMatch === undefined || + emphasisMatch.startIndex <= codeMatch.startIndex) + ? emphasisMatch + : codeMatch; + if (resultMatch === undefined) { return null; } - + const isCodeSpan = resultMatch === codeMatch; + const transformer = + textFormatTransformersIndex.transformersByTag[resultMatch.tag]; + if (!transformer) { + return null; + } + const {offset} = scan; + const startIndex = resultMatch.startIndex - offset; + const endIndex = resultMatch.endIndex - offset; const regexMatch: RegExpMatchArray = [ - textContent.slice(resultMatch.startIndex, resultMatch.endIndex), + text.slice(startIndex, endIndex), resultMatch.tag, resultMatch.content, ]; - regexMatch.index = resultMatch.startIndex; - regexMatch.input = textContent; + regexMatch.index = startIndex; + regexMatch.input = text; return { - endIndex: resultMatch.endIndex, - // resultTransformer is the registered code transformer (by identity) - // exactly when the chosen match is the code span. - isCodeSpan: resultTransformer === codeTransformer, + endIndex, + isCodeSpan, match: regexMatch, - startIndex: resultMatch.startIndex, - transformer: resultTransformer, + startIndex, + transformer, }; } // Finds all inline code spans, left to right and non-overlapping, per CommonMark // rules: https://spec.commonmark.org/#code-spans. A run opens a span and the -// next run of equal length closes it. An escaped backtick (`\``) cannot open a -// span, but backslashes are otherwise literal and don't prevent closing. +// next run of equal length closes it. An escaped backtick (`\``) is literal, +// and the rest of its run may open a span, but backslashes are otherwise +// literal and don't prevent closing. export function scanCodeSpans(text: string): { startIndex: number; endIndex: number; content: string; }[] { - const isEscaped = (index: number): boolean => { - let count = 0; - for (let i = index - 1; i >= 0 && text[i] === '\\'; i--) { - count++; - } - return count % 2 === 1; - }; - // Collect maximal backtick runs. const runs: {index: number; length: number}[] = []; let i = 0; @@ -153,25 +320,50 @@ export function scanCodeSpans(text: string): { } } + // The runs of each length, in order, to find a closer in, and how many of + // them the openers, which only move forward, have passed. + const runsByLength = new Map(); + runs.forEach(({length}, index) => { + const indexes = runsByLength.get(length); + if (indexes) { + indexes.push(index); + } else { + runsByLength.set(length, [index]); + } + }); + const passed = new Map(); + // The first run after `index` that is `length` long, or -1. + const findCloser = (index: number, length: number): number => { + const indexes = runsByLength.get(length); + if (indexes === undefined) { + return -1; + } + let next = passed.get(length) || 0; + while (next < indexes.length && indexes[next] <= index) { + next++; + } + passed.set(length, next); + return next < indexes.length ? indexes[next] : -1; + }; + const spans: {content: string; endIndex: number; startIndex: number}[] = []; let openIdx = 0; while (openIdx < runs.length) { - const opener = runs[openIdx]; - - // An escaped backtick run is a literal backtick and cannot open a span. - if (isEscaped(opener.index)) { - openIdx++; - continue; - } - - let closeIdx = -1; - for (let c = openIdx + 1; c < runs.length; c++) { - if (runs[c].length === opener.length) { - closeIdx = c; - break; + let opener = runs[openIdx]; + + // An escaped backtick is a literal backtick, and the rest of its run is + // a run of its own that can open a span. Inside a span a backslash is + // literal, so the whole run still closes one. + if (isEscaped(text, opener.index)) { + if (opener.length === 1) { + openIdx++; + continue; } + opener = {index: opener.index + 1, length: opener.length - 1}; } + const closeIdx = findCloser(openIdx, opener.length); + if (closeIdx === -1) { // No matching closer; treat this run as literal and try the next one. openIdx++; @@ -202,46 +394,40 @@ export function scanCodeSpans(text: string): { function scanDelimiters( text: string, - transformersIndex: TextFormatTransformersIndex, - excludeRanges: {start: number; end: number}[] = [], + delimiterChars: Set, + codeSpans: {startIndex: number; endIndex: number}[], + // What a run at the start (or end) of the text sees beyond it; the edge of + // the line when undefined. + before: string | undefined, + after: string | undefined, ): Delimiter[] { const delimiters: Delimiter[] = []; - const delimiterChars = new Set( - Object.keys(transformersIndex.transformersByTag) - .filter(tag => tag[0] !== '`') - .map(tag => tag[0]), - ); - const isEscaped = (index: number): boolean => { - let count = 0; - for (let i = index - 1; i >= 0 && text[i] === '\\'; i--) { - count++; + // The spans are in order and the scan only moves forward. + let span = 0; + const isInCodeSpan = (index: number): boolean => { + while (span < codeSpans.length && codeSpans[span].endIndex <= index) { + span++; } - return count % 2 === 1; - }; - - const isInExcludedRange = (index: number): boolean => { - return excludeRanges.some( - range => index >= range.start && index < range.end, - ); + return span < codeSpans.length && index >= codeSpans[span].startIndex; }; let i = 0; while (i < text.length) { const char = text[i]; - if (!delimiterChars.has(char) || isEscaped(i) || isInExcludedRange(i)) { + if (!delimiterChars.has(char) || isEscaped(text, i) || isInCodeSpan(i)) { i++; continue; } - let len = 1; - while (i + len < text.length && text[i + len] === char) { - len++; - } + const len = getDelimiterRunEdge(text, i + 1, 1) - i; - const canOpen = canEmphasis(char, text, i, len, true); - const canClose = canEmphasis(char, text, i, len, false); + const {canOpen, canClose} = getEmphasisSides( + char, + i === 0 ? before : characterBefore(text, i), + i + len === text.length ? after : characterAt(text, i + len), + ); if (canOpen || canClose) { delimiters.push({ @@ -260,19 +446,15 @@ function scanDelimiters( return delimiters; } +/** The outermost emphasis matches, in order. */ function processEmphasis( text: string, delimiters: Delimiter[], transformersIndex: TextFormatTransformersIndex, -): { - startIndex: number; - endIndex: number; - tag: string; - content: string; -} | null { +): FormatMatch[] { const openersBottom: Record = {}; let currentPos = 0; - let result = null; + const matches: FormatMatch[] = []; while (currentPos < delimiters.length) { const closer = delimiters[currentPos]; @@ -340,14 +522,7 @@ function processEmphasis( tag: matchedTag, }; - if ( - !result || - match.startIndex < result.startIndex || - (match.startIndex === result.startIndex && - match.endIndex > result.endIndex) - ) { - result = match; - } + matches.push(match); for (let j = openIdx + 1; j < currentPos; j++) { delimiters[j].active = false; @@ -376,56 +551,20 @@ function processEmphasis( } } - return result; -} - -function canEmphasis( - char: string, - text: string, - index: number, - length: number, - isOpen: boolean, -): boolean { - if (!isFlanking(text, index, length, isOpen)) { - return false; - } - if (char === '*') { - return true; - } - if (char === '_') { - if (!isFlanking(text, index, length, !isOpen)) { - return true; + // Pairs nest, so the outermost ones are those that no earlier one holds. + matches.sort( + (a, b) => a.startIndex - b.startIndex || b.endIndex - a.endIndex, + ); + const outermost: FormatMatch[] = []; + for (const match of matches) { + if ( + outermost.length === 0 || + match.startIndex >= outermost[outermost.length - 1].endIndex + ) { + outermost.push(match); } - const adjacentChar = isOpen ? text[index - 1] : text[index + length]; - return adjacentChar !== undefined && PUNCTUATION.test(adjacentChar); - } - return true; -} - -function isFlanking( - text: string, - index: number, - length: number, - isLeft: boolean, -): boolean { - const charBefore = text[index - 1]; - const charAfter = text[index + length]; - - const [primary, secondary] = isLeft - ? [charAfter, charBefore] - : [charBefore, charAfter]; - - if (primary === undefined || WHITESPACE.test(primary)) { - return false; - } - if (!PUNCTUATION.test(primary)) { - return true; } - return ( - secondary === undefined || - WHITESPACE.test(secondary) || - PUNCTUATION.test(secondary) - ); + return outermost; } export function importTextFormatTransformer( diff --git a/packages/lexical-markdown/src/importTextMatchTransformer.ts b/packages/lexical-markdown/src/importTextMatchTransformer.ts index 8f195ca2e86..2de4f01ecf0 100644 --- a/packages/lexical-markdown/src/importTextMatchTransformer.ts +++ b/packages/lexical-markdown/src/importTextMatchTransformer.ts @@ -12,15 +12,53 @@ import {scanCodeSpans} from './importTextFormatTransformer'; import {getLinkLabelOpener, LINK} from './MarkdownTransformers'; import {isEscaped} from './utils'; +/** + * A copy of `match` for `text`, which is what is left of the text it was + * found in after `distance` characters. + */ +function moveMatch( + match: RegExpExecArray, + text: string, + distance: number, +): RegExpExecArray { + const moved = [...match] as RegExpExecArray; + moved.index = match.index - distance; + moved.input = text; + moved.groups = match.groups; + if (match.indices) { + const indices = match.indices.map( + range => range && [range[0] - distance, range[1] - distance], + ) as RegExpIndicesArray; + const {groups} = match.indices; + if (groups) { + // Each named group's range is the one of its numbered group. + const ranges: Record = + Object.create(null); + for (const name in groups) { + const range = groups[name]; + ranges[name] = range && indices[match.indices.indexOf(range)]; + } + indices.groups = ranges as typeof groups; + } + moved.indices = indices; + } + return moved; +} + +/** Code spans, in order, from where they start to where they end. */ +export type CodeSpans = readonly {startIndex: number; endIndex: number}[]; + /** * The first link in `text`, starting at the `[` that really opens its text, - * with none of its syntax inside a code span. - * The import pattern's lazy label runs from the first `[` to the first `](`, - * so a literal `[` before the link (`[1] *see* [here](url)`) would otherwise - * start the match and leave the text between them unparsed. Matching again - * from the real opener lets that text go through the other transformers. + * with none of its syntax inside a code span. `codeSpans` are those of the + * text that `text` is what is left of after `offset`, when they are known. */ -function matchLink(text: string, regExp: RegExp): RegExpExecArray | null { +function matchLink( + text: string, + regExp: RegExp, + codeSpans: CodeSpans | undefined, + offset: number, +): RegExpExecArray | null { const search = new RegExp( regExp.source, regExp.flags.replace(/[gy]/g, '') + 'g', @@ -28,55 +66,134 @@ function matchLink(text: string, regExp: RegExp): RegExpExecArray | null { // Code spans bind tighter than link brackets // (https://spec.commonmark.org/0.31.2/#code-spans), so a link whose syntax // falls inside one is not a link. - const codeSpans = text.includes('`') ? scanCodeSpans(text) : []; - // The spans come in order, so a binary search finds the one that could hold - // `index`. Returns the end of that span, or -1 outside every span. - const codeSpanEnd = (index: number) => { + let spans = codeSpans; + let spanOffset = offset; + if (spans === undefined) { + spans = text.includes('`') ? scanCodeSpans(text) : []; + spanOffset = 0; + } + // The spans come in order, so a binary search finds the one that could + // hold `index`, or -1 outside every span. + const codeSpanAt = (index: number) => { let low = 0; - let high = codeSpans.length - 1; + let high = spans.length - 1; while (low <= high) { const middle = (low + high) >> 1; - const {startIndex, endIndex} = codeSpans[middle]; - if (index < startIndex) { + const {startIndex, endIndex} = spans[middle]; + if (index + spanOffset < startIndex) { high = middle - 1; - } else if (index >= endIndex) { + } else if (index + spanOffset >= endIndex) { low = middle + 1; } else { - return endIndex; + return middle; } } return -1; }; + // No link starts after the last `](`, and searching from each `[` there + // would walk on to the end of the text. + const lastCloser = text.lastIndexOf(']('); let match; - while ((match = search.exec(text)) !== null) { - const labelEnd = - match.index + 1 + (match[1] !== undefined ? match[1].length : 0); - const openerCodeEnd = codeSpanEnd(match.index); - if (openerCodeEnd !== -1) { + while ( + search.lastIndex < lastCloser && + (match = search.exec(text)) !== null + ) { + const openerSpan = codeSpanAt(match.index); + if (openerSpan !== -1) { // No link can open inside this code span. - search.lastIndex = openerCodeEnd; + search.lastIndex = spans[openerSpan].endIndex - spanOffset; continue; } + const labelEnd = match.index + 1 + match[1].length; if ( - isEscaped(text, match.index) || - codeSpanEnd(labelEnd) !== -1 || - codeSpanEnd(match.index + match[0].length - 1) !== -1 + codeSpanAt(labelEnd) !== -1 || + codeSpanAt(match.index + match[0].length - 1) !== -1 ) { - search.lastIndex = match.index + 1; + // A search from any `[` before the last character of the label ends + // at the same `](` and `)`, so it is not a link either. + search.lastIndex = labelEnd - 1; continue; } - const opener = match[1] !== undefined ? getLinkLabelOpener(match[1]) : -1; - if (opener === null || opener === -1) { + // A `[` that the label holds outside a code span is the link's real + // opener, and what comes before it is ordinary text, so the search goes + // on from there, where it finds the same link. A link with an empty + // label, as in `x[[](u)`, stays text, as `[](u)` does. + const opener = getLinkLabelOpener(match[1]); + if (opener !== null && opener >= 0) { + search.lastIndex = match.index + 1 + opener; + continue; + } + if (!isEscaped(text, match.index)) { return match; } - search.lastIndex = match.index + 1 + opener; + // The label of an escaped `[` holds no opener, so each escaped `[` up to + // the next one written as is finds what the last of them finds: that + // opener or nothing. + let next = match.index; + do { + next = text.indexOf('[', next + 1); + } while (next !== -1 && isEscaped(text, next)); + if (next === -1) { + break; + } + const last = text.lastIndexOf('[', next - 1); + search.lastIndex = last > match.index ? last : next; } return null; } +const lookless = new WeakMap(); + +/** + * Whether a match of `regExp` depends only on the text from where it starts + * on: it has no `^`, word boundary or lookbehind outside a character class, + * and no flag that makes it keep state. Its first match in what is left of a + * text after a cut is then the first one found at or after the cut. + */ +function looksOnlyAhead(regExp: RegExp): boolean { + let result = lookless.get(regExp); + if (result === undefined) { + const {source} = regExp; + result = !/[gyv]/.test(regExp.flags); + for (let i = 0, inClass = false; result && i < source.length; i++) { + const character = source[i]; + if (character === '\\') { + result = inClass || !/[bB]/.test(source[++i]); + } else if (inClass) { + inClass = character !== ']'; + } else if (character === '[') { + inClass = true; + } else { + result = + (character !== '^' && !source.startsWith('(?<', i)) || + /^\(\?<[^=!]/.test(source.slice(i, i + 4)); + } + } + lookless.set(regExp, result); + } + return result; +} + +/** + * The match each text-match transformer found in a text and the offset of + * the text it was found in, or `null` when there is none. Kept for the built-in link syntax and for patterns that only look + * ahead, for which it is still the first match in what is left of the text + * after any offset up to where it was found. + */ +export type TextMatches = Map< + TextMatchTransformer, + {match: RegExpExecArray; offset: number} | null +>; + export function findOutermostTextMatchTransformer( textNode_: TextNode, textMatchTransformers: TextMatchTransformer[], + // The matches found earlier in the text that `textNode_` is what is left + // of from `offset`. + matches: TextMatches = new Map(), + offset = 0, + // The code spans of that text, when they are known. + codeSpans?: CodeSpans, ): { startIndex: number; endIndex: number; @@ -95,12 +212,30 @@ export function findOutermostTextMatchTransformer( continue; } const text = textNode.getTextContent(); - // Transformers sharing the built-in link syntax need the same candidate rules. - const match = + const {importRegExp} = transformer; + // Transformers sharing the built-in link syntax need the same candidate + // rules. + const isLink = LINK.importRegExp !== undefined && - transformer.importRegExp.source === LINK.importRegExp.source - ? matchLink(text, transformer.importRegExp) - : text.match(transformer.importRegExp); + importRegExp.source === LINK.importRegExp.source; + let match: RegExpMatchArray | null; + if (isLink || looksOnlyAhead(importRegExp)) { + const found = matches.get(transformer); + if ( + found === undefined || + (found !== null && found.offset + found.match.index < offset) + ) { + const result = isLink + ? matchLink(text, importRegExp, codeSpans, offset) + : importRegExp.exec(text); + matches.set(transformer, result && {match: result, offset}); + match = result; + } else { + match = found && moveMatch(found.match, text, offset - found.offset); + } + } else { + match = text.match(importRegExp); + } if (!match) { continue; @@ -157,6 +292,8 @@ export function importFoundTextMatchTransformer( transformedNode?: TextNode; nodeBefore: TextNode | undefined; // If split nodeAfter: TextNode | undefined; // If split + // The text of `nodeAfter` before the transformer ran, which may change it. + textAfter: string | undefined; } | null { let transformedNode, nodeAfter, nodeBefore; @@ -172,12 +309,14 @@ export function importFoundTextMatchTransformer( if (!transformer.replace) { return null; } + const textAfter = nodeAfter && nodeAfter.getTextContent(); const potentialTransformedNode = transformer.replace(transformedNode, match); return { matchedNode: transformedNode, nodeAfter, nodeBefore, + textAfter, transformedNode: potentialTransformedNode || undefined, }; } diff --git a/packages/lexical-markdown/src/importTextTransformers.ts b/packages/lexical-markdown/src/importTextTransformers.ts index 8b26df811c8..0a510185c98 100644 --- a/packages/lexical-markdown/src/importTextTransformers.ts +++ b/packages/lexical-markdown/src/importTextTransformers.ts @@ -11,14 +11,26 @@ import type {TextMatchTransformer} from './MarkdownTransformers'; import {$isTextNode, type LexicalNode, type TextNode} from 'lexical'; import { + advanceFormatScan, findOutermostTextFormatTransformer, + findPairAtEdge, + type FormatScan, importTextFormatTransformer, + partlyHoldsCodeSpan, + scanTextFormats, } from './importTextFormatTransformer'; import { findOutermostTextMatchTransformer, importFoundTextMatchTransformer, + type TextMatches, } from './importTextMatchTransformer'; -import {unescapeText} from './utils'; +import { + characterAt, + characterBefore, + getDelimiterRunEdge, + getEmphasisSides, + unescapeText, +} from './utils'; /** * Returns true if the node can contain transformable markdown. @@ -33,41 +45,227 @@ export function canContainTransformableMarkdown( } /** - * Handles applying both text format and text match transformers. - * It finds the outermost text format or text match and applies it, - * then recursively calls itself to apply the next outermost transformer, + * Whether the delimiter run that starts `text` can open or close otherwise + * beside the edge of a line than beside `before`. Whether it can close + * matters even with nothing before it, through the rule of 3. + */ +function readsOtherwiseBesideEdge( + text: string, + before: string | undefined, + outside: string | undefined, +): boolean { + const edge = getDelimiterRunEdge(text, 1, 1); + const after = edge < text.length ? characterAt(text, edge) : outside; + const atEdge = getEmphasisSides(text[0], undefined, after); + const besideBefore = getEmphasisSides(text[0], before, after); + return ( + atEdge.canOpen !== besideBefore.canOpen || + atEdge.canClose !== besideBefore.canClose + ); +} + +/** The markers whose runs CommonMark and GFM define. */ +const SPEC_MARKERS = '*_~'; + +/** + * What a delimiter run cut at `index` of `text` sees beyond its far end once + * the text is split there: before the run that starts at `index` when `step` + * is -1, after the run that ends there when it is 1. `outside` is what lies + * beyond the text. Beside a text match, which is written in a syntax of its + * own, a run that the cut does not go through sees the edge of a line. + */ +function characterBeyondRun( + text: string, + index: number, + step: 1 | -1, + outside: string | undefined, + delimiters: Set, + isBesideTextMatch = false, +): string | undefined { + // Only a delimiter's run is walked: beside anything else, what lies beyond + // is never read, and a long run of it would be walked at every cut. + const beside = text[step === 1 ? index - 1 : index]; + const edge = + beside === '`' || delimiters.has(beside) + ? getDelimiterRunEdge(text, index, step) + : index; + if (isBesideTextMatch && edge === index) { + return undefined; + } + return step === 1 + ? edge < text.length + ? characterAt(text, edge) + : outside + : edge > 0 + ? characterBefore(text, edge) + : outside; +} + +/** + * Handles applying both text format and text match transformers. It finds + * the outermost text format or text match and applies it, goes on with the + * text after it the same way, and then, from the last match back to the + * first, imports the text before each match and then the text inside it, * until there are no more transformers to apply. */ export function importTextTransformers( textNode: TextNode, textFormatTransformersIndex: TextFormatTransformersIndex, textMatchTransformers: TextMatchTransformer[], + // The characters around this text in the text it was split from, so that + // a delimiter run at its edge reads as it did there. + before?: string, + after?: string, ) { + // The text after each match is imported in this loop rather than by + // recursion, so that a long paragraph does not nest a call per match, and + // with what was found in the whole text, which still holds for the rest of + // it as long as no match there reached into what came before. + const deferred: Deferred[] = []; + let rest: TextRest | null = {after, before, node: textNode, scan: null}; + while (rest !== null) { + rest = importOutermost( + rest, + textFormatTransformersIndex, + textMatchTransformers, + deferred, + ); + } + for (let i = deferred.length - 1; i >= 0; i--) { + const {outside, inside, matched} = deferred[i]; + importTextPart(outside, textFormatTransformersIndex, textMatchTransformers); + importTextPart(inside, textFormatTransformersIndex, textMatchTransformers); + if ( + !canContainTransformableMarkdown(inside.node) && + canContainTransformableMarkdown(matched) && + matched.getParent() !== null + ) { + // A text match transformer that returned no node may leave the matched + // node in place or move it into a new parent. Read its escapes once + // without trying the same match again. A parent also exists when + // importing into a detached tree. + matched.setTextContent(unescapeText(matched.getTextContent())); + } + } +} + +/** Text to import, with the characters around it in the text it was cut from. */ +interface TextPart { + node: LexicalNode | undefined; + before: string | undefined; + after: string | undefined; +} + +function importTextPart( + {node, before, after}: TextPart, + textFormatTransformersIndex: TextFormatTransformersIndex, + textMatchTransformers: TextMatchTransformer[], +) { + if (canContainTransformableMarkdown(node)) { + importTextTransformers( + node, + textFormatTransformersIndex, + textMatchTransformers, + before, + after, + ); + } +} + +/** + * What a match leaves to import once the text after it is done: the text + * before it, then the text inside it, or the escapes of the matched node of + * a text match that returned no node. + */ +interface Deferred { + outside: TextPart; + inside: TextPart; + matched: LexicalNode | undefined; +} + +/** + * The formats and text matches found in a text, for what is left of it, which is + * `rest`. + */ +interface TextScan { + formats: FormatScan; + matches: TextMatches; + rest: string; +} + +/** The text after a match, and the scan of the text it was cut from. */ +interface TextRest { + node: TextNode; + before: string | undefined; + after: string | undefined; + scan: TextScan | null; +} + +/** + * Applies the outermost transformer in `textNode`, adds the import of the + * text before and inside its match to `deferred`, and returns the text after + * it, for the caller to import next. + */ +function importOutermost( + {node: textNode, before, after, scan: previousScan}: TextRest, + textFormatTransformersIndex: TextFormatTransformersIndex, + textMatchTransformers: TextMatchTransformer[], + deferred: Deferred[], +): TextRest | null { + const text = textNode.getTextContent(); + const scan: TextScan = + previousScan !== null && previousScan.rest === text + ? previousScan + : { + formats: scanTextFormats( + text, + textFormatTransformersIndex, + before, + after, + ), + matches: new Map(), + rest: text, + }; + const {formats} = scan; + const {delimiters} = formats; let foundTextFormat = findOutermostTextFormatTransformer( - textNode, + text, textFormatTransformersIndex, + formats, ); let foundTextMatch = findOutermostTextMatchTransformer( textNode, textMatchTransformers, + scan.matches, + formats.offset, + // The scan finds code spans only for a code transformer. + textFormatTransformersIndex.transformersByTag['`'] + ? formats.codeSpans + : undefined, ); if (foundTextFormat && foundTextMatch) { - if (foundTextFormat.isCodeSpan) { - // Code spans bind tighter than text-match transformers (e.g. equations - // or links). A code span must never be partially consumed by a text - // match, so prefer the code format unless the text match fully wraps it - // (in which case the code span is part of the match's raw content). - // https://github.com/meta/lexical/issues/8687 + // Code spans bind tighter than text-match transformers (e.g. equations + // or links). A code span must never be partially consumed by a text + // match, so the match goes only when it fully wraps the span (in which + // case the span is part of the match's raw content). + // https://github.com/meta/lexical/issues/8687 + const {startIndex, endIndex} = foundTextMatch; + if (partlyHoldsCodeSpan(formats, startIndex, endIndex)) { + // Apply the code span, or the emphasis before it, and not the text + // match, which would take part of the span. + foundTextMatch = null; + } else if (foundTextFormat.isCodeSpan) { + // The text match goes first when it wraps the code span or ends before + // it; one after it waits for the span. if ( - foundTextMatch.startIndex <= foundTextFormat.startIndex && - foundTextMatch.endIndex >= foundTextFormat.endIndex + (startIndex <= foundTextFormat.startIndex && + endIndex >= foundTextFormat.endIndex) || + endIndex <= foundTextFormat.startIndex ) { - // foundTextMatch wraps the code span - apply foundTextMatch foundTextFormat = null; } else { - // Apply the code span foundTextMatch = null; } } else if ( @@ -85,6 +283,85 @@ export function importTextTransformers( } } + // Defers the import of the text before a match from `startIndex` to + // `endIndex` and of the text `inside` it (or the escapes of `matched`), + // and returns the text after it, each with what lies beyond the run cut at + // its edge. A text match's transformer has run, and when it changed the + // text after it, that text is imported on its own, as before it was cut + // from the line; `textAfter` is what that text was before, and `null` for + // a format match. + const importAround = ( + { + nodeBefore, + nodeAfter, + }: { + nodeBefore: LexicalNode | undefined; + nodeAfter: LexicalNode | undefined; + }, + textAfter: string | null | undefined, + startIndex: number, + endIndex: number, + inside: TextPart, + matched: LexicalNode | undefined, + ): TextRest | null => { + const isTextMatch = textAfter !== null; + deferred.push({ + inside, + matched, + outside: { + after: characterBeyondRun( + text, + startIndex, + 1, + after, + delimiters, + isTextMatch, + ), + before, + node: nodeBefore, + }, + }); + if (!canContainTransformableMarkdown(nodeAfter)) { + return null; + } + const rest = nodeAfter.getTextContent(); + if (textAfter !== null && textAfter !== rest) { + return {after: undefined, before: undefined, node: nodeAfter, scan: null}; + } + const restBefore = characterBeyondRun( + text, + endIndex, + -1, + before, + delimiters, + isTextMatch, + ); + // The scan read a run at the start of the rest beside the end of the + // match. Read beside the edge of a line instead, it can only change the + // pair it opens, since it has nothing before it in the rest to close. + let isReusable = advanceFormatScan( + formats, + formats.offset + endIndex, + !isTextMatch, + ); + if ( + isReusable && + isTextMatch && + restBefore === undefined && + delimiters.has(rest[0]) && + readsOtherwiseBesideEdge(rest, characterBefore(text, endIndex), after) + ) { + isReusable = findPairAtEdge(formats, rest, textFormatTransformersIndex); + } + scan.rest = rest; + return { + after, + before: restBefore, + node: nodeAfter, + scan: isReusable ? scan : null, + }; + }; + if (foundTextFormat) { const result = importTextFormatTransformer( textNode, @@ -93,28 +370,29 @@ export function importTextTransformers( foundTextFormat.transformer, foundTextFormat.match, ); - - if (canContainTransformableMarkdown(result.nodeAfter)) { - importTextTransformers( - result.nodeAfter, - textFormatTransformersIndex, - textMatchTransformers, - ); - } - if (canContainTransformableMarkdown(result.nodeBefore)) { - importTextTransformers( - result.nodeBefore, - textFormatTransformersIndex, - textMatchTransformers, - ); - } - if (canContainTransformableMarkdown(result.transformedNode)) { - importTextTransformers( - result.transformedNode, - textFormatTransformersIndex, - textMatchTransformers, - ); - } + const {startIndex, endIndex, isCodeSpan} = foundTextFormat; + const tag = foundTextFormat.match[1]; + // The content reads a run at its edge beside the tag, so that it flanks + // as it does in the line, when it is of a marker that CommonMark or GFM + // define and the tag does not cut it; otherwise beside the edge of a + // line, as the rule of 3 would measure a cut run without the part the + // tag took. A code span's content is not read for emphasis. + const beside = (char: string) => + !isCodeSpan && SPEC_MARKERS.includes(char) && char !== tag[0] + ? tag[0] + : undefined; + return importAround( + result, + null, + startIndex, + endIndex, + { + after: beside(text[endIndex - tag.length - 1]), + before: beside(text[startIndex + tag.length]), + node: result.transformedNode, + }, + undefined, + ); } else if (foundTextMatch) { const result = importFoundTextMatchTransformer( textNode, @@ -124,49 +402,21 @@ export function importTextTransformers( foundTextMatch.match, ); if (!result) { - return; - } - - if (canContainTransformableMarkdown(result.nodeAfter)) { - importTextTransformers( - result.nodeAfter, - textFormatTransformersIndex, - textMatchTransformers, - ); - } - if (canContainTransformableMarkdown(result.nodeBefore)) { - importTextTransformers( - result.nodeBefore, - textFormatTransformersIndex, - textMatchTransformers, - ); - } - if (canContainTransformableMarkdown(result.transformedNode)) { - importTextTransformers( - result.transformedNode, - textFormatTransformersIndex, - textMatchTransformers, - ); - } else if ( - result.transformedNode === undefined && - canContainTransformableMarkdown(result.matchedNode) && - result.matchedNode.getParent() !== null - ) { - // A void return may leave the matched node in place or move it into a - // new parent. Read its escapes once without trying the same match again. - // A parent also exists when importing into a detached tree. - result.matchedNode.setTextContent( - unescapeText(result.matchedNode.getTextContent()), - ); + return null; } + return importAround( + result, + result.textAfter, + foundTextMatch.startIndex, + foundTextMatch.endIndex, + {after: undefined, before: undefined, node: result.transformedNode}, + result.transformedNode === undefined ? result.matchedNode : undefined, + ); } - // Handle escape characters. Only a node that no transformer matched is - // unescaped here: retained non-code results above have already had their - // escapes read, and code spans must keep their backslashes. - if (!foundTextFormat && !foundTextMatch) { - const textContent = textNode.getTextContent(); - const unescapedText = unescapeText(textContent); - textNode.setTextContent(unescapedText); - } + // Handle escape characters. Only a node that no transformer matched gets + // here: retained non-code results above have already had their escapes + // read, and code spans must keep their backslashes. + textNode.setTextContent(unescapeText(text)); + return null; } diff --git a/packages/lexical-markdown/src/utils.ts b/packages/lexical-markdown/src/utils.ts index 200b5c10b56..806f78dd09b 100644 --- a/packages/lexical-markdown/src/utils.ts +++ b/packages/lexical-markdown/src/utils.ts @@ -56,9 +56,146 @@ export function transformersByType(transformers: Transformer[]): Readonly<{ }; } -export const PUNCTUATION_OR_SPACE = /[!-/:-@[-`{-~\s]/; -export const WHITESPACE = /\s/; -export const PUNCTUATION = /[!"#$%&'()*+,\-./:;<=>?@[\]^_`{|}~]/; +// CommonMark counts every character in the Unicode P (punctuation) and S +// (symbol) general categories as punctuation, which includes all of the ASCII +// punctuation characters. Its Unicode whitespace is Zs plus tab, line feed, +// form feed and carriage return; JavaScript's `\s` adds others, such as +// U+FEFF. Delimiter flanking depends on both. +// https://spec.commonmark.org/0.31.2/#unicode-punctuation-character +// https://spec.commonmark.org/0.31.2/#unicode-whitespace-character +const PUNCTUATION = /[\p{P}\p{S}]/u; +const WHITESPACE = /[\p{Zs}\t\n\f\r]/u; + +const WHITESPACE_CLASS = 0; +const PUNCTUATION_CLASS = 1; +const OTHER_CLASS = 2; + +/** How flanking sees `character`; the edge of the line is whitespace. */ +function classify(character: string | undefined): number { + return character === undefined || WHITESPACE.test(character) + ? WHITESPACE_CLASS + : PUNCTUATION.test(character) + ? PUNCTUATION_CLASS + : OTHER_CLASS; +} + +/** + * Whether a delimiter run with `primary` on the side it faces (after it for + * left-flanking, before it for right-flanking) and `secondary` on the other + * side is flanking. `undefined` is the edge of the line. + * https://spec.commonmark.org/0.31.2/#left-flanking-delimiter-run + */ +export function isFlankedBy( + primary: string | undefined, + secondary: string | undefined, +): boolean { + return isFlankedByClass(classify(primary), classify(secondary)); +} + +function isFlankedByClass(primary: number, secondary: number): boolean { + return ( + primary === OTHER_CLASS || + (primary === PUNCTUATION_CLASS && secondary !== OTHER_CLASS) + ); +} + +/** + * Whether a delimiter run of `marker` between `before` and `after` can open + * and close emphasis; `_` opens or closes inside a word only beside + * punctuation. + * https://spec.commonmark.org/0.31.2/#can-open-emphasis + */ +export function getEmphasisSides( + marker: string, + before: string | undefined, + after: string | undefined, +): {canOpen: boolean; canClose: boolean} { + const beforeClass = classify(before); + const afterClass = classify(after); + const isLeft = isFlankedByClass(afterClass, beforeClass); + const isRight = isFlankedByClass(beforeClass, afterClass); + return marker === '_' + ? { + canClose: isRight && (!isLeft || afterClass === PUNCTUATION_CLASS), + canOpen: isLeft && (!isRight || beforeClass === PUNCTUATION_CLASS), + } + : {canClose: isRight, canOpen: isLeft}; +} + +export function isPunctuationOrSpace(character: string): boolean { + return classify(character) !== OTHER_CLASS; +} + +// Readers built on micromark, such as `@lexical/mdast`, read a character +// outside the Basic Multilingual Plane one UTF-16 unit at a time, so as a +// letter, and count JavaScript's whitespace, U+FEFF among it, as whitespace. +// Export has to open and close for those readers as well. +const MICROMARK_ONLY_WHITESPACE = /[\v\u2028\u2029\ufeff]/; +function asMicromarkReads(character: string | undefined): string | undefined { + return character === undefined + ? character + : character.length > 1 + ? 'a' + : MICROMARK_ONLY_WHITESPACE.test(character) + ? ' ' + : character; +} + +/** {@link isFlankedBy}, both for CommonMark and for micromark. */ +export function isPortablyFlankedBy( + primary: string | undefined, + secondary: string | undefined, +): boolean { + return ( + isFlankedBy(primary, secondary) && + isFlankedBy(asMicromarkReads(primary), asMicromarkReads(secondary)) + ); +} + +/** {@link getEmphasisSides}, both for CommonMark and for micromark. */ +export function getPortableEmphasisSides( + marker: string, + before: string | undefined, + after: string | undefined, +): {canOpen: boolean; canClose: boolean} { + const sides = getEmphasisSides(marker, before, after); + const micromark = getEmphasisSides( + marker, + asMicromarkReads(before), + asMicromarkReads(after), + ); + return { + canClose: sides.canClose && micromark.canClose, + canOpen: sides.canOpen && micromark.canOpen, + }; +} + +/** Whether CommonMark or micromark reads `character` as a letter. */ +export function isLetterForSomeReader(character: string): boolean { + return ( + !isPunctuationOrSpace(character) || + !isPunctuationOrSpace(asMicromarkReads(character)!) + ); +} + +/** The character (code point) that starts at `index`, if any. */ +export function characterAt(text: string, index: number): string | undefined { + const code = text.codePointAt(index); + return code === undefined ? undefined : String.fromCodePoint(code); +} + +/** The character (code point) that ends just before `index`, if any. */ +export function characterBefore( + text: string, + index: number, +): string | undefined { + const pair = characterAt(text, index - 2); + return pair !== undefined && pair.length === 2 ? pair : text[index - 1]; +} + +export function toCharacterReference(character: string): string { + return `&#${character.codePointAt(0)};`; +} const MARKDOWN_EMPTY_LINE_REG_EXP = /^\s{0,3}$/; @@ -113,3 +250,26 @@ export function isEscaped(text: string, index: number): boolean { } return backslashes % 2 === 1; } + +/** + * Where the delimiter run that `text` is cut in at `index` ends: walking from + * `index` by `step` over the delimiter beside the cut, the first edge with + * something else beyond it. Walking back, an escaped delimiter is not part of + * the run. + */ +export function getDelimiterRunEdge( + text: string, + index: number, + step: 1 | -1, +): number { + const delimiter = text[step === 1 ? index - 1 : index]; + let edge = index; + while ( + step === 1 + ? edge < text.length && text[edge] === delimiter + : edge > 0 && text[edge - 1] === delimiter && !isEscaped(text, edge - 1) + ) { + edge += step; + } + return edge; +}