fix(webui): render TeX math delimiters

webui/src/lib/remark-tex-math.ts

@@ -0,0 +1,297 @@
+import type { Root } from "mdast";
+import type { Code, Construct, Effects, Extension, State, Token } from "micromark-util-types";
+import type { Plugin } from "unified";
+import type {} from "micromark-extension-math";
+
+const BACKSLASH = 92;
+const DOLLAR = 36;
+const LEFT_PAREN = 40;
+const RIGHT_PAREN = 41;
+const LEFT_BRACKET = 91;
+const RIGHT_BRACKET = 93;
+const SPACE = 32;
+const CARET = 94;
+const UNDERSCORE = 95;
+const EQUALS = 61;
+const PLUS = 43;
+const SLASH = 47;
+const LESS_THAN = 60;
+const GREATER_THAN = 62;
+const LEFT_BRACE = 123;
+const RIGHT_BRACE = 125;
+const PIPE = 124;
+
+type ProcessorData = {
+  micromarkExtensions?: Extension[];
+};
+
+const texMathSyntax: Extension = {
+  flow: {
+    [BACKSLASH]: {
+      tokenize: tokenizeTexMathFlow,
+      concrete: true,
+      name: "texMathFlow",
+    },
+  },
+  text: {
+    [BACKSLASH]: {
+      tokenize: tokenizeTexMathText,
+      name: "texMathText",
+    },
+    [DOLLAR]: {
+      tokenize: tokenizeGuardedDollarMathText,
+      name: "guardedDollarMathText",
+    },
+  },
+};
+
+export const remarkTexMath: Plugin<[], Root> = function remarkTexMath() {
+  const data = this.data() as ProcessorData;
+  const micromarkExtensions =
+    data.micromarkExtensions || (data.micromarkExtensions = []);
+
+  micromarkExtensions.push(texMathSyntax);
+};
+
+function isLineEnding(code: Code): boolean {
+  return code === -5 || code === -4 || code === -3;
+}
+
+function isDigit(code: Code): boolean {
+  return code !== null && code >= 48 && code <= 57;
+}
+
+function isOpeningDollarBlocked(code: Code): boolean {
+  return code === null || code === DOLLAR || code === SPACE || isLineEnding(code);
+}
+
+function isMathSignal(code: Code): boolean {
+  return code === BACKSLASH
+    || code === CARET
+    || code === UNDERSCORE
+    || code === EQUALS
+    || code === PLUS
+    || code === SLASH
+    || code === LESS_THAN
+    || code === GREATER_THAN
+    || code === LEFT_BRACE
+    || code === RIGHT_BRACE
+    || code === PIPE;
+}
+
+const texMathFlowClose: Construct = {
+  tokenize: tokenizeTexMathFlowClose,
+  partial: true,
+};
+
+// Model output commonly uses `$...$`; numeric-only spans are usually prices, not formulas.
+function tokenizeGuardedDollarMathText(effects: Effects, ok: State, nok: State): State {
+  let hasMathSignal = false;
+  let hasContent = false;
+  let firstDataCode: Code = null;
+  let previousDataCode: Code = null;
+
+  return start;
+
+  function start(code: Code): State | undefined {
+    effects.enter("mathText");
+    effects.enter("mathTextSequence");
+    effects.consume(code);
+    return open;
+  }
+
+  function open(code: Code): State | undefined {
+    if (isOpeningDollarBlocked(code)) return nok(code);
+
+    effects.exit("mathTextSequence");
+    effects.enter("mathTextData");
+    return data(code);
+  }
+
+  function data(code: Code): State | undefined {
+    if (code === null || isLineEnding(code)) {
+      effects.exit("mathTextData");
+      return nok(code);
+    }
+
+    if (code === DOLLAR) {
+      effects.exit("mathTextData");
+      effects.enter("mathTextSequence");
+      effects.consume(code);
+      effects.exit("mathTextSequence");
+      effects.exit("mathText");
+      return close;
+    }
+
+    consumeData(code);
+    return code === BACKSLASH ? escaped : data;
+  }
+
+  function escaped(code: Code): State | undefined {
+    if (code === null || isLineEnding(code)) {
+      effects.exit("mathTextData");
+      return nok(code);
+    }
+
+    consumeData(code);
+    return data;
+  }
+
+  function close(code: Code): State | undefined {
+    if (!hasContent || previousDataCode === SPACE) return nok(code);
+    if (isDigit(firstDataCode) && !hasMathSignal) return nok(code);
+    return ok(code);
+  }
+
+  function consumeData(code: Code): void {
+    firstDataCode ??= code;
+    hasContent = true;
+    hasMathSignal ||= isMathSignal(code);
+    previousDataCode = code;
+    effects.consume(code);
+  }
+}
+
+function tokenizeTexMathText(effects: Effects, ok: State, nok: State): State {
+  let closeSequence: Token | undefined;
+
+  return start;
+
+  function start(code: Code): State | undefined {
+    effects.enter("mathText");
+    effects.enter("mathTextSequence");
+    effects.consume(code);
+    return open;
+  }
+
+  function open(code: Code): State | undefined {
+    if (code !== LEFT_PAREN) return nok(code);
+
+    effects.consume(code);
+    effects.exit("mathTextSequence");
+    effects.enter("mathTextData");
+    return data;
+  }
+
+  function data(code: Code): State | undefined {
+    if (code === null) {
+      effects.exit("mathTextData");
+      return nok(code);
+    }
+
+    if (code === BACKSLASH) {
+      effects.exit("mathTextData");
+      closeSequence = effects.enter("mathTextSequence");
+      effects.consume(code);
+      return close;
+    }
+
+    effects.consume(code);
+    return data;
+  }
+
+  function close(code: Code): State | undefined {
+    if (code === RIGHT_PAREN) {
+      effects.consume(code);
+      effects.exit("mathTextSequence");
+      effects.exit("mathText");
+      return ok;
+    }
+
+    if (closeSequence) closeSequence.type = "mathTextData";
+    return data(code);
+  }
+}
+
+function tokenizeTexMathFlow(effects: Effects, ok: State, nok: State): State {
+  return start;
+
+  function start(code: Code): State | undefined {
+    effects.enter("mathFlow");
+    effects.enter("mathFlowFence");
+    effects.enter("mathFlowFenceSequence");
+    effects.consume(code);
+    return open;
+  }
+
+  function open(code: Code): State | undefined {
+    if (code !== LEFT_BRACKET) return nok(code);
+
+    effects.consume(code);
+    effects.exit("mathFlowFenceSequence");
+    effects.exit("mathFlowFence");
+    return contentStart;
+  }
+
+  function contentStart(code: Code): State | undefined {
+    if (code === null) return nok(code);
+
+    if (isLineEnding(code)) {
+      effects.enter("lineEnding");
+      effects.consume(code);
+      effects.exit("lineEnding");
+      return contentStart;
+    }
+
+    if (code === BACKSLASH) {
+      return effects.attempt(texMathFlowClose, done, contentStartAfterBackslash)(code);
+    }
+
+    effects.enter("mathFlowValue");
+    return content(code);
+  }
+
+  function content(code: Code): State | undefined {
+    if (code === null) {
+      effects.exit("mathFlowValue");
+      return nok(code);
+    }
+
+    if (isLineEnding(code)) {
+      effects.exit("mathFlowValue");
+      effects.enter("lineEnding");
+      effects.consume(code);
+      effects.exit("lineEnding");
+      return contentStart;
+    }
+
+    if (code === BACKSLASH) {
+      effects.exit("mathFlowValue");
+      return effects.attempt(texMathFlowClose, done, contentStartAfterBackslash)(code);
+    }
+
+    effects.consume(code);
+    return content;
+  }
+
+  function contentStartAfterBackslash(code: Code): State | undefined {
+    effects.enter("mathFlowValue");
+    effects.consume(code);
+    return content;
+  }
+
+  function done(code: Code): State | undefined {
+    effects.exit("mathFlow");
+    return ok(code);
+  }
+}
+
+function tokenizeTexMathFlowClose(effects: Effects, ok: State, nok: State): State {
+  return start;
+
+  function start(code: Code): State | undefined {
+    effects.enter("mathFlowFence");
+    effects.enter("mathFlowFenceSequence");
+    effects.consume(code);
+    return close;
+  }
+
+  function close(code: Code): State | undefined {
+    if (code !== RIGHT_BRACKET) return nok(code);
+
+    effects.consume(code);
+    effects.exit("mathFlowFenceSequence");
+    effects.exit("mathFlowFence");
+    return ok;
+  }
+}