MeCrab

Implementation code

// crates/mecrab-wasm/src/analyzer.rs:200-278 — verbatim, the code running above
pub fn analyze_tokens(&self, text: &str) -> Result<Vec<Token>, MecrabWasmError> {
    let dict = self.dictionary()?;
    check_length(text)?;

    // char.def's own grouping flag decides what gets cut — the flag whose
    // handling is the cubic path. The default type identifies the category so
    // that two adjacent categories are two runs, exactly as mecrab groups
    // them. Characters outside the BMP fall to a default char info whose
    // group bit is 0, so an emoji wall is never cut here; mecrab's own
    // lattice still groups the Default category, bounded at
    // MAX_GROUPING_SIZE + 1 = 25 characters (measured: 4,000 emoji is 3,975
    // single-character tokens plus one 25-character node), which is exactly
    // why this guard does not need to cut it too.
    let bounds = segment_bounds(text, |character| {
        let info = dict.char_def.get_char_info(character);
        if info.group() {
            Some(info.default_type())
        } else {
            None
        }
    });

    let solver = ViterbiSolver::new(dict);
    let mut tokens = Vec::new();
    let mut unmatched = 0u32;

    for (segment_start, segment_end) in bounds {
        let segment = text.get(segment_start..segment_end).ok_or_else(|| {
            MecrabWasmError::internal(format!(
                "segment [{segment_start}..{segment_end}] is not a slice of the input"
            ))
        })?;

        let lattice = Lattice::build(segment, dict).map_err(|error| {
            MecrabWasmError::internal(format!("mecrab could not build a lattice: {error}"))
        })?;
        let path = solver.solve(&lattice).map_err(|error| {
            MecrabWasmError::internal(format!("mecrab could not solve the lattice: {error}"))
        })?;

        for node in &path {
            // BOS/EOS are zero-width and are already dropped by mecrab's own
            // backward pass; dropping them again by WIDTH rather than by the
            // surface string EOS itself means a user who types EOS gets their
            // token back.
            if node.end_byte <= node.start_byte {
                continue;
            }
            // Whitespace is not a morpheme, and upstream mecab is the
            // authority: piping 'foo bar baz' through mecab emits three
            // nodes and no whitespace node, and -Owakati answers with
            // single spaces. mecrab's lattice emits a 記号,空白 node per run
            // instead, which reached the page as a focusable but visually
            // empty chip and made the wakati separator indistinguishable
            // from the token beside it. Dropped here rather than at the
            // display layer so BOTH ops inherit it: wakati_line maps over
            // exactly this Vec, which is what keeps "the wakati line is the
            // chips, joined" true by construction. Offsets stay honest —
            // they are still the shim's own measurements of the input, and
            // the only gap they can now leave is whitespace.
            //
            // No backticks in this range: it is copied verbatim into a
            // JavaScript template literal on the page (SNIPPET-SYNC).
            if node.surface.chars().all(char::is_whitespace) {
                continue;
            }
            let is_unknown = if let Some(flag) = unknown_flag(&lattice, node) {
                flag
            } else {
                unmatched = unmatched.saturating_add(1);
                field_count(&node.feature) != IPADIC_FEATURE_FIELDS
            };
            tokens.push(token_from(node, segment_start, is_unknown)?);
        }
    }

    self.unmatched.set(unmatched);
    Ok(tokens)
}

This is the code running above, right now.