const STOP = new Set(
  "a an and are as at be but by can did do does for from has have he her his how i in into is it its me my of on or our she so that the their them then there these they this to us was we what when where which who why will with would you your about tell show give".split(
    " ",
  ),
);

/** Lowercase, strip accents/punctuation, keep dots inside domains (quote.dev). */
export function normalize(input: string): string {
  return input
    .normalize("NFKD")
    .replace(/[\u0300-\u036f]/g, "")
    .toLowerCase()
    .replace(/[^a-z0-9.+#/\-\s]/g, " ")
    .replace(/(^|\s)[./\-]+|[./\-]+(?=\s|$)/g, " ")
    .replace(/\s+/g, " ")
    .trim();
}

/** Very small suffix stemmer — enough for "agents"→"agent", "shipping"→"ship". */
export function stem(token: string): string {
  if (token.length <= 3 || /[.\d]/.test(token)) return token;
  for (const [suf, rep] of [
    ["ies", "y"],
    ["ing", ""],
    ["ed", ""],
    ["es", ""],
    ["s", ""],
  ] as const) {
    if (token.endsWith(suf) && token.length - suf.length >= 3) {
      const base = token.slice(0, -suf.length) + rep;
      return base.replace(/(.)\1$/, "$1");
    }
  }
  return token;
}

export function tokenize(input: string, keepStop = false): string[] {
  const out: string[] = [];
  for (const raw of normalize(input).split(" ")) {
    if (!raw) continue;
    if (!keepStop && STOP.has(raw)) continue;
    out.push(stem(raw));
    // also index the pieces of compound tokens (quote.dev → quote, dev; geo/aeo → geo, aeo)
    if (/[./\-]/.test(raw)) for (const part of raw.split(/[./\-]+/)) if (part && part.length > 1 && !STOP.has(part)) out.push(stem(part));
  }
  return out;
}