#!/usr/bin/env python3
"""Memory: one local index over everything Argoni may read, and word search (brief 0110).

    bin/memory build [--wait]        read everything again, from nothing (--wait: make the
                                     meaning vectors now, in the foreground)
    bin/memory update                a sweep: read the files whose size or time moved,
                                     drop the ones gone (every summary and job runs it)
    bin/memory update FILE...        read these files now (the app's file watcher);
                                     a FILE that is gone drops out
    bin/memory update --budget S     stop reading after S seconds; the next sweep goes on
    bin/memory update --wait         and make the meaning vectors now
    bin/memory find "question" [-k N] [--json]
    bin/memory home < TEXT           where a note likely goes, with no model and no network:
                                     up to three places under data/ as JSON
                                     {"homes": [{"path", "why"}]} (brief 0216, part 6)
    bin/memory status [--json]
    bin/memory context "question" [--json]
                                     the pieces of notes an answer may use (at most six,
                                     about 2,500 tokens), numbered, as the AI's prompt
    bin/memory cite CTX.json < ANSWER
                                     the citation check: only sentences ending with the
                                     number of a piece that was sent are shown (JSON)
    bin/memory deeper [--context CTX.json] "question"
                                     the prompt for "Look deeper": a full chat turn that
                                     reads the folders itself
    bin/memory forget ID... [--file PATH...]
                                     erase these chat lines (and the rows of these files in
                                     the notes' index) from memory at once, so that not a
                                     byte of them is left in the index files (brief 0120).
                                     Exit: 0 done, 1 chat not reached, 2 bad arguments,
                                     3 deleted but clean-up not whole (try again),
                                     4 unexpected failure

Every verb takes --chat DIR (else ARGONI_CHAT): the runner's record of the chat.

The files stay the only truth. The index is one SQLite file,
data/.summary/memory.sqlite: gitignored, never authority, rebuilt when lost.
Design: docs/design/memory.md (§3, §4.1).

What is read, and nothing else:

    data/        always, except .summary/, restored/ (what the runner moved
                 aside), hidden files and folders, and the
                 health, money and family domains while those are off
                 (lifecfg.sensitive_allowed); a note whose subject or area is
                 one of them is skipped the same way
    projects     the repos the user chose (lifecfg.repos(): repos_folders and
                 repos, the same list and skips as every engine script); inside
                 them .md and .txt only, unless config.json `memory_code` is
                 true, which adds code files
    never        .env files, links, hidden folders, node_modules and build
                 output, files over 1 MB, any folder in config.json
                 `memory_skip` (a list: "domains/personal" inside data/, or
                 "~/Repos/old" anywhere), anything outside the chosen list

A section is the unit: every `#`, `##` or `###` heading starts one; a long
one is split at paragraphs, about 1,500 characters each. Each section keeps
its file and first line, so every result names its source.

find, in order, with no model:

    1. plain lookup   decision/brief/report/idea NNNN, PR NNN, a person's
                      name (domains/people/), a project's name, time words
                      ("today", "yesterday", "this week", "last week", a
                      weekday, a date; Arabic too): the files themselves
    2. word search    FTS5 (porter stemmer), title and heading above the body;
                      the best section of each of the best files, with small
                      boosts: the project the question names, decisions for
                      "decide" questions, the kind it names, newer files
Arabic: the plain tokenizer keeps "القرار" and "قرار" apart and keeps the
vowel marks, so each section also carries its Arabic words folded (marks
and tatweel dropped, alef, ya and ta marbuta forms joined, the light10
prefixes and suffixes taken off) in its own column, and an Arabic question
is folded the same way.

Meaning search (brief 0111): when argoni-embed is there (ARGONI_EMBED, the path
the app gives; else engine/embed/out/ in a development checkout) and config.json
`memory_meaning` is not false, every update and build ends by starting
`argoni-embed index` in the background (low priority, one at a time), which
gives each section 384 numbers from multilingual-e5-small, kept in the same
file by the section's text hash, so a rebuild keeps them. find then also asks
`argoni-embed search` for the 50 nearest sections, and the word and meaning
rankings are joined by reciprocal rank fusion (MEANING_WEIGHT). Without it,
find says "words only" and works as before.

The chat (brief 0120): when ARGONI_CHAT (or --chat) names the runner's record folder, the
lines the user and Argoni said are found too. They are NEVER indexed in data/: a copy there
would hand the whole conversation to any run that opens the file (and let a run plant lines
that read as the user's). The chat has its own index, same shape, at
<cache>/chat/<record folder name>/memory.sqlite, where <cache> is ~/Library/Caches/Argoni
(ARGONI_CHAT_CACHE changes it). Only a line whose exact text hash the runner recorded beside
it, and whose id is not in forgotten.ids, is indexed. Inside a run the record is not
reachable (the fence), so there the chat is "not reached", by design.
Standard library only (the Mac's python3: 3.9, SQLite with FTS5).
"""
import datetime
import errno
import fcntl
import hashlib
import json
import os
import re
import sqlite3
import stat
import subprocess
import sys
import time

sys.dont_write_bytecode = True
BIN = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, BIN)
import lifecfg  # noqa: E402

LIFE = lifecfg.root()
DB = os.path.join(LIFE, ".summary", "memory.sqlite")
SCHEMA = 2
# Reciprocal rank fusion of the word and meaning rankings: score = sum of weight / (RRF_K + rank).
RRF_K = 60
MEANING_WEIGHT = 1.0

TEXT_EXT = (".md", ".txt")
CODE_EXT = (".py", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".swift", ".go", ".rs", ".java", ".kt", ".rb",
            ".c", ".h", ".cc", ".cpp", ".hpp", ".m", ".mm", ".cs", ".php", ".sh", ".sql", ".html", ".css",
            ".scss", ".vue", ".svelte", ".yaml", ".yml", ".toml")
MAX_BYTES = 1000000
MAX_CHARS = 1500
SENSITIVE = ("health", "money", "family")
WIDE = 50   # sections taken from the word index before grouping by file
# From this many meaning vectors on, status says a question may pass a tenth of a second on
# an older Mac (brief 0115, this Mac's efficiency cores as a stand-in for an M1: 68 ms at the
# slowest at a million, 336 ms at five million, and 7 GB to update the graph there).
MEANING_WARN = 2000000
# The small boosts on word search, measured on the made-up folder (tests/memory-bench).
BOOST = {"named": 1.3, "decide": 1.5, "kind": 1.5, "page": 1.1, "lifted": 1.3, "new": 1.15, "recent": 1.05}

STOP = set("""a an the of to in on for and or is are was were be been do did does what which who whom
when where why how i me my we our you your it its this that these those with about at by from as
any there their them they he she his her not no yes can could should would will shall may might
have has had get got so if then than too very just up out into over again more most some such only
own same other all each both few s t don now find show tell please""".split())
AR_STOP = set("""في من على الى عن ما ماذا هل اين متى كيف لماذا لم لن هذا هذه ذلك تلك التي الذي الذين
و او ثم ان كان كانت مع عند كل بعض اي انا انت نحن هو هي هم لي لك له لها قد""".split())

QUERY_SECONDS = 600   # no statement runs this long without a file read between (a planted trigger)
QUIET = [0.0]         # when the last file was read or the index opened

SCHEMA_SQL = [
    "create table meta(key text primary key, value text)",
    "create table files(id integer primary key, path text unique, source text, rel text, show text,"
    " kind text, subject text, updated text, title text, size int, mtime real, sections int)",
    "create table sections(id integer primary key, file int, title text, heading text, line int,"
    " text text, pathw text, extra text, hash text)",
    "create index sections_hash on sections(hash)",
    # argoni-embed's: one vector per distinct section text, kept across rebuilds
    "create table if not exists vectors(hash text primary key, v blob)",
    "create index sections_file on sections(file)",
    "create virtual table fts using fts5(pathw, title, heading, text, extra, content='sections',"
    " content_rowid='id', tokenize='porter unicode61 remove_diacritics 2')",
    "create trigger sections_ai after insert on sections begin"
    " insert into fts(rowid, pathw, title, heading, text, extra)"
    " values (new.id, new.pathw, new.title, new.heading, new.text, new.extra); end",
    "create trigger sections_ad after delete on sections begin"
    " insert into fts(fts, rowid, pathw, title, heading, text, extra)"
    " values ('delete', old.id, old.pathw, old.title, old.heading, old.text, old.extra); end",
]

# The chat index only. A side table, not the heading column: the heading is searched and
# shown, so an id there would pollute the words, and argoni-embed reads the main tables, not
# this one. A trigger keeps it whole when a day file's sections are dropped.
CHAT_SQL = [
    "create table chat_ids(section integer primary key, id text, who text, t text, reply_to text)",
    "create index chat_ids_id on chat_ids(id)",
    "create index chat_ids_reply on chat_ids(reply_to)",
    "create trigger chat_ids_ad after delete on sections begin delete from chat_ids where section = old.id; end",
]


# ---------------------------------------------------------------- Arabic

AR_MARKS = re.compile("[ؐ-ًؚ-ٰٟۖ-ۭـ]")
AR_MAP = str.maketrans({"أ": "ا", "إ": "ا", "آ": "ا", "ٱ": "ا", "ى": "ي", "ة": "ه", "ؤ": "و", "ئ": "ي",
                        **{chr(0x660 + i): str(i) for i in range(10)},
                        **{chr(0x6f0 + i): str(i) for i in range(10)}})
AR_WORD = re.compile("[ء-غف-ي٠-٩ٱ-ۓ۰-۹]+")
AR_PREFIXES = ("وال", "بال", "كال", "فال", "لل", "ال")
AR_SUFFIXES = ("ها", "ان", "ات", "ون", "ين", "يه", "ه", "ي")


def ar_norm(text):
    """Vowel marks and tatweel out; the letter forms people type either way joined."""
    return AR_MARKS.sub("", text).translate(AR_MAP)


def ar_stem(word):
    """Light10 (Larkey, Ballesteros and Connell): no dictionary, prefixes and suffixes only."""
    w = ar_norm(word)
    if len(w) > 3 and w.startswith("و"):
        w = w[1:]
    for p in AR_PREFIXES:
        if w.startswith(p) and len(w) - len(p) >= 2:
            w = w[len(p):]
            break
    for s in AR_SUFFIXES:
        if w.endswith(s) and len(w) - len(s) >= 2:
            w = w[:-len(s)]
    return w


def ar_words(text):
    return AR_WORD.findall(ar_norm(text))


# ---------------------------------------------------------------- what may be read

def settings():
    cfg = lifecfg.config(LIFE)
    skips = []
    for entry in cfg.get("memory_skip") or []:
        entry = str(entry or "").strip().rstrip("/")
        if not entry:
            continue
        if entry.startswith(("~", "/")):
            skips.append(os.path.realpath(lifecfg.expand_mac(entry)))
        else:
            skips.append(os.path.realpath(os.path.join(LIFE, entry)))
    ext = TEXT_EXT + (CODE_EXT if cfg.get("memory_code") else ())
    return {"skips": skips, "ext": ext, "sensitive": lifecfg.sensitive_allowed(cfg), "code": bool(cfg.get("memory_code")),
            "meaning": cfg.get("memory_meaning") is not False}


def embedder():
    """argoni-embed, with its model files beside it, or None."""
    for path in (os.environ.get("ARGONI_EMBED"), os.path.join(os.path.dirname(BIN), "embed", "out", "argoni-embed")):
        if path and os.path.isfile(path) and os.access(path, os.X_OK) and \
                os.path.isdir(os.path.join(os.path.dirname(path), "e5-small.mlmodelc")):
            return path
    return None


def meaning_state(db, st, exact=False):
    """(on, one plain line): whether find uses meaning, and why not when it does not.
    exact (status): count only vectors a section uses (about 0.4 s at a million); find
    counts every vector (tens of ms at a million), which can overcount while vectors of
    sections not yet read again are kept (meta.left), so find may say "on" a little early
    (brief 0115)."""
    if not st["meaning"]:
        return False, "meaning search off (memory_meaning is false)"
    if not embedder():
        return False, "meaning search off (argoni-embed is not installed here)"
    total = db.execute("select count(distinct hash) from sections").fetchone()[0]
    have = db.execute("select count(*) from vectors where hash in (select hash from sections)" if exact
                      else "select count(*) from vectors").fetchone()[0]
    have = min(have, total)
    if not have:
        return False, "meaning search not ready yet (0 of %d sections)" % total
    if have < total:
        return True, "meaning search on, %d of %d sections so far" % (have, total)
    return True, "meaning search on"


def start_meaning(st, wait=False):
    """After a sweep: give new sections their vectors, in the background unless wait. The
    chat index (when there is one) follows the notes' index, one at a time."""
    exe = embedder()
    if not exe or not st["meaning"]:
        return None
    dbs = [DB] + [p for p in [chat_index_file()] if p]
    if wait:
        out = None
        for i, path in enumerate(dbs):
            p = subprocess.run([exe, "index", "--db", path, "--fast"], capture_output=True, text=True)
            try:
                r = json.loads(p.stdout.strip().splitlines()[-1])
            except (ValueError, IndexError):
                r = {"error": (p.stderr or p.stdout).strip()[-200:] or "exit %d" % p.returncode}
            if i == 0:
                out = r
            elif r.get("error"):
                print("memory: chat meaning vectors not made: %s" % r["error"])
        return out
    try:
        fd = lifecfg.open_append(os.path.join(os.path.dirname(DB), "memory-embed.log"))
    except OSError:
        fd = subprocess.DEVNULL  # no log beats a log written through a link
    # One shell runs the indexes one after the other, so two never work at once.
    args = [exe, "index", "--db", DB] if len(dbs) == 1 else \
        ["/bin/sh", "-c", '"$0" index --db "$1"; "$0" index --db "$2"', exe] + dbs
    try:
        subprocess.Popen(args, stdout=fd, stderr=fd, stdin=subprocess.DEVNULL, start_new_session=True)
    finally:
        if fd != subprocess.DEVNULL:
            os.close(fd)
    return {"started": True}


def meaning_start(question, n=WIDE, path=None):
    """Start argoni-embed's search (the notes' index unless path names the chat's) and return
    at once, so two searches (notes, chat) run side by side; meaning_finish collects it."""
    exe = embedder()
    if not exe:
        return None
    return subprocess.Popen([exe, "search", "--db", path or DB, "-k", str(n), question],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)


def meaning_finish(proc):
    """[(section id, cosine)] nearest first, or None when it could not run."""
    if proc is None:
        return None
    out, _ = proc.communicate()
    try:
        return [(int(i), float(sc)) for i, sc in json.loads(out)["results"]]
    except (ValueError, KeyError, TypeError):
        return None


def meaning_search(question, n=WIDE, path=None):
    """meaning_start and meaning_finish in one go."""
    return meaning_finish(meaning_start(question, n, path))


_SOURCES = []


def sources():
    """[(label, folder)]: the life folder first, then every repo the user chose. Found once
    per run: `update FILE...` asks for every file, and each ask would walk the folders."""
    if _SOURCES:
        return _SOURCES[0]
    out = [(lifecfg.OWN, os.path.realpath(LIFE))]
    repos, _ = lifecfg.repos(LIFE)
    names = {}
    for r in repos:
        names.setdefault(os.path.basename(r.rstrip("/")), []).append(r)
    for name, paths in names.items():
        for r in paths:
            label = name if len(paths) == 1 else "%s/%s" % (os.path.basename(os.path.dirname(r)), name)
            out.append((label, os.path.realpath(r)))
    _SOURCES.append(out)
    return out


def allowed_name(name, st):
    low = name.lower()
    return not name.startswith(".") and low.endswith(st["ext"]) and not low.startswith(".env")


# The Argoni folder (data/ and the engine beside it) is read once, as the life folder,
# with its own rules (health, money and family while off): never again through a
# chosen repo that happens to hold it.
OWN_TOP = (os.path.realpath(LIFE),) + ((os.path.realpath(os.path.dirname(LIFE)),)
                                        if os.path.isdir(os.path.join(os.path.dirname(LIFE), "engine")) else ())


def skipped_dir(path, rel_parts, label, st):
    real = os.path.realpath(path)
    if any(real == s or real.startswith(s + os.sep) for s in st["skips"]):
        return True
    if label != lifecfg.OWN and real in OWN_TOP:
        return True
    if label == lifecfg.OWN and len(rel_parts) == 1 and rel_parts[0] in lifecfg.NOT_NOTES:
        return True   # restored/, the runs' proposals and archives: not the user's notes (0117)
    if label == lifecfg.OWN and not st["sensitive"] and len(rel_parts) == 2 and rel_parts[0] == "domains" \
            and rel_parts[1] in SENSITIVE:
        return True
    return False


def walk(st):
    """(label, top, path) for every file that may be read, the life folder first."""
    for label, top in sources():
        for d, dirs, names in os.walk(top):
            rel = os.path.relpath(d, top)
            parts = [] if rel == "." else rel.split(os.sep)
            keep = []
            for x in sorted(dirs):
                p = os.path.join(d, x)
                if x.startswith(".") or x in lifecfg.SKIP_DIRS or os.path.islink(p) \
                        or skipped_dir(p, parts + [x], label, st):
                    continue
                keep.append(x)
            dirs[:] = keep
            for n in sorted(names):
                p = os.path.join(d, n)
                if allowed_name(n, st) and not os.path.islink(p):
                    yield label, top, p


def place(path, st):
    """(label, top) when this one file may be read, else None. The same rules as walk()."""
    real = os.path.realpath(path)
    if os.path.islink(path) or not allowed_name(os.path.basename(path), st):
        return None
    for label, top in sources():
        if not real.startswith(top + os.sep):
            continue
        parts = os.path.relpath(real, top).split(os.sep)
        for i, part in enumerate(parts[:-1]):
            sub = os.path.join(top, *parts[:i + 1])
            if part.startswith(".") or part in lifecfg.SKIP_DIRS or skipped_dir(sub, parts[:i + 1], label, st):
                return None
        return label, top
    return None


# ---------------------------------------------------------------- reading one file

def frontmatter(lines):
    """(flat key: value dict, index of the first body line)."""
    if not lines or lines[0].strip() != "---":
        return {}, 0
    meta = {}
    for i in range(1, min(len(lines), 60)):
        if lines[i].strip() == "---":
            return meta, i + 1
        m = re.match(r"([A-Za-z_]+):\s*(.*)", lines[i])
        if m:
            meta[m.group(1)] = m.group(2).strip().strip('"')
    return {}, 0


def kind_of(label, rel, meta, code):
    if code:
        return "code"
    parts = rel.split("/")
    if label == lifecfg.OWN:
        first = parts[0]
        if rel == "now.md":
            return "now"
        if rel == "projects/README.md":
            return "projects"
        if rel.startswith("domains/people/") and parts[-1] != "README.md":
            return "person"
        k = {"me": "me", "projects": "project", "decisions": "decision", "briefs": "brief", "reports": "report",
             "ideas": "idea", "work": "ledger", "log": "log", "inbox": "inbox", "domains": "note"}.get(first)
        return k or note_kind(meta) or "note"
    if rel == "docs/standing.md":
        return "standing"
    for folder, k in (("decisions", "decision"), ("briefs", "brief"), ("reports", "report"), ("research", "research")):
        if "/%s/" % folder in "/" + rel:
            return k
    if parts[-1].lower().startswith("readme"):
        return "readme"
    return note_kind(meta) or "doc"


def note_kind(meta):
    """A note's own `kind:`; never "chat", which only a line of the record is (brief 0120)."""
    k = meta.get("kind")
    return None if k == "chat" else k


def split(lines, start, title, code):
    """[(title, heading, first line, text)]. Headings outside code fences start a
    section; a section over MAX_CHARS is cut at a blank line."""
    out, heading, buf = [], "", []
    fence = False

    def flush():
        i = 0
        while i < len(buf):
            while i < len(buf) and not buf[i][1].strip():
                i += 1
            if i >= len(buf):
                break
            first, size, j, cut = buf[i][0], 0, i, None
            while j < len(buf) and size + len(buf[j][1]) < MAX_CHARS:
                size += len(buf[j][1]) + 1
                if not buf[j][1].strip() and size > MAX_CHARS // 3:
                    cut = j
                j += 1
            end = j if j >= len(buf) else (cut or max(j, i + 1))
            text = "\n".join(l for _, l in buf[i:end]).rstrip()
            if text:
                out.append((title, heading, first, text))
            i = end

    for n, line in enumerate(lines[start:], start + 1):
        if line.lstrip().startswith(("```", "~~~")):
            fence = not fence
        m = None if (fence or code) else re.match(r"(#{1,3}) +(.*)", line)
        if m:
            # A bare title stays its own section: measured, it helps meaning search find a
            # note by its topic (0112: merging it into the next section lost 2 of 7).
            flush()
            buf = []
            if m.group(1) == "#" and not out:
                title = m.group(2).strip()
            heading = m.group(2).strip()
            buf.append((n, line))
        else:
            buf.append((n, line))
    flush()
    return out


def read_file(db, label, top, path, st):
    """Index one file (its old rows dropped first). Returns its section count, or None
    when it may not be kept (gone, unreadable, too big, or health/money/family while off)."""
    QUIET[0] = time.time()
    drop(db, path)
    try:
        # Never through a link, never a pipe or a device: this runs outside every fence,
        # after jobs, on files a run may have planted (0117, security r1).
        fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
        with os.fdopen(fd, "rb") as fh:
            s = os.fstat(fh.fileno())
            if not stat.S_ISREG(s.st_mode) or s.st_size > MAX_BYTES:
                return None
            text = fh.read(MAX_BYTES + 1).decode("utf-8", errors="replace")
    except OSError:
        return None
    lines = text.split("\n")
    meta, start = frontmatter(lines)
    if label == lifecfg.OWN and not st["sensitive"] and \
            (meta.get("subject") in SENSITIVE or meta.get("area") in SENSITIVE):
        return None
    rel = os.path.relpath(path, top).replace(os.sep, "/")
    code = not path.lower().endswith(TEXT_EXT)
    show = rel if label == lifecfg.OWN else "%s: %s" % (label, rel)
    title = os.path.splitext(os.path.basename(path))[0]
    rows = split(lines, start, title, code)
    if rows:
        title = rows[-1][0]
    pathw = re.sub(r"[/_.\-]+", " ", "%s %s" % ("" if label == lifecfg.OWN else label, rel))
    cur = db.execute("insert into files(path, source, rel, show, kind, subject, updated, title, size, mtime, sections)"
                     " values (?,?,?,?,?,?,?,?,?,?,?)",
                     (path, label, rel, show, kind_of(label, rel, meta, code), meta.get("subject"),
                      meta.get("updated") or meta.get("date"), title, s.st_size, s.st_mtime, len(rows)))
    fid = cur.lastrowid
    db.executemany("insert into sections(file, title, heading, line, text, pathw, extra, hash)"
                   " values (?,?,?,?,?,?,?,?)",
                   [(fid, t, h, n, x, pathw, " ".join(ar_stem(w) for w in ar_words(" ".join((t, h, x)))),
                     hashlib.sha1(("%s\n%s" % (t, x)).encode("utf-8")).hexdigest()[:20])
                    for t, h, n, x in rows])
    return len(rows)


def drop(db, path):
    row = db.execute("select id from files where path = ?", (path,)).fetchone()
    if row:
        db.execute("delete from sections where file = ?", (row[0],))
        db.execute("delete from files where id = ?", (row[0],))


# ---------------------------------------------------------------- the index file

def connect(chat_path=None, again=True):
    """The index, made (or remade, when its shape is older than this script) on first use.
    Transactions are explicit, so a rebuild is one transaction: a reader sees the old
    index until the new one is whole. chat_path: the chat's index file instead (its
    folders were checked by chat_index_file); it never sits in data/."""
    path = chat_path or DB
    if not chat_path:
        os.makedirs(os.path.dirname(DB), exist_ok=True)
        try:
            lifecfg.inside(DB, LIFE)  # no link on the way: .summary/ itself could point elsewhere
        except OSError as e:
            sys.exit("memory: %s; the index is not opened" % e)
    # The index and the files SQLite writes beside it (and truncates) must be plain files:
    # a link planted in .summary/ would point those writes at a file elsewhere (brief 0115).
    for f in (path, path + "-wal", path + "-shm", path + "-journal"):
        # One name only: SQLite truncates -shm and -wal in place, so a hard link would empty
        # the file it shares with (code review of brief 0116).
        if os.path.lexists(f) and (not stat.S_ISREG(os.lstat(f).st_mode) or os.lstat(f).st_nlink != 1):
            sys.exit("memory: %s is not a plain file; delete it and run memory build"
                     % (os.path.basename(f) if chat_path else os.path.relpath(f, LIFE)))
    db = sqlite3.connect(path, timeout=15, isolation_level=None)
    if chat_path:
        # A deleted chat line must leave no copy in the pages SQLite frees (brief 0120), and
        # VACUUM's copy of the file stays in memory, not in free blocks of the disk.
        db.execute("pragma secure_delete=on")
        db.execute("pragma temp_store=memory")
    # The index sits in data/.summary, which a run can write (security of 0117, r2 S1): its
    # schema may run nothing it did not put there itself, and no query runs for ever.
    db.execute("pragma trusted_schema=off")
    QUIET[0] = time.time()
    db.set_progress_handler(lambda: 1 if time.time() - QUIET[0] > QUERY_SECONDS else 0, 100000)
    db.execute("pragma journal_mode=wal")
    db.execute("pragma synchronous=normal")
    have = {r[0] for r in db.execute("select name from sqlite_master where type = 'table'")}
    row = db.execute("select value from meta where key = 'schema'").fetchone() if "meta" in have else None
    strange = foreign(db, bool(chat_path))
    if strange and chat_path and again:
        # The chat index is a cache of the record: anything in it this script did not make
        # (a planted table, a changed trigger) means it is removed and made again.
        db.close()
        chat_remove_index(chat_path)
        return connect(chat_path, False)
    if strange:
        # Only what neither this script nor argoni-embed makes: the index itself is kept.
        db.execute("begin immediate")
        for kind, name in strange:
            db.execute('drop %s if exists "%s"' % (kind, name.replace('"', '""')))
        db.execute("commit")
    if not row or row[0] != str(SCHEMA) or (chat_path and "chat_ids" not in have):
        used = "files" in have and db.execute("select count(*) from files").fetchone()[0] > 0
        if used:
            db.execute("pragma secure_delete=on")
        db.execute("begin immediate")
        make(db, bool(chat_path))
        db.execute("commit")
        if used:
            # The dropped pages held the old text. Vectors are kept: the sections come back with
            # the same hashes on the next sweep, and make() never drops them (staff r3, 1); a
            # vector no line uses any more holds no text and goes at the next erase.
            finish_erase(db, path)
    return db


# argoni-embed's own objects in the same file (engine/embed/argoni-embed.swift, annTables),
# exactly as it makes them: kept, never taken for planted ones (staff r3, 1).
EMBED_SQL = [
    "create table if not exists vkeys(key integer primary key autoincrement, hash text not null unique)",
    "create table if not exists vindex(name text primary key, value integer not null)",
    "CREATE TRIGGER vectors_key after insert on vectors begin insert into vkeys(hash)"
    " select new.hash where not exists (select 1 from vkeys where hash = new.hash); end",
]


def expected_schema(chat=False):
    """{(type, name): sql} that SCHEMA_SQL and argoni-embed make (and, for the chat's index,
    CHAT_SQL), read from a scratch database in memory."""
    m = sqlite3.connect(":memory:")
    try:
        for stmt in SCHEMA_SQL + EMBED_SQL + (CHAT_SQL if chat else []):
            m.execute(stmt)
        return {(t, n): s for t, n, s in m.execute("select type, name, sql from sqlite_master")}
    finally:
        m.close()


def foreign(db, chat=False):
    """[(type, name)] in the index that this script did not make, or made differently."""
    want = expected_schema(chat)
    out = []
    for t, n, s in db.execute("select type, name, sql from sqlite_master"):
        if (t, n) in want and want[(t, n)] == s:
            continue
        # SQLite's own: only these names, and an automatic index has no text (security r3, S1).
        if n in ("sqlite_sequence", "sqlite_stat1", "sqlite_stat4") or (t == "index" and n.startswith("sqlite_autoindex_") and s is None):
            continue
        if t in ("table", "view", "trigger", "index"):
            out.append((t, n))
    return [(t, n) for t, n in out if t in ("view", "trigger")] + [(t, n) for t, n in out if t in ("index", "table")]


def make(db, chat=False):
    """Every table from nothing, inside the caller's transaction."""
    for name in ("chat_ids", "fts", "sections", "files", "meta"):  # never vectors: they are keyed by text
        db.execute("drop table if exists %s" % name)
    for stmt in SCHEMA_SQL + (CHAT_SQL if chat else []):
        db.execute(stmt)
    set_meta(db, schema=SCHEMA)


def set_meta(db, **kv):
    db.executemany("insert or replace into meta values (?, ?)", [(k, str(v)) for k, v in kv.items()])


def get_meta(db):
    return dict(db.execute("select key, value from meta"))


def sweep(db, st, budget=None, fresh=False):
    """Read new and changed files, drop gone ones. With a budget (seconds), stop
    reading when it is spent; what is left is read by the next sweep."""
    t0 = time.time()
    db.execute("begin immediate")
    used = fresh and db.execute("select count(*) from files").fetchone()[0] > 0
    if used:
        db.execute("pragma secure_delete=on")
    if fresh:
        make(db)
    meta = get_meta(db)
    if meta.get("sensitive") not in (None, str(st["sensitive"])):
        # health, money or family switched: every note is weighed again
        db.execute("update files set size = -1")
    known = {p: (s, m) for p, s, m in db.execute("select path, size, mtime from files")}
    seen, todo = set(), []
    for label, top, path in walk(st):
        seen.add(path)
        try:
            s = os.stat(path)
        except OSError:
            continue
        if known.get(path) != (s.st_size, s.st_mtime):
            todo.append((label, top, path))
    gone = [p for p in known if p not in seen]
    for p in gone:
        drop(db, p)
    read = sections = 0
    tried = 0
    for label, top, path in todo:
        if budget is not None and time.time() - t0 > budget:
            break
        n = read_file(db, label, top, path, st)
        tried += 1
        read += n is not None  # a file not kept (too big, or kept out) is not counted as read
        sections += n or 0
    left = len(todo) - tried
    stamp = datetime.datetime.now().astimezone().isoformat(timespec="seconds")
    set_meta(db, sensitive=st["sensitive"], swept_at=stamp, left=left)
    if fresh:
        set_meta(db, built_at=stamp)
    db.execute("commit")
    if fresh or read > 500:
        db.execute("insert into fts(fts) values ('optimize')")
    if used or meta.get("erase"):
        erase_rows(db, DB, force=True)   # a rebuild dropped a used index, or an earlier clean-up was not whole
    return {"read": read, "gone": len(gone), "sections": sections, "left": left, "seconds": time.time() - t0}


def update_files(db, st, paths):
    """Read the named files now. A relative FILE is inside data/ (else the current folder)."""
    out = []
    db.execute("begin immediate")
    for p in paths:
        path = p if os.path.isabs(p) else os.path.join(LIFE, p)
        if not os.path.isabs(p) and not os.path.exists(path) and os.path.exists(p):
            path = os.path.abspath(p)
        if not os.path.exists(path):
            drop(db, path)
            drop(db, os.path.realpath(path))
            out.append("%s: gone, dropped" % p)
            continue
        real = os.path.realpath(path)
        where = place(path, st)
        if not where:
            drop(db, real)
            out.append("%s: not something Argoni may read, not indexed" % p)
            continue
        n = read_file(db, where[0], where[1], real, st)
        out.append("%s: %s" % (p, "not kept" if n is None else "%d section(s)" % n))
    db.execute("commit")
    return out


# ---------------------------------------------------------------- the chat (brief 0120)

CHAT_DAY = re.compile(r"(\d{4}-\d{2}-\d{2})\.jsonl")
CHAT_ID = re.compile(r"m-[0-9]{8}-[0-9]{6}-[0-9a-f]{6}")
CHAT_MAX_BYTES = 16 * 1024 * 1024   # one day's file, and the lists beside it
# A chat line joins an answer only when it is a real match. Measured on 300 made-up chat lines
# mixed with the made-up two-year folder (14 questions about a line in other words, 12 about
# something else): the first scored 0.799 to 0.900, the second 0.738 to 0.850. Just above the
# highest of the second kind keeps every unrelated question out, and keeps 6 of the 14 by meaning
# alone (the others are found by words, when two of the question's words are in the line).
CHAT_MEANING_MIN = 0.855
CHAT_MAX_RESULTS = 2                # chat lines in one answer, in find and in context
CHAT_ARG = [None]                   # --chat DIR, set by main
CHAT_PARALLEL = [True]              # the two meaning searches start together (the bench turns it off)
_CHAT = {}                          # what was found about the record this run: folder, why not, the open index


def chat_dir_fd(path):
    """A handle on the record folder, or OSError with a plain reason: a real folder (not a
    link) that is this user's and that nobody else can open (0700 is how the runner makes it)."""
    try:
        fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
    except OSError as e:
        raise OSError(e.errno, "%s is not a real folder (%s)" % (os.path.basename(path.rstrip("/")) or path, e.strerror))
    s = os.fstat(fd)
    if s.st_uid != os.getuid() or s.st_mode & 0o077:
        os.close(fd)
        raise OSError(0, "the record folder is not private to this user (it must be 0700)")
    return fd


def chat_read(dfd, name, cap=CHAT_MAX_BYTES):
    """(bytes, stat of the same open file) of a plain file in the record folder with one name,
    or None (missing, a link, a pipe, another name elsewhere, too big): the runner's care. The
    stat is taken from the descriptor before the read, so a line appended in between makes the
    next sweep read the file again instead of being lost."""
    try:
        fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=dfd)
    except OSError:
        return None
    with os.fdopen(fd, "rb") as fh:
        s = os.fstat(fh.fileno())
        if not stat.S_ISREG(s.st_mode) or s.st_nlink != 1 or s.st_size > cap:
            return None
        return fh.read(cap + 1)[:cap], s


def chat_cache_root():
    return os.environ.get("ARGONI_CHAT_CACHE") or os.path.join(os.path.expanduser("~"), "Library", "Caches", "Argoni")


def chat_trusted_dir(path, create):
    """True when path is a real folder of this user that nobody else may write, made 0700 when
    missing (the way argoni-embed checks its own cache folder)."""
    try:
        s = os.lstat(path)
    except FileNotFoundError:
        if not create:
            return False
        try:
            os.mkdir(path, 0o700)
            s = os.lstat(path)
        except OSError:
            return False
    except OSError:
        return False
    return stat.S_ISDIR(s.st_mode) and s.st_uid == os.getuid() and not s.st_mode & 0o022


def chat_folder():
    """(the record folder's path, None) or (None, one plain reason). Unset, unreadable or
    not private: not reached, never an error."""
    if "folder" in _CHAT:
        return _CHAT["folder"]
    path = CHAT_ARG[0] or os.environ.get("ARGONI_CHAT") or ""
    if not path:
        out = (None, "no record folder given")
    else:
        path = os.path.abspath(os.path.expanduser(path))
        _CHAT["path"] = path
        try:
            os.close(chat_dir_fd(path))
            out = (path, None)
        except OSError as e:
            out = (None, e.strerror or str(e))
            # Gone (not unplugged: its folder is still there): the cache of it is an orphan.
            _CHAT["gone"] = e.errno == errno.ENOENT and os.path.isdir(os.path.dirname(path))
    _CHAT["folder"] = out
    return out


def chat_index_file(create=False):
    """The chat index's path when the record is reached and its cache folders are trusted
    (and, unless create, the file already exists), else None. Never in data/."""
    folder, _ = chat_folder()
    if not folder:
        return None
    key = os.path.basename(folder.rstrip("/"))
    root = chat_cache_root()
    real = os.path.realpath(root)
    life = os.path.realpath(LIFE)
    if real == life or real.startswith(life + os.sep):
        # A cache inside data/ would hand the conversation to any run that opens the file.
        _CHAT["folder"] = (None, "the cache folder is inside your life folder")
        return None
    if not chat_trusted_dir(os.path.dirname(root.rstrip("/")), False):
        _CHAT["folder"] = (None, "the cache folder is not safe to use")
        return None
    path = root
    for part in ("", "chat", key):
        path = os.path.join(path, part) if part else path
        if not chat_trusted_dir(path, create):
            if create:
                _CHAT["folder"] = (None, "the cache folder is not safe to use")
            return None
        if create:
            try:
                os.chmod(path, 0o700)
            except OSError:
                pass
    index = os.path.join(path, "memory.sqlite")
    return index if create or os.path.exists(index) else None


def chat_remove_index(path):
    """The chat's index files removed, no copy kept (it holds the conversation), its graph with
    them, and the cached connection forgotten. Only this index; never the notes'."""
    db = _CHAT.pop("db", None)
    if db is not None:
        try:
            db.close()
        except sqlite3.Error:
            pass
    drop_graph(path)
    _CHAT["remade"] = True   # forget reports it: the lines it was asked for are gone with the index
    for f in (path, path + "-wal", path + "-shm", path + "-journal"):
        if os.path.lexists(f):
            try:
                os.remove(f)
            except OSError:
                pass


def chat_connect(create=False):
    """The chat's index (made when create), or None. It is a cache of the record: one that is
    broken, or holds anything this script did not make, is removed and made again."""
    path = chat_index_file(create)
    if not path:
        return None
    if "db" not in _CHAT:
        os.umask(0o077)   # the index holds the conversation: its files are the user's alone
        try:
            _CHAT["db"] = connect(path)
        except sqlite3.DatabaseError as e:
            if not any(w in str(e).lower() for w in BROKEN):
                raise
            chat_remove_index(path)
            _CHAT["db"] = connect(path)
    return _CHAT["db"]


def chat_safe(fn, *args, default=None, **kw):
    """Run a chat step. A broken chat index is removed and the step runs once more on a fresh
    one; any other database trouble means "no chat this time". The notes' index is never
    touched for a chat fault."""
    for attempt in (0, 1):
        try:
            return fn(*args, **kw)
        except sqlite3.DatabaseError as e:
            path = chat_index_file()
            if path and any(w in str(e).lower() for w in BROKEN) and attempt == 0:
                chat_remove_index(path)
                continue
            return default
    return default


def chat_cleanup_orphan():
    """The record folder is gone (and the folder it was in is not): its cache is removed, so
    the conversation is not kept for ever. Run by update and forget, given the record path."""
    folder, _ = chat_folder()
    if folder or not _CHAT.get("gone"):
        return False
    key = os.path.basename(_CHAT["path"].rstrip("/"))
    root = chat_cache_root()
    d = os.path.join(root, "chat", key)
    if not all(chat_trusted_dir(x, False) for x in (root, os.path.join(root, "chat"), d)):
        return False
    graph = os.path.join(d, "memory.sqlite")
    if os.path.exists(graph):
        drop_graph(graph)
    for n in os.listdir(d):
        p = os.path.join(d, n)
        if n.startswith(("memory.sqlite", "memory-embed")) and stat.S_ISREG(os.lstat(p).st_mode):
            os.unlink(p)
    try:
        os.rmdir(d)
    except OSError:
        return False
    return True


def chat_label(who):
    return "you" if who == "user" else "Argoni"


def chat_time(t):
    m = re.match(r"\d{4}-\d{2}-\d{2}[T ](\d\d:\d\d)", str(t or ""))
    return m.group(1) if m else ""


def chat_read_day(db, dfd, name, real):
    """Index one day file (its old rows dropped first). Returns (entries kept, unverified,
    whether a line that was indexed before is gone), or None when the file may not be kept.
    Only an entry whose exact line's sha256 stands beside its id in the day's .hashes is kept
    (design: memory cites a line as the user's words only if its hash is recorded), and never
    a forgotten id."""
    QUIET[0] = time.time()
    path = os.path.join(real, name)
    before = {r[0] for r in db.execute("select c.id from chat_ids c join sections s on s.id = c.section"
                                       " where s.file in (select id from files where path = ?)", (path,))}
    hashes = chat_hashes(db, path)
    drop(db, path)
    day = name[:-len(".jsonl")]
    db.execute("delete from meta where key = ?", ("unv:" + day,))
    got = chat_read(dfd, name)
    if got is None:
        # Refused (too big, a second name, a link): its rows were dropped above, so the sweep
        # must erase them too. A mark row (kind "refused", no sections) stands for the day; its
        # size is -1, so the next sweep looks again and a day whose cause is gone (a second name
        # removed, which changes neither size nor time) comes back (staff r3, 2).
        try:
            s = os.stat(name, dir_fd=dfd, follow_symlinks=False)
        except OSError:
            s = None
        if s is not None:
            try:
                side = os.stat(day + ".hashes", dir_fd=dfd, follow_symlinks=False)
            except OSError:
                side = None
            db.execute("insert into files(path, source, rel, show, kind, subject, updated, title, size, mtime, sections)"
                       " values (?,?,?,?,?,?,?,?,?,?,?)",
                       (path, "chat", "chat/" + day, "chat, " + day, "refused", None, day, "chat, " + day,
                        -1, max(s.st_mtime, side.st_mtime if side else 0), 0))
        return 0, 0, bool(before)
    raw, s = got
    side = {}
    side_raw = chat_read(dfd, day + ".hashes")
    for l in (side_raw[0] if side_raw else b"").decode("utf-8", "replace").splitlines():
        a, _, b = l.partition(" ")
        side[a] = b.strip()
    forgotten = (chat_read(dfd, "forgotten.ids") or (b"", None))[0]
    forgotten = set(forgotten.decode("utf-8", "replace").split())
    rows, unverified = [], 0
    for n, line in enumerate(raw.decode("utf-8", "replace").split("\n"), 1):
        if not line.strip():
            continue
        try:
            e = json.loads(line)
        except ValueError:
            continue
        if not isinstance(e, dict) or not isinstance(e.get("id"), str) or not CHAT_ID.fullmatch(e["id"]) \
                or not isinstance(e.get("text"), str) or not e["text"].strip():
            continue
        if e["id"] in forgotten:
            continue
        if side.get(e["id"]) != hashlib.sha256(line.encode("utf-8", "replace")).hexdigest():
            unverified += 1
            continue
        rows.append((n, e))
    # The sidecar is written after the day file, so the later of the two times is the change.
    mtime = max(s.st_mtime, side_raw[1].st_mtime if side_raw else 0)
    cur = db.execute("insert into files(path, source, rel, show, kind, subject, updated, title, size, mtime, sections)"
                     " values (?,?,?,?,?,?,?,?,?,?,?)",
                     (path, "chat", "chat/" + day, "chat, " + day, "chat", None, day, "chat, " + day,
                      s.st_size, mtime, len(rows)))
    fid = cur.lastrowid
    for n, e in rows:
        who = e.get("who") if isinstance(e.get("who"), str) else "argoni"
        # No title, no path words and no "You said" in the index: words that every line holds
        # would match every question. Who and when are in chat_ids; the heading keeps the time.
        heading = "%s, %s" % (chat_time(e.get("t")), who)
        text = e["text"]
        c = db.execute("insert into sections(file, title, heading, line, text, pathw, extra, hash)"
                       " values (?,?,?,?,?,?,?,?)",
                       (fid, "", heading, n, text, "",
                        " ".join(ar_stem(w) for w in ar_words(" ".join((heading, text)))),
                        hashlib.sha1(("\n%s" % text).encode("utf-8")).hexdigest()[:20]))
        reply = e.get("reply_to") if isinstance(e.get("reply_to"), str) else None
        db.execute("insert into chat_ids(section, id, who, t, reply_to) values (?,?,?,?,?)",
                   (c.lastrowid, e["id"], who, str(e.get("t") or ""), reply))
    set_meta(db, **{"unv:" + day: unverified})
    after = {e["id"] for _, e in rows}
    after_hashes = set(chat_hashes(db, path))
    left = bool((before - after) or (set(hashes) - after_hashes))
    return len(rows), unverified, left


def embed_cache_dir():
    return os.environ.get("ARGONI_EMBED_CACHE") or os.path.join(os.path.expanduser("~"), "Library", "Caches", "Argoni", "vectors")


def graph_stem(path):
    """The 16 hex characters every graph file of this index file starts with (argoni-embed's
    graphStem: FNV-1a, 64 bits, over the real path)."""
    h = 0xcbf29ce484222325
    for b in os.path.realpath(path).encode("utf-8"):
        h = ((h ^ b) * 0x100000001b3) & 0xFFFFFFFFFFFFFFFF
    return "%016x" % h


def drop_graph(path):
    """Unlink this index file's graph files (plain files only) in argoni-embed's cache."""
    d = embed_cache_dir()
    stem = graph_stem(path)
    try:
        if not stat.S_ISDIR(os.lstat(d).st_mode):
            return
        names = os.listdir(d)
    except OSError:
        return
    for n in names:
        if n == stem + ".path" or re.fullmatch(r"%s-\d+\.usearch(\.[0-9a-f]{6})?" % stem, n):
            p = os.path.join(d, n)
            try:
                if stat.S_ISREG(os.lstat(p).st_mode):
                    os.unlink(p)
            except OSError:
                pass


def embed_lock(index, wait=None):
    """(fd, busy): argoni-embed's own lock on this index file (the same name, taken the same
    way), waited for at most `wait` seconds (ARGONI_EMBED_LOCK_WAIT, else 10). busy: the wait
    ran out, and the caller goes on without it."""
    wait = float(os.environ.get("ARGONI_EMBED_LOCK_WAIT") or 10) if wait is None else wait
    try:
        fd = os.open(os.path.realpath(index) + "-embed.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o644)
    except OSError:
        return None, False
    end = time.time() + wait
    while True:
        try:
            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
            return fd, False
        except OSError:
            if time.time() >= end:
                os.close(fd)
                return None, True
            time.sleep(0.1)


def scrub_orphans(db):
    """Inside the caller's transaction: every vector no section uses goes, in one statement. If
    the graph may hold one (an orphan has a key in vkeys), the graph's row goes too, and then
    the orphans' keys (the next index run builds the graph again; search is exact meanwhile).
    Returns (vectors removed, 'dropped' | 'none')."""
    have = {r[0] for r in db.execute("select name from sqlite_master where type = 'table'")}
    if "vectors" not in have:
        return 0, "none"
    keys = {"vindex", "vkeys"} <= have
    graph = "none"
    if keys and db.execute("select 1 from vindex where name = 'graph'").fetchone() and \
            db.execute("select 1 from vkeys where hash not in (select hash from sections) limit 1").fetchone():
        db.execute("delete from vindex where name = 'graph'")
        graph = "dropped"
    n = db.execute("delete from vectors where hash not in (select hash from sections)").rowcount
    if keys and not db.execute("select 1 from vindex where name = 'graph'").fetchone():
        db.execute("delete from vkeys where hash not in (select hash from sections)")
    return n, graph


def checkpoint_truncate(db):
    """True when the log was truncated. A reader holding the file makes the result "busy": tried
    again for about two seconds, then false."""
    # The default wait for a busy file is 15 s a try; here a try waits a tenth of a second.
    db.execute("pragma busy_timeout=100")
    try:
        for _ in range(10):
            row = db.execute("pragma wal_checkpoint(TRUNCATE)").fetchone()
            if row and row[0] == 0:
                return True
            time.sleep(0.2)
        return False
    finally:
        db.execute("pragma busy_timeout=15000")


def finish_erase(db, path, graph="none"):
    """After a commit that deleted rows: no copy of them left in the file. The deleted pages are
    zeroed (secure_delete); the FTS segments are merged (so the words leave the index itself,
    not only its content table); the log is truncated; the file is rewritten, its temporary
    copy kept in memory, not in free disk blocks. Returns whether all of it ran. If not, a
    "needs erase" mark stays in meta and the next sweep finishes it."""
    db.execute("pragma secure_delete=on")
    db.execute("pragma temp_store=memory")
    if graph == "dropped":
        drop_graph(path)
    ok = False
    try:
        db.execute("insert into fts(fts) values ('optimize')")
        ok = checkpoint_truncate(db)
        if ok:
            db.execute("pragma busy_timeout=2000")
            try:
                db.execute("vacuum")
            finally:
                db.execute("pragma busy_timeout=15000")
            ok = checkpoint_truncate(db)
    except sqlite3.OperationalError:
        ok = False
    try:
        if ok:
            db.execute("delete from meta where key = 'erase'")
            db.execute("insert or replace into meta values ('erases', coalesce((select cast(value as integer) from meta"
                       " where key = 'erases'), 0) + 1)")
        else:
            set_meta(db, erase=1)
    except sqlite3.OperationalError:
        pass
    return ok


def erase_rows(db, path, delete=None, force=False):
    """The one way rows leave an index for good. Under argoni-embed's lock (waited for, bounded),
    in one transaction: delete(db) if given, then every orphan vector; then finish_erase. Runs
    when something was deleted, or a "needs erase" mark is set (or force). Returns {"vectors",
    "graph", "erased", "embed_busy", "ran"}."""
    out = {"vectors": 0, "graph": "none", "erased": True, "embed_busy": False, "ran": False}
    pending = db.execute("select 1 from meta where key = 'erase'").fetchone() is not None
    if delete is None and not pending and not force:
        return out   # nothing left the index: nothing to clean, so no lock and no rewrite
    fd, out["embed_busy"] = embed_lock(path)
    try:
        db.execute("begin immediate")
        try:
            gone = delete(db) if delete is not None else None
            # An earlier run could not get argoni-embed's lock: a graph made since may hold a vector
            # that left, even though no orphan has a key any more. With the lock, it goes now.
            stale = fd is not None and db.execute("select 1 from meta where key = 'dropgraph'").fetchone() is not None
            if stale and db.execute("select 1 from sqlite_master where name = 'vindex'").fetchone():
                db.execute("delete from vindex where name = 'graph'")
            out["vectors"], out["graph"] = scrub_orphans(db)
            if stale:
                out["graph"] = "dropped"
            if gone == 0 and not pending and not force and not out["vectors"] and not stale:
                db.execute("commit")   # the delete matched nothing: no rewrite of the file
                return out
            db.execute("commit")
        except BaseException:
            try:
                db.execute("rollback")
            except sqlite3.Error:
                pass
            raise
        out["erased"] = finish_erase(db, path, out["graph"])
        out["ran"] = True
        try:
            if out["embed_busy"]:
                # After finish_erase, whose own success clears the "erase" mark.
                set_meta(db, erase=1, dropgraph=1)
            elif stale and out["erased"]:
                db.execute("delete from meta where key = 'dropgraph'")
        except sqlite3.OperationalError:
            pass
    finally:
        if fd is not None:
            os.close(fd)
    return out


def chat_hashes(db, path):
    return [r[0] for r in db.execute("select hash from sections where file in (select id from files where path = ?)",
                                     (path,))]


def chat_sweep(budget=None, fresh=False):
    """Read new and changed day files, drop gone ones. Returns {"days", "gone", "entries",
    "unverified", "seconds", "erased"}, or None when the chat is not reached. When a line
    that was indexed has left (a day gone, a line forgotten or changed), no copy of it is left
    in the index file afterwards."""
    t0 = time.time()
    folder, _ = chat_folder()
    if not folder:
        return None
    fresh = fresh or not chat_index_file()
    db = chat_connect(create=True)
    if db is None:
        return None
    index = chat_index_file()
    try:
        dfd = chat_dir_fd(folder)
    except OSError:
        return None
    try:
        real = os.path.realpath(folder)
        db.execute("begin immediate")
        had = db.execute("select count(*) from files").fetchone()[0] > 0
        if fresh:
            db.execute("pragma secure_delete=on")
            make(db, True)
        meta = get_meta(db)
        fs = chat_read(dfd, "forgotten.ids")
        stamp = "%d:%s" % (fs[1].st_size, fs[1].st_mtime) if fs else "none"
        every = meta.get("forgotten") != stamp   # a newly forgotten id: every day is read again
        known = {p: (s, m) for p, s, m in db.execute("select path, size, mtime from files")}
        days = []
        for name in sorted(os.listdir(dfd)):
            if not CHAT_DAY.fullmatch(name):
                continue
            try:
                s = os.stat(name, dir_fd=dfd, follow_symlinks=False)
            except OSError:
                continue
            try:
                side = os.stat(name[:-len(".jsonl")] + ".hashes", dir_fd=dfd, follow_symlinks=False)
            except OSError:
                side = None
            if stat.S_ISREG(s.st_mode):
                days.append((name, s.st_size, max(s.st_mtime, side.st_mtime if side else 0)))
        seen = {os.path.join(real, n) for n, _, _ in days}
        gone = [p for p in known if p not in seen]
        left = fresh and had
        for p in gone:
            left = left or bool(chat_hashes(db, p))
            drop(db, p)
            db.execute("delete from meta where key = ?", ("unv:" + os.path.basename(p)[:-len(".jsonl")],))
        read = entries = 0
        stopped = False
        for name, size, mtime in days:
            path = os.path.join(real, name)
            if not every and known.get(path) == (size, mtime):
                continue
            if budget is not None and time.time() - t0 > budget:
                stopped = True
                break
            r = chat_read_day(db, dfd, name, real)
            read += 1
            if r:
                entries += r[0]
                left = left or r[2]
        # The stamp is saved only when every day was read again: a stop part way keeps the rest due.
        set_meta(db, swept_at=datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
                 **({} if stopped else {"forgotten": stamp}))
        db.execute("commit")
        # A sweep that dropped a line also leaves no copy, even when no `forget` was called.
        er = erase_rows(db, index, delete=(lambda d: None) if left else None)
        unv = sum(int(v) for k, v in db.execute("select key, value from meta where key like 'unv:%'"))
        return {"days": read, "gone": len(gone), "entries": entries, "unverified": unv,
                "seconds": time.time() - t0, "erased": er["erased"]}
    finally:
        os.close(dfd)


def chat_status():
    """{"reached": bool, ...}: what status and find say about the chat."""
    folder, why = chat_folder()
    if not folder:
        return {"reached": False, "why": why}
    db = chat_connect(create=False)
    if db is None or not get_meta(db).get("swept_at"):   # none, or made again and not yet read
        return {"reached": True, "indexed": False}
    days, entries = db.execute("select count(*), coalesce(sum(sections), 0) from files where kind = 'chat'").fetchone()   # not the marks of refused days
    unv = sum(int(v) for k, v in db.execute("select key, value from meta where key like 'unv:%'"))
    return {"reached": True, "indexed": True, "days": days, "entries": entries, "unverified": unv,
            "vectors": db.execute("select count(*) from vectors").fetchone()[0]}


def chat_hit(db, sid):
    """One chat result for a section id, or None. The key "source" is set here and nowhere
    else: a note whose front matter says `kind: chat` is still a note."""
    r = db.execute("select s.id, s.line, s.heading, f.path, f.updated, c.id, c.who, c.t"
                   " from sections s join files f on f.id = s.file join chat_ids c on c.section = s.id"
                   " where s.id = ?", (sid,)).fetchone()
    if not r:
        return None
    sid, line, heading, path, day, eid, who, t = r
    out = {"file": path, "show": "chat, %s %s" % (day, chat_time(t)), "line": line, "heading": heading,
           "title": "", "kind": "chat", "source": "chat", "section": sid, "updated": day, "lines": [],
           "who": who, "id": eid, "day": day, "t": t}
    if who == "user":
        # Only lines whose hash the runner recorded are ever indexed (chat_read_day), so every
        # user line here is verified: this is what lets a caller quote it as the user's words.
        out["words"] = "user"
    return out


def is_chat(r):
    return r.get("source") == "chat"


def chat_prepare(question, st, meaning):
    """Before the notes are searched: make sure the chat index exists, and start its meaning
    search so it runs beside the notes' one. Returns {"reached", "why", "proc"}."""
    folder, why = chat_folder()
    if not folder:
        return {"reached": False, "why": why, "proc": None}

    def go():
        if not chat_status().get("indexed"):
            chat_sweep(fresh=True)   # the first question builds it, as it does for the notes
        db = chat_connect(create=False)
        if db is None or not meaning:
            return None
        if meaning_state(db, st)[0]:
            return meaning_start(question, WIDE, chat_index_file())
        return None
    proc = chat_safe(go)
    if not chat_folder()[0]:
        return {"reached": False, "why": chat_folder()[1], "proc": None}
    return {"reached": True, "why": None, "proc": proc}


def chat_search(question, prep):
    """[result] best first, at most CHAT_MAX_RESULTS, from the chat index: word search and meaning
    search joined by reciprocal rank fusion, line by line (a line is the unit here, not a day).
    A line counts only as a real match: at least two of the question's words (one, if the
    question has one), or a meaning score of CHAT_MEANING_MIN or more."""
    db = chat_connect(create=False)
    if db is None:
        return []
    match, marks = terms(question)
    words = []
    if match:
        need = min(2, len(marks))
        for sid, text in db.execute("select f.rowid, s.text from fts f join sections s on s.id = f.rowid"
                                    " where fts match ? order by bm25(fts, 2.0, 4.0, 2.0, 1.0, 1.0) limit ?",
                                    (match, WIDE)):
            low = text.lower()
            ar = {ar_stem(w) for w in ar_words(text)}
            if sum(1 for k, w in marks if (w in ar if k == "ar" else w in low)) >= need:
                words.append(sid)
    near = [(sid, cos) for sid, cos in (meaning_finish(prep["proc"]) or []) if cos >= CHAT_MEANING_MIN]
    score = {}
    for rank, sid in enumerate(words):
        score[sid] = score.get(sid, 0) + 1.0 / (RRF_K + rank + 1)
    for rank, (sid, _) in enumerate(near):
        score[sid] = score.get(sid, 0) + MEANING_WEIGHT / (RRF_K + rank + 1)
    out = []
    for sid in sorted(score, key=lambda i: -score[i])[:CHAT_MAX_RESULTS]:
        r = chat_hit(db, sid)
        if r:
            text = db.execute("select text from sections where id = ?", (sid,)).fetchone()[0]
            r["line"], r["lines"] = best_lines(text, r["line"], marks)
            r["score"] = round(score[sid] * 1000, 2)
            out.append(r)
    return out


def join_chat(notes, chat):
    """The notes' ranking and the chat's, one list: reciprocal rank fusion on rank alone, so a
    chat line can rank among notes. On a tie the note goes first."""
    key = {}
    for rank, r in enumerate(notes):
        key[id(r)] = (-1.0 / (RRF_K + rank + 1), 0, rank)
    for rank, r in enumerate(chat):
        key[id(r)] = (-1.0 / (RRF_K + rank + 1), 1, rank)
    return sorted(notes + chat, key=lambda r: key[id(r)])


def chat_piece(r):
    """(first line, last line, text) for a chat result: the entry, with the line it answers or
    the reply to it when that is short."""
    db = chat_connect(create=False)
    if db is None:
        return None
    me = db.execute("select s.text, c.id, c.who, c.reply_to from sections s join chat_ids c on c.section = s.id"
                    " where s.id = ?", (r["section"],)).fetchone()
    if not me:
        return None
    text, eid, who, reply_to = me

    def near(where, arg):
        row = db.execute("select s.text, c.who from sections s join chat_ids c on c.section = s.id where " + where
                         + " order by s.id limit 1", (arg,)).fetchone()
        return row if row and len(row[0]) <= 400 else None

    asked = near("c.id = ?", reply_to) if reply_to else None
    answered = near("c.reply_to = ?", eid)
    parts = []
    if asked:
        parts.append("%s: %s" % (chat_label(asked[1]), asked[0]))
    parts.append(("%s: %s" % (chat_label(who), text)) if asked or answered else text)
    if answered:
        parts.append("%s: %s" % (chat_label(answered[1]), answered[0]))
    out = "\n".join(parts)[:PIECE_CHARS]
    return r["line"], r["line"] + out.count("\n"), out


def forget(ids, files):
    """Erase these chat lines from memory (and these files' rows from the notes' index) so that
    no copy is left in the index files. Returns the JSON line's dict; "embed_busy" (the lock
    of argoni-embed was not free in time) and "erased": false (a reader held the file) mean the
    lines are deleted but the clean-up is not whole: the caller tries again.

    The command's exit codes: 0 done; 1 the chat was not reached (nothing of it forgotten);
    2 wrong arguments; 3 deleted but the clean-up is not whole, try again; 4 an unexpected
    failure (the lines may still be there). A broken chat index is removed, not a failure."""
    res = {"forgot": 0, "vectors": 0, "graph": "none", "files": [], "erased": True, "embed_busy": False}
    if ids:
        chat_cleanup_orphan()
        folder, why = chat_folder()
        res["chat"] = "searched" if folder else "not reached: %s" % why
        if folder:
            _CHAT.pop("remade", None)

            def chat_half():
                db = chat_connect(create=False)
                if db is None:
                    return None
                n = [0]

                def delete(d):
                    # The lookup is made here, under the write lock: a sweep that ran meanwhile
                    # may have given the lines new section numbers (no stale number is used).
                    for i in ids:
                        n[0] += d.execute("delete from sections where id in (select section from chat_ids where id = ?)",
                                          (i,)).rowcount   # the trigger takes it out of the words index
                    d.execute("update files set sections = (select count(*) from sections where file = files.id)")
                    return n[0]
                er = erase_rows(db, chat_index_file(), delete=delete)
                return n[0], er
            got = chat_safe(chat_half, default="trouble")
            if got == "trouble":
                res["chat"] = "not reached: the chat index could not be used"
            elif got is not None:
                n, er = got
                res["forgot"] = n   # rows really deleted, not rows looked up
                res["vectors"] += er["vectors"]
                res["graph"] = er["graph"]
                res["erased"] = res["erased"] and er["erased"]
                res["embed_busy"] = res["embed_busy"] or er["embed_busy"]
            if _CHAT.pop("remade", None) and res["chat"] == "searched":
                # A broken or foreign index was removed: nothing of the lines is left in it, and
                # the record still lists the forgotten ids, so the next sweep leaves them out.
                res["chat"] = "index made again"
    if files and os.path.exists(DB):
        ndb = connect()
        ndb.execute("pragma secure_delete=on")
        done = []
        for f in files:
            p = f if os.path.isabs(f) else os.path.join(LIFE, f)
            if not os.path.isabs(f) and not os.path.exists(p) and os.path.exists(f):
                p = os.path.abspath(f)   # as `update FILE` reads a relative name
            for q in dict.fromkeys([p, os.path.realpath(p)]):
                if ndb.execute("select 1 from files where path = ?", (q,)).fetchone():
                    done.append(q)

        def delete(d):
            for q in done:
                drop(d, q)
        er = erase_rows(ndb, DB, delete=delete if done else None)
        res["files"] = done
        res["vectors"] += er["vectors"]
        if er["graph"] == "dropped":
            res["graph"] = "dropped"
        res["erased"] = res["erased"] and er["erased"]
        res["embed_busy"] = res["embed_busy"] or er["embed_busy"]
    return res


# ---------------------------------------------------------------- find

NUMBERED = [
    (r"\b(?:decision|قرار|القرار)\s*#?(\d{1,4})\b", "decision", ("decisions/{n}-%",), "docs/decisions/{n}-%"),
    (r"\b(?:brief|موجز|الموجز)\s*#?(\d{1,4})\b", "brief", ("briefs/{n}-%",), "docs/briefs/{n}-%"),
    (r"\b(?:report|تقرير|التقرير)\s*#?(\d{1,4})\b", "report", ("reports/{n}-%",), "docs/reports/{n}-%"),
    (r"\b(?:idea|فكرة|الفكرة|فكره|الفكره)\s*#?(\d{1,4})\b", "idea", ("ideas/{n}-%/idea.md", "ideas/{n}-%"), None),
]
PR = re.compile(r"\b(?:PR|pull request)\s*#?(\d{1,6})\b|/pull/(\d{1,6})\b", re.I)
WEEKDAYS = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]
AR_WEEKDAYS = {"الاثنين": 0, "الإثنين": 0, "الثلاثاء": 1, "الأربعاء": 2, "الاربعاء": 2, "الخميس": 3,
               "الجمعة": 4, "السبت": 5, "الأحد": 6, "الاحد": 6}
DECIDE = re.compile(r"\b(decide|decided|decision|decisions|chose|choose|chosen|choice|picked)\b|قرر|اختر|اخترت", re.I)
KIND_WORDS = {"decision": r"\bdecisions?\b|قرار", "idea": r"\bideas?\b|فكر", "person": r"\b(person|people|who)\b|من هو|من هي",
              "brief": r"\bbriefs?\b", "report": r"\breports?\b|تقرير", "log": r"\b(log|logs|diary)\b"}


def days(question, today, week_ends):
    """(label, [dates]) for the time words in the question, or None."""
    q = question.lower()
    end_wd = WEEKDAYS.index(week_ends) if week_ends in WEEKDAYS else 3
    since_end = (today.weekday() - end_wd - 1) % 7   # days since the week began
    week_start = today - datetime.timedelta(days=since_end)
    span = lambda a, b: [a + datetime.timedelta(days=i) for i in range((b - a).days + 1)]  # noqa: E731
    m = re.search(r"\b(\d{4})-(\d{2})-(\d{2})\b", question)
    if m:
        try:
            d = datetime.date(*map(int, m.groups()))
            return m.group(0), [d]
        except ValueError:
            pass
    if re.search(r"\blast week\b|الأسبوع الماضي|الاسبوع الماضي", q):
        return "last week", span(week_start - datetime.timedelta(days=7), week_start - datetime.timedelta(days=1))
    if re.search(r"\bthis week\b|هذا الأسبوع|هذا الاسبوع|الأسبوع ده", q):
        return "this week", span(week_start, today)
    if re.search(r"\bthis month\b|هذا الشهر", q):
        return "this month", span(today.replace(day=1), today)
    if re.search(r"\byesterday\b|أمس|امس|البارحة", q):
        return "yesterday", [today - datetime.timedelta(days=1)]
    if re.search(r"\btoday\b|اليوم|النهارده", q):
        return "today", [today]
    m = re.search(r"\b(\d{1,2}) days ago\b", q)
    if m:
        return m.group(0), [today - datetime.timedelta(days=int(m.group(1)))]
    for i, name in enumerate(WEEKDAYS):
        if re.search(r"\b(on|last)\s+%s\b" % name, q):
            back = (today.weekday() - i) % 7 or 7
            return name.capitalize(), [today - datetime.timedelta(days=back)]
    for name, i in AR_WEEKDAYS.items():
        if name in question:
            back = (today.weekday() - i) % 7 or 7
            return name, [today - datetime.timedelta(days=back)]
    return None


def file_row(db, fid, why):
    r = db.execute("select path, show, kind, updated, title from files where id = ?", (fid,)).fetchone()
    return {"why": why, "file": r[0], "show": r[1], "kind": r[2], "updated": r[3], "title": r[4], "line": 1}


def lookup(db, question, cfg):
    """The files a name, a number or a time word points at. No ranking."""
    out, have = [], set()

    def add(fid, why, line=1, text=None):
        if fid in have and text is None:
            return
        have.add(fid)
        row = file_row(db, fid, why)
        row["line"] = line
        if text:
            row["lines"] = [text]
        out.append(row)

    for pat, kind, life_globs, repo_glob in NUMBERED:
        for m in re.finditer(pat, question, re.I):
            n = "%04d" % int(m.group(1))
            why = "%s %s" % (kind, n)
            for g in life_globs:
                for (fid,) in db.execute("select id from files where source = ? and rel like ? order by rel",
                                         (lifecfg.OWN, g.format(n=n))):
                    add(fid, why)
            if repo_glob:
                for (fid,) in db.execute("select id from files where source != ? and rel like ? order by source",
                                         (lifecfg.OWN, repo_glob.format(n=n))):
                    add(fid, why)
    for m in PR.finditer(question):
        n = m.group(1) or m.group(2)
        pat = re.compile(r"\b(?:PR|pull request)\s*#?%s\b|/pull/%s\b" % (n, n), re.I)
        rows = db.execute("select s.file, s.line, s.text from fts join sections s on s.id = fts.rowid"
                          " join files f on f.id = s.file where fts match ? and f.kind in ('ledger', 'log', 'report',"
                          " 'brief', 'project', 'note', 'inbox', 'standing') limit 200", ('"%s"' % n,)).fetchall()
        for fid, line, text in rows:
            for off, l in enumerate(text.split("\n")):
                if pat.search(l):
                    add(fid, "PR %s" % n, line + off, l.strip())
                    break
    words = set(re.findall(r"[\w\-]+", question.lower()))
    low = question.lower()
    for fid, rel, title in db.execute("select id, rel, title from files where source = ? and kind = 'person'",
                                      (lifecfg.OWN,)):
        name = (title or "").strip()
        slug = os.path.splitext(os.path.basename(rel))[0]
        first = name.split()[0].lower() if name else ""
        if (name and name.lower() in low) or slug in words or (len(first) > 2 and first in words):
            add(fid, "person: %s" % (name or slug))
            out[-1]["named"] = (name or slug).lower().split() + [slug]
    for fid, rel, title in db.execute("select id, rel, title from files where source = ? and kind = 'project'",
                                      (lifecfg.OWN,)):
        slug = os.path.splitext(os.path.basename(rel))[0].lower()
        if slug in words or slug.replace("-", " ") in low:
            add(fid, "project: %s" % slug)
            out[-1]["named"] = [slug]
    span = days(question, datetime.date.today(), str(cfg.get("week_ends") or "thursday").lower())
    if span:
        label, dates = span
        for d in dates:
            for (fid,) in db.execute("select id from files where source = ? and rel = ?",
                                     (lifecfg.OWN, "log/%s.md" % d.isoformat())):
                add(fid, "%s: the log of %s" % (label, d.isoformat()))
        if len(dates) > 1:
            y, w, _ = dates[-1].isocalendar()
            for (fid,) in db.execute("select id from files where source = ? and rel = ?",
                                     (lifecfg.OWN, "log/%d-W%02d-review.md" % (y, w))):
                add(fid, "%s: the weekly review" % label)
        out.extend(events(dates, label))
    return out, span


def events(dates, label):
    """What summary.json already holds for those days (commits, briefs, read-backs...)."""
    try:
        with open(os.path.join(LIFE, ".summary", "summary.json"), encoding="utf-8") as fh:
            tasks = json.load(fh).get("tasks") or []
    except (OSError, ValueError):
        return []
    want = {d.isoformat() for d in dates}
    out = []
    for t in tasks:
        for e in t.get("events") or []:
            if e.get("date") in want and e.get("kind") != "log":
                out.append({"why": "%s: %s" % (label, e.get("kind")), "file": None,
                            "show": t.get("id"), "kind": "event", "updated": e.get("date"),
                            "title": t.get("what"), "line": None, "lines": [e.get("line") or ""]})
    out.sort(key=lambda e: e["updated"] or "", reverse=True)
    return out[:20]


def terms(question):
    """(FTS5 query, the words to show matched lines by)."""
    parts, marks = [], []
    for w in re.findall(r"[\w']+", ar_norm(question)):
        if AR_WORD.fullmatch(w):
            if w in AR_STOP or len(w) < 2:
                continue
            stem = ar_stem(w)
            parts.append('extra : "%s"' % stem)
            marks.append(("ar", stem))
            continue
        lw = w.lower().strip("'")
        if lw in STOP or len(lw) < 2 and not lw.isdigit():
            continue
        parts.append('"%s"' % lw.replace('"', ""))
        marks.append(("en", lw[:max(4, len(lw) - 2)] if not lw.isdigit() else lw))
    return " OR ".join(parts), marks


def best_lines(text, first, marks, n=2):
    """The lines of a section holding the most question words, with their line numbers."""
    scored = []
    for off, line in enumerate(text.split("\n")):
        if not line.strip():
            continue
        low = line.lower()
        ar = {ar_stem(w) for w in ar_words(line)} if any(k == "ar" for k, _ in marks) else set()
        hits = sum(1 for k, w in marks if (w in ar if k == "ar" else w in low))
        if hits:
            scored.append((-hits, off, line.strip()))
    scored.sort()
    picked = sorted(scored[:n], key=lambda s: s[1])
    if not picked:
        line = next((l.strip() for l in text.split("\n") if l.strip()), "")
        return first, [line[:240]]
    return first + picked[0][1], [l[:240] for _, _, l in picked]


def word_search(db, question, k, cfg, lifted=()):
    match, marks = terms(question)
    if not match:
        return []
    rows = db.execute("select rowid, bm25(fts, 2.0, 4.0, 2.0, 1.0, 1.0) as r from fts where fts match ?"
                      " order by r limit ?", (match, WIDE)).fetchall()
    if not rows:
        return []
    ids = [r[0] for r in rows]
    score = {r[0]: -r[1] for r in rows}
    sec = {r[0]: r[1:] for r in db.execute(
        "select s.id, s.file, s.heading, s.line, s.text, f.path, f.show, f.kind, f.updated, f.title, f.source, f.rel,"
        " f.subject from sections s join files f on f.id = s.file where s.id in (%s)" % ",".join("?" * len(ids)), ids)}
    low = question.lower()
    words = set(re.findall(r"[\w\-]+", low))
    named = {s for s in words if s in names(db)}
    decide = bool(DECIDE.search(question))
    kinds = {kd for kd, pat in KIND_WORDS.items() if re.search(pat, question, re.I)}
    today = datetime.date.today()
    best = {}
    for sid in ids:
        if sid not in sec:
            continue
        fid, heading, line, text, path, show, kind, updated, title, source, rel, subject = sec[sid]
        s = score[sid]
        if named and (source in named or (subject or "").lower() in named
                      or any(n in rel.lower() for n in named)):
            s *= BOOST["named"]
        if decide and kind == "decision":
            s *= BOOST["decide"]
        if kind in kinds:
            s *= BOOST["kind"]
        if kind in ("now", "project", "standing"):
            s *= BOOST["page"]
        if path in lifted:
            s *= BOOST["lifted"]
        age = age_days(updated, path, today)
        if age is not None:
            s *= BOOST["new"] if age <= 14 else BOOST["recent"] if age <= 90 else 1.0
        if fid not in best or s > best[fid]["score"]:
            at, lines = best_lines(text, line, marks)
            best[fid] = {"file": path, "show": show, "line": at, "heading": heading, "kind": kind, "section": sid,
                         "updated": updated, "title": title, "lines": lines, "score": round(s, 3)}
    return sorted(best.values(), key=lambda r: -r["score"])[:k]


_NAMES = None


def names(db):
    """Project names a question may narrow by: project pages and repo labels."""
    global _NAMES
    if _NAMES is None:
        _NAMES = {os.path.splitext(os.path.basename(r))[0].lower() for (r,) in
                  db.execute("select rel from files where source = ? and kind = 'project'", (lifecfg.OWN,))}
        _NAMES |= {s.lower() for (s,) in db.execute("select distinct source from files where source != ?",
                                                     (lifecfg.OWN,))}
    return _NAMES


def age_days(updated, path, today):
    try:
        return (today - datetime.date.fromisoformat(str(updated)[:10])).days
    except ValueError:
        pass
    m = re.search(r"(\d{4}-\d{2}-\d{2})", os.path.basename(path))
    if m:
        try:
            return (today - datetime.date.fromisoformat(m.group(1))).days
        except ValueError:
            return None
    return None


def find(db, question, k=5, meaning=True, st=None, weight=None):
    t0 = time.time()
    cfg = lifecfg.config(LIFE)
    on, line = meaning_state(db, st or settings()) if meaning else (False, "words only")
    proc = meaning_start(question) if on else None
    prep = chat_prepare(question, st or settings(), meaning)   # starts the chat's search beside it
    looked, span = lookup(db, question, cfg)
    # A name alone ("Sam", "pantry", "how is pantry") opens its page first. In a longer
    # question the name is one word among several: its page is only lifted in the ranking.
    _, marks = terms(question)
    lifted = set()
    for r in [r for r in looked if r.get("named")]:
        rest = [w for _, w in marks if not any(w in n or n.startswith(w) for n in r["named"])]
        if len(rest) > 2:
            looked.remove(r)
            lifted.add(r["file"])
    for r in looked:
        r.pop("named", None)
    words = word_search(db, question, WIDE, cfg, lifted)
    near = meaning_finish(proc) if on else None
    if near is None:
        found = words
        if on:
            line = "meaning search failed this time; words only"
    else:
        found = fuse(db, words, near, terms(question)[1], weight)
    shown = {r["file"] for r in looked}
    found = [r for r in found if r["file"] not in shown]
    chat = prep
    if chat["reached"]:
        found = join_chat(found, chat_safe(chat_search, question, prep, default=[]))
        chat = {"reached": chat_folder()[0] is not None, "why": chat_folder()[1]}
    found = found[:k]
    left = db.execute("select value from meta where key = 'left'").fetchone()
    return {"question": question, "lookup": looked, "found": found,
            "took_ms": round((time.time() - t0) * 1000, 1), "meaning": line,
            "chat": "searched" if chat["reached"] else "not reached: %s" % chat["why"],
            "not_read_yet": int(left[0]) if left else 0}


HOME_KINDS = {"person", "project", "idea", "decision", "brief", "report"}
HOME_NEEDS = re.compile(r"\b(need|needs|must|have to|has to|buy|call|pay|remind|reminder|deadline|due|todo|to-do|"
                        r"don't forget|dont forget|book|renew)\b|محتاج|لازم|اشتري|كلم|ادفع", re.I)
HOME_IDEA = re.compile(r"\bideas?\b|فكرة", re.I)


def life_rel(path):
    """A path inside data/ as a path relative to it, or None (outside it, or not there)."""
    try:
        root = os.path.realpath(LIFE)
        real = os.path.realpath(path)
    except (OSError, ValueError, TypeError):
        return None
    if real == root or not real.startswith(root + os.sep) or not os.path.exists(real):
        return None
    rel = os.path.relpath(real, root)
    return None if rel.split(os.sep)[0].startswith(".") else rel


def homes(db, text, n=3):
    """Where a note likely goes, no model: a project, a person or a numbered page the words name; then
    the words "idea", "decide" and the needs words, each to its own folder; else the files holding
    the best sections by word search. At most n, each with a plain reason. Paths are relative to data/."""
    out, seen = [], set()

    def add(path, why):
        rel = life_rel(path) if os.path.isabs(path) else path
        if rel and os.path.basename(rel).lower().startswith("readme"):
            return
        if rel and rel not in seen and len(out) < n:
            seen.add(rel)
            out.append({"path": rel, "why": why})

    looked, _ = lookup(db, text, lifecfg.config(LIFE))
    for r in looked:
        if r.get("file") and r.get("kind") in HOME_KINDS:
            add(r["file"], r["why"])
    for rx, folder, why in ((DECIDE, "decisions", "you spoke of a decision"), (HOME_IDEA, "ideas", "you spoke of an idea"),
                            (HOME_NEEDS, "domains/personal", "something you need to do")):
        if rx.search(text) and os.path.isdir(os.path.join(LIFE, folder)):
            add(folder, why)
    if len(out) < n:
        for r in word_search(db, text, n * 3, lifecfg.config(LIFE)):
            if r.get("kind") in HOME_KINDS | {"note"}:
                add(r["file"], "it matches %s" % (r.get("title") or r.get("show") or "this page"))
    return out


def fuse(db, words, near, marks, weight=None):
    """Word ranking (files, boosted) and meaning ranking (best section per file), joined
    by reciprocal rank fusion. A file found by meaning alone shows its nearest section."""
    weight = MEANING_WEIGHT if weight is None else weight
    ids = [i for i, _ in near]
    rows = {r[0]: r[1:] for r in db.execute(
        "select s.id, f.path, f.show, f.kind, f.updated, f.title, s.heading, s.line, s.text"
        " from sections s join files f on f.id = s.file where s.id in (%s)" % ",".join("?" * len(ids)), ids)} \
        if ids else {}
    by_meaning, info = [], {}
    for sid, cos in near:
        if sid not in rows:
            continue
        path, show, kind, updated, title, heading, line, text = rows[sid]
        if path in info:
            continue
        at, lines = best_lines(text, line, marks)
        info[path] = {"file": path, "show": show, "line": at, "heading": heading, "kind": kind, "section": sid,
                      "updated": updated, "title": title, "lines": lines, "meaning": round(cos, 4)}
        by_meaning.append(path)
    score = {}
    for rank, r in enumerate(words):
        score[r["file"]] = score.get(r["file"], 0) + 1.0 / (RRF_K + rank + 1)
        info.setdefault(r["file"], r)
    for rank, path in enumerate(by_meaning):
        score[path] = score.get(path, 0) + weight / (RRF_K + rank + 1)
    word_rows = {r["file"]: r for r in words}
    out = []
    for path in sorted(score, key=lambda p: -score[p]):
        r = dict(word_rows.get(path) or info[path])
        r["score"] = round(score[path] * 1000, 2)
        if path in info and "meaning" in info[path]:
            r["meaning"] = info[path]["meaning"]
        out.append(r)
    return out


# ---------------------------------------------------------------- answers (brief 0112)

CONTEXT_SECTIONS = 6       # at most this many pieces of notes reach the AI
CONTEXT_CHARS = 9000       # about 2,500 tokens of English (Arabic takes more tokens a character)
PIECE_CHARS = 1500         # one piece at most
ANSWER_RULES = os.path.join(os.path.dirname(BIN), "prompts", "answer.md")


def piece(db, section_id=None, file_path=None):
    """(first line, last line, text) of one section, or of a file's opening sections."""
    if section_id is not None:
        # The section, and the ones after it in the same file while the piece is short.
        rows = db.execute("select s.line, s.text from sections s join sections o on o.id = ?"
                          " where s.file = o.file and s.line >= o.line order by s.line limit 4",
                          (section_id,)).fetchall()
        out, size = [], 0
        for line, t in rows:
            if out and size >= 400:
                break
            out.append((line, t))
            size += len(t)
        rows = out
    else:
        rows = db.execute("select s.line, s.text from sections s join files f on f.id = s.file"
                          " where f.path = ? order by s.line", (file_path,)).fetchall()
    text, first = "", None
    for line, t in rows:
        if first is None:
            first = line
        text = (text + "\n\n" + t).strip() if text else t
        if len(text) >= PIECE_CHARS:
            break
    if first is None:
        return None
    text = re.sub(r"<!--.*?-->\n?", "", text, flags=re.S)[:PIECE_CHARS]
    if len(text) == PIECE_CHARS and "\n" in text:
        text = text[:text.rfind("\n")]
    return first, first + text.count("\n"), text


def context(db, question, n=CONTEXT_SECTIONS, budget=CONTEXT_CHARS):
    """The question, today, and the best pieces of notes, numbered, for the AI to answer
    from. Lookups first (the files a number, a name or a day points at), then the
    search's best section of each file."""
    res = find(db, question, k=n)
    pieces, used, seen, chats = [], 0, set(), 0
    for r in res["lookup"] + res["found"]:
        if not r.get("file") or len(pieces) >= n:
            continue
        if is_chat(r):
            if chats >= CHAT_MAX_RESULTS:
                continue
            got = chat_safe(chat_piece, r)
        else:
            got = piece(db, section_id=r.get("section")) if r.get("section") else piece(db, file_path=r["file"])
        if not got or (r["file"], got[0]) in seen:
            continue
        first, last, text = got
        if used + len(text) > budget:
            text = text[:max(0, budget - used)]
            if len(text) < 200:
                break
        seen.add((r["file"], first))
        used += len(text)
        chats += is_chat(r)
        show = "%s, %s" % (r["show"], chat_label(r.get("who"))) if is_chat(r) else r["show"]
        pieces.append({"n": len(pieces) + 1, "file": r["file"], "show": show, "line": first,
                       "end_line": first + text.count("\n"), "kind": "chat" if is_chat(r) else r.get("kind"),
                       "updated": r.get("updated"), "text": text})
    today = datetime.date.today()
    head = ["Today is %s, %s." % (today.strftime("%A"), today.isoformat()), "",
            "The question: %s" % question.strip(), "",
            "The pieces of the user's notes found for it (%d):" % len(pieces)]
    if not pieces:
        head.append("(none: nothing in the notes matched)")
    for p in pieces:
        where = p["show"] if p["kind"] == "chat" else "%s, lines %d-%d%s" % (
            p["show"], p["line"], p["end_line"], ", updated %s" % p["updated"] if p["updated"] else "")
        head += ["", "[%d] %s" % (p["n"], where), p["text"]]
    return {"question": question, "today": today.isoformat(), "pieces": pieces,
            "prompt": "\n".join(head) + "\n", "rules": ANSWER_RULES, "meaning": res["meaning"],
            "chars": used}


CITE = re.compile(r"\[(\d+(?:\s*[,،]\s*\d+)*)\]")
NOT_FOUND = "NOT IN THE NOTES"


def cite(answer, ctx):
    """The citation check: every sentence must end with the number of a piece that was
    sent; a sentence without one is not shown. Returns what the app shows."""
    known = {p["n"]: p for p in ctx.get("pieces") or []}
    text = answer.strip()
    deeper = text.startswith(NOT_FOUND) or not known
    if text.startswith(NOT_FOUND):
        text = text[len(NOT_FOUND):].lstrip(" .:-\n")
    shown, dropped = [], []
    # A full stop inside quotes ("Not yet. Alex decides.") does not end the sentence.
    text = re.sub(r'"[^"\n]*"|“[^”\n]*”|«[^»\n]*»', lambda m: m.group(0).replace(".", "\u2024"), text)
    for para in [p for p in re.split(r"\n\s*\n|\n(?=[-*] )", text) if p.strip()]:
        for sent in re.split(r"(?<=[.!?؟。])\s+(?=\S)|(?<=\])\s+(?=\S)", para.strip()):
            sent = sent.strip().replace("\u2024", ".")
            if not sent:
                continue
            nums = [int(x) for m in CITE.finditer(sent) for x in re.split(r"[,،]", m.group(1))]
            bad = [x for x in nums if x not in known]
            if not nums or bad or not CITE.search(sent[-12:]):
                dropped.append({"text": sent, "why": "a number that was not sent: %s" % bad if bad
                                else "no source number at its end"})
                continue
            shown.append({"text": re.sub(r"\s+([.!?؟،,;:])", r"\1", CITE.sub("", sent)).strip(), "sources": [
                {"n": x, "file": known[x]["file"], "show": known[x]["show"], "line": known[x]["line"]}
                for x in dict.fromkeys(nums)]})
    if not shown:
        deeper = True
    return {"shown": shown, "dropped": dropped, "deeper": deeper,
            "line": None if shown else "I did not find that in your notes. Look deeper?"}


def deeper(question, ctx=None):
    """Road 2: the prompt for a full chat turn that reads the folder itself."""
    lines = ["%s" % question.strip(), "",
             "(From Argoni: the quick answer could not answer this from the pieces of notes it was given"
             + (", which were: " + "; ".join("%s line %d" % (p["show"], p["line"]) for p in ctx["pieces"])
                if ctx and ctx.get("pieces") else "") +
             ". Look in the life folder and the project folders yourself. Cite file and line for every"
             " fact. If it is not there, say so plainly.)"]
    return "\n".join(lines) + "\n"


# ---------------------------------------------------------------- the commands

def status(db):
    meta = get_meta(db)
    files, sections = db.execute("select count(*), coalesce(sum(sections), 0) from files").fetchone()
    by = dict(db.execute("select source, count(*) from files group by source"))
    size = sum(os.path.getsize(DB + x) for x in ("", "-wal") if os.path.exists(DB + x))
    return {"files": files, "sections": sections, "mb": round(size / 1e6, 1), "sources": by,
            "built_at": meta.get("built_at"), "swept_at": meta.get("swept_at"),
            "left": int(meta.get("left") or 0), "meaning": meaning_state(db, settings(), exact=True)[1],
            "vectors": db.execute("select count(*) from vectors").fetchone()[0],
            "path": os.path.relpath(DB, LIFE), "chat": chat_safe(chat_status, default={"reached": False, "why": "its index could not be read"})} | slow_warning(db)


def slow_warning(db):
    """{"warning": one plain line} once the vectors pass MEANING_WARN, else {}."""
    n = db.execute("select count(*) from vectors").fetchone()[0]
    if n < MEANING_WARN:
        return {}
    return {"warning": "meaning search: %d sections; on an older Mac (an M1 or M2, or one with 8 GB) a question may "
                       "take more than a tenth of a second, and updating the index needs several GB" % n}


def say_meaning(r):
    if not r:
        return
    if r.get("error"):
        print("memory: meaning vectors not made: %s" % r["error"])
    elif "embedded" in r:
        print("memory: %d section(s) given meaning vectors, %.1f s" % (r["embedded"], r.get("seconds", 0)))
    elif r.get("skipped"):
        print("memory: meaning vectors: %s" % r["skipped"])


def say_chat(r):
    if r:
        print("memory chat: %d day(s) read, %d gone, %d line(s), %d unverified line(s) not indexed, %.2f s"
              % (r["days"], r["gone"], r["entries"], r["unverified"], r["seconds"]))


def say_sweep(verb, r):
    left = "; %d file(s) left for the next sweep" % r["left"] if r["left"] else ""
    print("memory %s: %d file(s) read, %d gone, %d section(s), %.2f s%s"
          % (verb, r["read"], r["gone"], r["sections"], r["seconds"], left))


def main(argv):
    if not argv or argv[0] in ("-h", "--help", "help"):
        print(__doc__.split("\n\n")[1])
        return 0 if argv else 2
    verb, rest = argv[0], argv[1:]
    as_json = "--json" in rest
    rest = [a for a in rest if a != "--json"]
    if "--chat" in rest:
        i = rest.index("--chat")
        if i + 1 >= len(rest):
            print("memory: --chat needs the record folder", file=sys.stderr)
            return 2
        CHAT_ARG[0] = rest[i + 1]
        rest = rest[:i] + rest[i + 2:]
    st = settings()
    wait = "--wait" in rest
    rest = [a for a in rest if a != "--wait"]
    if verb == "build":
        db = connect()
        say_sweep("build", sweep(db, st, fresh=True))
        chat_cleanup_orphan()
        say_chat(chat_safe(chat_sweep, fresh=True))
        say_meaning(start_meaning(st, wait))
        return 0
    if verb == "update":
        budget = None
        if "--budget" in rest:
            i = rest.index("--budget")
            try:
                budget = float(rest[i + 1])
            except (IndexError, ValueError):
                print("memory: --budget needs a number of seconds", file=sys.stderr)
                return 2
            rest = rest[:i] + rest[i + 2:]
        fresh = not os.path.exists(DB)
        db = connect()
        if rest:
            for line in update_files(db, st, rest):
                print("memory: " + line)
            say_meaning(start_meaning(st, wait))
            return 0
        r = sweep(db, st, budget, fresh=fresh)
        say_sweep("build" if fresh else "update", r)
        chat_cleanup_orphan()
        c = chat_safe(chat_sweep, None if budget is None else max(0.0, budget - r["seconds"]))
        say_chat(c)
        if r["read"] or r["gone"] or wait or fresh or (c and (c["days"] or c["gone"])):
            say_meaning(start_meaning(st, wait))
        return 0
    if verb == "find":
        k = 5
        if "-k" in rest:
            i = rest.index("-k")
            try:
                k = int(rest[i + 1])
            except (IndexError, ValueError):
                print("memory: -k needs a number", file=sys.stderr)
                return 2
            rest = rest[:i] + rest[i + 2:]
        question = " ".join(rest).strip()
        if not question:
            print('memory: find needs a question: bin/memory find "what did I decide about ..."', file=sys.stderr)
            return 2
        if not os.path.exists(DB):
            r = sweep(connect(), st, fresh=True)
            print("memory: no index yet, built it (%d files, %.1f s)" % (r["read"], r["seconds"]), file=sys.stderr)
        res = find(connect(), question, k)
        if as_json:
            print(json.dumps(res, ensure_ascii=False, indent=1))
            return 0
        for r in res["lookup"]:
            where = "%s:%s" % (r["show"], r["line"]) if r["file"] else r["show"]
            print("%s  (%s)" % (where, r["why"]))
            for l in r.get("lines") or []:
                print("    " + l)
        if res["lookup"] and res["found"]:
            print("")
        for r in res["found"]:
            if is_chat(r):
                print("%s (%s): %s" % (r["show"], chat_label(r["who"]), (r["lines"] or [""])[0]))
                continue
            print("%s:%d  %s" % (r["show"], r["line"], r["heading"] or r["title"]))
            for l in r["lines"]:
                print("    " + l)
        if not res["lookup"] and not res["found"]:
            print("Nothing found for that. Try other words, or fewer.")
        if res["not_read_yet"]:
            print("(%d file(s) not read yet: the first build goes on at the next sweep)" % res["not_read_yet"])
        print("(%.0f ms; %s)" % (res["took_ms"], res["meaning"]))
        if res["chat"] != "searched":
            print("(chat %s)" % res["chat"])
        return 0
    if verb == "home":
        text = sys.stdin.read()[:20000]
        if not os.path.exists(DB):
            sweep(connect(), st, fresh=True)
        print(json.dumps({"homes": homes(connect(), text)}, ensure_ascii=False))
        return 0
    if verb == "context":
        question = " ".join(rest).strip()
        if not question:
            print('memory: context needs a question', file=sys.stderr)
            return 2
        if not os.path.exists(DB):
            sweep(connect(), st, fresh=True)
        ctx = context(connect(), question)
        print(json.dumps(ctx, ensure_ascii=False, indent=1) if as_json else ctx["prompt"], end="" if not as_json else "\n")
        return 0
    if verb == "cite":
        if not rest or not os.path.isfile(rest[0]):
            print("memory: cite needs the context file (memory context --json): bin/memory cite CTX.json < answer",
                  file=sys.stderr)
            return 2
        with open(rest[0], encoding="utf-8") as fh:
            ctx = json.load(fh)
        print(json.dumps(cite(sys.stdin.read(), ctx), ensure_ascii=False, indent=1))
        return 0
    if verb == "deeper":
        ctx = None
        if rest and rest[0] == "--context":
            with open(rest[1], encoding="utf-8") as fh:
                ctx = json.load(fh)
            rest = rest[2:]
        print(deeper(" ".join(rest) or (ctx or {}).get("question", ""), ctx), end="")
        return 0
    if verb == "status":
        if not os.path.exists(DB):
            print(json.dumps({"files": 0, "built_at": None}) if as_json else "memory: no index yet (bin/memory build)")
            return 0
        s = status(connect())
        if as_json:
            print(json.dumps(s, ensure_ascii=False, indent=1))
        else:
            print("memory: %d files, %d sections, %.1f MB; built %s, swept %s%s; %s"
                  % (s["files"], s["sections"], s["mb"], s["built_at"] or "-", s["swept_at"] or "-",
                     "; %d left" % s["left"] if s["left"] else "", s["meaning"]))
            if s.get("warning"):
                print("memory: " + s["warning"])
            c = s["chat"]
            if not c["reached"]:
                print("memory: chat not reached (%s)" % c["why"])
            elif not c.get("indexed"):
                print("memory: chat reached, not indexed yet")
            else:
                print("memory: chat: %d day(s), %d line(s)%s" % (
                    c["days"], c["entries"],
                    "; %d unverified line(s) not indexed" % c["unverified"] if c["unverified"] else ""))
            for src, n in sorted(s["sources"].items(), key=lambda x: (x[0] != lifecfg.OWN, x[0])):
                print("  %-20s %d files" % (src, n))
        return 0
    if verb == "forget":
        files = []
        if "--file" in rest:
            i = rest.index("--file")
            files, rest = rest[i + 1:], rest[:i]
        bad = [i for i in rest if not CHAT_ID.fullmatch(i)]
        if bad or not (rest or files) or ("--file" in argv and not files):
            print("memory: forget needs chat line ids (m-YYYYMMDD-HHMMSS-xxxxxx) and/or --file PATH...%s"
                  % ("; not an id: %s" % bad[0] if bad else ""), file=sys.stderr)
            return 2
        try:
            res = forget(rest, files)
        except sqlite3.DatabaseError as e:
            if any(w in str(e).lower() for w in BROKEN):
                raise   # the notes' index: run_main sets it aside and runs this again
            print("memory: forget failed: %s" % e, file=sys.stderr)
            return 4
        except Exception as e:   # never exit 1 for a crash: 1 means "chat not reached"
            print("memory: forget failed: %s: %s" % (type(e).__name__, e), file=sys.stderr)
            return 4
        print(json.dumps(res, ensure_ascii=False))
        if res["embed_busy"] or not res["erased"]:
            return 3   # the lines are deleted, the clean-up is not whole: try again
        return 1 if rest and res["chat"].startswith("not reached") else 0
    print("memory: unknown command %r (build, update, find, home, context, cite, deeper, status, forget)" % verb, file=sys.stderr)
    return 2


# What a broken index says (security of 0117, r3 L1); never "interrupted" or "locked".
BROKEN = ("malformed", "no such module", "fts5", "not a database", "corrupt")


def set_aside(why):
    """The index is a cache: a broken one is moved aside (one copy kept, for a look), its
    -wal and -shm removed, and the next open makes a fresh one."""
    for f in (DB + "-wal", DB + "-shm", DB + "-journal"):
        if os.path.lexists(f):
            os.remove(f)
    if os.path.lexists(DB):
        try:
            os.replace(DB, DB + ".broken")
        except OSError:  # a folder planted at that name (security r4, A2): the cache goes
            os.remove(DB)
    print("memory: the index was broken (%s); set aside as %s, and made again"
          % (why, os.path.relpath(DB + ".broken", LIFE)), file=sys.stderr)


def run_main(argv):
    try:
        return main(argv)
    except sqlite3.DatabaseError as e:
        if not any(w in str(e).lower() for w in BROKEN):
            raise
        set_aside(str(e)[:120])
    return main(argv)


if __name__ == "__main__":
    sys.exit(run_main(sys.argv[1:]))
