#!/usr/bin/env python3
"""Hindsight Meta-Memory: monthly dedup + decay report.

Silent when nothing to do (empty stdout = no message sent).
Outputs a summary only when memories were actually cleaned."""
import subprocess
import re
import sys
from collections import defaultdict
from datetime import datetime, timezone

PSQL = "/home/debian/.pg0/installation/18.1.0/bin/psql"
DB_ARGS = ["-h", "127.0.0.1", "-p", "5432", "-U", "hindsight", "-d", "hindsight"]

def psql_query(sql):
    result = subprocess.run(
        [PSQL] + DB_ARGS + ["-t", "-A", "-F", "\t", "-c", sql],
        capture_output=True, text=True
    )
    return result.stdout.strip()

def psql_exec(sql):
    result = subprocess.run(
        [PSQL] + DB_ARGS + ["-c", sql],
        capture_output=True, text=True
    )
    return result.stdout.strip()

def normalize(t):
    t = re.sub(r'\s*\|.*$', '', t)
    t = t.lower()
    t = re.sub(r'[^\w\s]', '', t)
    t = re.sub(r'\s+', ' ', t).strip()
    return t

def word_set(t):
    return set(normalize(t).split())

def main():
    rows = psql_query("SELECT id, text, created_at, access_count FROM memory_units WHERE bank_id='hermes' ORDER BY created_at;")
    if not rows:
        return

    entries = []
    for line in rows.split('\n'):
        if '\t' in line:
            parts = line.split('\t')
            entries.append({"id": parts[0], "text": parts[1],
                           "created": parts[2] if len(parts) > 2 else "",
                           "access": int(parts[3]) if len(parts) > 3 and parts[3].isdigit() else 0})

    initial_count = len(entries)
    ids_to_delete = []

    # Round 1: Exact text duplicates (keep oldest)
    by_text = defaultdict(list)
    for e in entries:
        by_text[e["text"]].append(e)
    for text, group in by_text.items():
        if len(group) > 1:
            group.sort(key=lambda e: e["created"])
            for e in group[1:]:
                ids_to_delete.append(e["id"])

    # Round 2: Normalized text duplicates (keep longest)
    by_norm = defaultdict(list)
    for e in entries:
        by_norm[normalize(e["text"])].append(e)
    for norm, group in by_norm.items():
        if len(group) > 1:
            group.sort(key=lambda e: len(e["text"]), reverse=True)
            for e in group[1:]:
                if e["id"] not in ids_to_delete:
                    ids_to_delete.append(e["id"])

    # Round 3: Near-duplicates via Jaccard similarity > 0.65 (keep longest)
    texts_with_words = [(e, word_set(e["text"])) for e in entries if e["id"] not in ids_to_delete]
    used = set()
    for i, (e1, ws1) in enumerate(texts_with_words):
        if e1["id"] in used:
            continue
        cluster = [e1]
        for j, (e2, ws2) in enumerate(texts_with_words):
            if i >= j or e2["id"] in used:
                continue
            if not ws1 or not ws2:
                continue
            jaccard = len(ws1 & ws2) / len(ws1 | ws2) if (ws1 | ws2) else 0
            if jaccard > 0.65:
                cluster.append(e2)
                used.add(e2["id"])
        if len(cluster) > 1:
            cluster.sort(key=lambda e: len(e["text"]), reverse=True)
            used.add(cluster[0]["id"])
            for e in cluster[1:]:
                if e["id"] not in ids_to_delete:
                    ids_to_delete.append(e["id"])

    deleted = 0
    if ids_to_delete:
        for i in range(0, len(ids_to_delete), 50):
            chunk = ids_to_delete[i:i+50]
            id_list = ",".join(f"'{uid}'" for uid in chunk)
            result = psql_exec(f"DELETE FROM memory_units WHERE id IN ({id_list});")
            if "DELETE" in result:
                num = int(re.search(r'DELETE (\d+)', result).group(1))
                deleted += num

    final_count = initial_count - deleted
    old_unused = psql_query(
        "SELECT COUNT(*) FROM memory_units WHERE bank_id='hermes' "
        "AND access_count = 0 AND created_at < NOW() - INTERVAL '6 months';"
    )

    if deleted == 0 and (not old_unused or old_unused == "0"):
        return

    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
    print(f"🧹 Hindsight Meta-Memory Cleanup — {now}")
    print(f"   Memories: {initial_count} → {final_count} (−{deleted} duplicates removed)")
    if old_unused and old_unused != "0":
        print(f"   ⚠️ {old_unused} memories older than 6 months with 0 accesses (consider manual review)")

if __name__ == "__main__":
    main()
