mihailkudravcev
335 lines
12 KiB
Python
335 lines
12 KiB
Python
"""Запись индексов: JSONL, Markdown, SQLite FTS5."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import sqlite3
|
||
from collections import defaultdict
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any, Iterable
|
||
|
||
from .parse_meta import ObjectRec
|
||
|
||
|
||
def _utcnow() -> str:
|
||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
|
||
def write_jsonl(path: Path, rows: Iterable[dict[str, Any]]) -> int:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
n = 0
|
||
with path.open("w", encoding="utf-8") as f:
|
||
for row in rows:
|
||
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def write_objects_markdown(path: Path, objects: list[ObjectRec], *, title: str) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
by_type: dict[str, list[ObjectRec]] = defaultdict(list)
|
||
for obj in objects:
|
||
by_type[obj.object_type].append(obj)
|
||
|
||
lines: list[str] = [
|
||
f"# {title}",
|
||
"",
|
||
f"Сгенерировано: {_utcnow()}. Быстрый поиск метаданных без обхода XML.",
|
||
"",
|
||
]
|
||
for otype in sorted(by_type):
|
||
lines.append(f"## {otype} ({len(by_type[otype])})")
|
||
lines.append("")
|
||
for obj in sorted(by_type[otype], key=lambda o: o.name.casefold()):
|
||
syn = f" — {obj.synonym}" if obj.synonym else ""
|
||
lines.append(f"### `{obj.full_name}`{syn}")
|
||
if obj.comment:
|
||
lines.append(f"- Комментарий: {obj.comment}")
|
||
if obj.tooltip:
|
||
lines.append(f"- Подсказка: {obj.tooltip}")
|
||
if obj.owners:
|
||
lines.append(f"- Владельцы: {', '.join(obj.owners)}")
|
||
if obj.register_records:
|
||
lines.append(f"- Движения: {', '.join(obj.register_records)}")
|
||
if obj.based_on:
|
||
lines.append(f"- Вводится на основании: {', '.join(obj.based_on)}")
|
||
attrs = [f for f in obj.fields if f.kind != "TabularSection"]
|
||
if attrs:
|
||
lines.append("- Поля:")
|
||
for f in attrs:
|
||
loc = f".{f.tabular_section}" if f.tabular_section else ""
|
||
types = ", ".join(f.types) if f.types else ""
|
||
syn_f = f" ({f.synonym})" if f.synonym else ""
|
||
type_s = f" : `{types}`" if types else ""
|
||
refs = ""
|
||
if f.refs:
|
||
refs = " → " + ", ".join(
|
||
f"{r['kind']}.{r['name']}" for r in f.refs
|
||
)
|
||
lines.append(
|
||
f" - `{f.kind}{loc}.{f.name}`{syn_f}{type_s}{refs}"
|
||
)
|
||
lines.append("")
|
||
path.write_text("\n".join(lines), encoding="utf-8")
|
||
|
||
|
||
def write_refs_markdown(path: Path, objects: list[ObjectRec], *, title: str) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
inv: dict[str, list[str]] = defaultdict(list)
|
||
for obj in objects:
|
||
for f in obj.fields:
|
||
for r in f.refs:
|
||
target = f"{r['kind']}.{r['name']}"
|
||
loc = f".{f.tabular_section}" if f.tabular_section else ""
|
||
inv[target].append(f"{obj.full_name}{loc}.{f.name}")
|
||
|
||
lines = [
|
||
f"# {title}",
|
||
"",
|
||
f"Сгенерировано: {_utcnow()}.",
|
||
"",
|
||
]
|
||
for target in sorted(inv, key=lambda s: s.casefold()):
|
||
lines.append(f"## `{target}`")
|
||
for src in sorted(set(inv[target])):
|
||
lines.append(f"- `{src}`")
|
||
lines.append("")
|
||
path.write_text("\n".join(lines), encoding="utf-8")
|
||
|
||
|
||
def write_sqlite_fts(path: Path, objects: list[ObjectRec], config: str) -> None:
|
||
if path.exists():
|
||
path.unlink()
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
conn = sqlite3.connect(path)
|
||
try:
|
||
conn.execute("PRAGMA journal_mode=WAL")
|
||
conn.execute(
|
||
"""
|
||
CREATE TABLE objects (
|
||
id INTEGER PRIMARY KEY,
|
||
config TEXT NOT NULL,
|
||
object_type TEXT NOT NULL,
|
||
name TEXT NOT NULL,
|
||
full_name TEXT NOT NULL,
|
||
synonym TEXT,
|
||
comment TEXT,
|
||
path TEXT,
|
||
json TEXT NOT NULL
|
||
)
|
||
"""
|
||
)
|
||
conn.execute(
|
||
"""
|
||
CREATE VIRTUAL TABLE objects_fts USING fts5(
|
||
full_name,
|
||
synonym,
|
||
comment,
|
||
fields_text,
|
||
object_id UNINDEXED,
|
||
tokenize='unicode61'
|
||
)
|
||
"""
|
||
)
|
||
conn.execute(
|
||
"""
|
||
CREATE TABLE fields (
|
||
object_id INTEGER NOT NULL,
|
||
kind TEXT,
|
||
name TEXT,
|
||
synonym TEXT,
|
||
types TEXT,
|
||
tabular_section TEXT,
|
||
refs_json TEXT,
|
||
FOREIGN KEY(object_id) REFERENCES objects(id)
|
||
)
|
||
"""
|
||
)
|
||
conn.execute(
|
||
"""
|
||
CREATE TABLE refs (
|
||
from_full_name TEXT NOT NULL,
|
||
from_field TEXT NOT NULL,
|
||
to_kind TEXT NOT NULL,
|
||
to_name TEXT NOT NULL
|
||
)
|
||
"""
|
||
)
|
||
|
||
for obj in objects:
|
||
fields_text = " ".join(
|
||
" ".join(
|
||
filter(
|
||
None,
|
||
[
|
||
f.kind,
|
||
f.name,
|
||
f.synonym,
|
||
f.comment,
|
||
f.tabular_section,
|
||
" ".join(f.types),
|
||
],
|
||
)
|
||
)
|
||
for f in obj.fields
|
||
)
|
||
payload = json.dumps(obj.to_dict(), ensure_ascii=False)
|
||
cur = conn.execute(
|
||
"""
|
||
INSERT INTO objects
|
||
(config, object_type, name, full_name, synonym, comment, path, json)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
config,
|
||
obj.object_type,
|
||
obj.name,
|
||
obj.full_name,
|
||
obj.synonym,
|
||
obj.comment,
|
||
obj.path,
|
||
payload,
|
||
),
|
||
)
|
||
oid = cur.lastrowid
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO objects_fts
|
||
(full_name, synonym, comment, fields_text, object_id)
|
||
VALUES (?, ?, ?, ?, ?)
|
||
""",
|
||
(obj.full_name, obj.synonym or "", obj.comment or "", fields_text, oid),
|
||
)
|
||
for f in obj.fields:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO fields
|
||
(object_id, kind, name, synonym, types, tabular_section, refs_json)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
oid,
|
||
f.kind,
|
||
f.name,
|
||
f.synonym,
|
||
", ".join(f.types),
|
||
f.tabular_section,
|
||
json.dumps(f.refs, ensure_ascii=False),
|
||
),
|
||
)
|
||
for r in f.refs:
|
||
loc = f"{f.tabular_section}." if f.tabular_section else ""
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO refs
|
||
(from_full_name, from_field, to_kind, to_name)
|
||
VALUES (?, ?, ?, ?)
|
||
""",
|
||
(obj.full_name, f"{loc}{f.name}", r["kind"], r["name"]),
|
||
)
|
||
|
||
conn.execute("CREATE INDEX idx_objects_type ON objects(object_type)")
|
||
conn.execute("CREATE INDEX idx_objects_name ON objects(name)")
|
||
conn.execute("CREATE INDEX idx_refs_to ON refs(to_kind, to_name)")
|
||
conn.execute("CREATE INDEX idx_refs_from ON refs(from_full_name)")
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def write_manifest(path: Path, data: dict[str, Any]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
data = {**data, "generated_at": _utcnow()}
|
||
path.write_text(
|
||
json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
|
||
|
||
def write_type_summaries(out_dir: Path, objects: list[ObjectRec]) -> None:
|
||
by_type: dict[str, list[ObjectRec]] = defaultdict(list)
|
||
for obj in objects:
|
||
by_type[obj.object_type].append(obj)
|
||
types_dir = out_dir / "by_type"
|
||
types_dir.mkdir(parents=True, exist_ok=True)
|
||
for otype, items in by_type.items():
|
||
lines = [f"# {otype}", "", f"Объектов: {len(items)}", ""]
|
||
for obj in sorted(items, key=lambda o: o.name.casefold()):
|
||
syn = f" — {obj.synonym}" if obj.synonym else ""
|
||
n_fields = len([f for f in obj.fields if f.kind != "TabularSection"])
|
||
lines.append(f"- `{obj.name}`{syn} (полей: {n_fields})")
|
||
(types_dir / f"{otype}.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
def write_global_index_md(cache_root: Path, summary: dict[str, Any]) -> Path:
|
||
"""Сводка по единому index.sqlite."""
|
||
cache_root.mkdir(parents=True, exist_ok=True)
|
||
configs = summary.get("configs") or []
|
||
lines = [
|
||
"# Индекс метаданных и модулей 1С (проект)",
|
||
"",
|
||
f"Сгенерировано: {_utcnow()}.",
|
||
f"Режим: `{summary.get('mode', '?')}`, БД: `cache/1c_meta/index.sqlite`.",
|
||
f"Потоки: {summary.get('workers', '?')}, время: {summary.get('elapsed_sec', '?')} с.",
|
||
"",
|
||
"Поиск: `python tools/index/query_1c.py search \"…\"`",
|
||
"Переиндексация: `python tools/index/index_1c.py reindex` (инкремент) / `--full`.",
|
||
"",
|
||
"## Конфигурации",
|
||
"",
|
||
"| Конфигурация | Объектов | Полей | Ссылок | Модулей |",
|
||
"|---|---:|---:|---:|---:|",
|
||
]
|
||
for c in sorted(configs, key=lambda x: x.get("name") or x.get("config") or ""):
|
||
name = c.get("name") or c.get("config") or "?"
|
||
lines.append(
|
||
"| `{name}` | {objects} | {fields} | {refs} | {modules} |".format(
|
||
name=name,
|
||
objects=c.get("objects", 0),
|
||
fields=c.get("fields", 0),
|
||
refs=c.get("refs", 0),
|
||
modules=c.get("modules", 0),
|
||
)
|
||
)
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"## Статистика прогона",
|
||
"",
|
||
f"- meta разобрано: {summary.get('meta_parsed', 0)}, пропущено (без изменений): {summary.get('meta_skipped', 0)}",
|
||
f"- modules разобрано: {summary.get('modules_parsed', 0)}, пропущено: {summary.get('modules_skipped', 0)}",
|
||
f"- удалено файлов: {summary.get('files_removed', 0)}, ошибок: {summary.get('errors', 0)}",
|
||
"",
|
||
"## Артефакты",
|
||
"",
|
||
"- `index.sqlite` — единый кэш проекта (объекты, поля, ссылки, модули, FTS)",
|
||
"- `INDEX.md` — эта сводка",
|
||
"- `md/<config>/` — опциональный Markdown (`reindex --md`)",
|
||
"",
|
||
]
|
||
)
|
||
path = cache_root / "INDEX.md"
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
write_manifest(
|
||
cache_root / "manifest.json",
|
||
{
|
||
"db": "index.sqlite",
|
||
"mode": summary.get("mode"),
|
||
"configs": configs,
|
||
"summary": {
|
||
k: summary.get(k)
|
||
for k in (
|
||
"meta_parsed",
|
||
"meta_skipped",
|
||
"modules_parsed",
|
||
"modules_skipped",
|
||
"files_removed",
|
||
"errors",
|
||
"elapsed_sec",
|
||
"workers",
|
||
)
|
||
},
|
||
},
|
||
)
|
||
return path
|