Детализирована документация. Небольшие изменения имен параметров.
This commit is contained in:
mihailkudravcev
@@ -0,0 +1,334 @@
|
||||
"""Запись индексов: JSONL, Markdown, SQLite FTS5."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .parse_meta import ObjectRec
|
||||
|
||||
|
||||
def _utcnow() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def write_jsonl(path: Path, rows: Iterable[dict[str, Any]]) -> int:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
n = 0
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
for row in rows:
|
||||
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def write_objects_markdown(path: Path, objects: list[ObjectRec], *, title: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
by_type: dict[str, list[ObjectRec]] = defaultdict(list)
|
||||
for obj in objects:
|
||||
by_type[obj.object_type].append(obj)
|
||||
|
||||
lines: list[str] = [
|
||||
f"# {title}",
|
||||
"",
|
||||
f"Сгенерировано: {_utcnow()}. Быстрый поиск метаданных без обхода XML.",
|
||||
"",
|
||||
]
|
||||
for otype in sorted(by_type):
|
||||
lines.append(f"## {otype} ({len(by_type[otype])})")
|
||||
lines.append("")
|
||||
for obj in sorted(by_type[otype], key=lambda o: o.name.casefold()):
|
||||
syn = f" — {obj.synonym}" if obj.synonym else ""
|
||||
lines.append(f"### `{obj.full_name}`{syn}")
|
||||
if obj.comment:
|
||||
lines.append(f"- Комментарий: {obj.comment}")
|
||||
if obj.tooltip:
|
||||
lines.append(f"- Подсказка: {obj.tooltip}")
|
||||
if obj.owners:
|
||||
lines.append(f"- Владельцы: {', '.join(obj.owners)}")
|
||||
if obj.register_records:
|
||||
lines.append(f"- Движения: {', '.join(obj.register_records)}")
|
||||
if obj.based_on:
|
||||
lines.append(f"- Вводится на основании: {', '.join(obj.based_on)}")
|
||||
attrs = [f for f in obj.fields if f.kind != "TabularSection"]
|
||||
if attrs:
|
||||
lines.append("- Поля:")
|
||||
for f in attrs:
|
||||
loc = f".{f.tabular_section}" if f.tabular_section else ""
|
||||
types = ", ".join(f.types) if f.types else ""
|
||||
syn_f = f" ({f.synonym})" if f.synonym else ""
|
||||
type_s = f" : `{types}`" if types else ""
|
||||
refs = ""
|
||||
if f.refs:
|
||||
refs = " → " + ", ".join(
|
||||
f"{r['kind']}.{r['name']}" for r in f.refs
|
||||
)
|
||||
lines.append(
|
||||
f" - `{f.kind}{loc}.{f.name}`{syn_f}{type_s}{refs}"
|
||||
)
|
||||
lines.append("")
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def write_refs_markdown(path: Path, objects: list[ObjectRec], *, title: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
inv: dict[str, list[str]] = defaultdict(list)
|
||||
for obj in objects:
|
||||
for f in obj.fields:
|
||||
for r in f.refs:
|
||||
target = f"{r['kind']}.{r['name']}"
|
||||
loc = f".{f.tabular_section}" if f.tabular_section else ""
|
||||
inv[target].append(f"{obj.full_name}{loc}.{f.name}")
|
||||
|
||||
lines = [
|
||||
f"# {title}",
|
||||
"",
|
||||
f"Сгенерировано: {_utcnow()}.",
|
||||
"",
|
||||
]
|
||||
for target in sorted(inv, key=lambda s: s.casefold()):
|
||||
lines.append(f"## `{target}`")
|
||||
for src in sorted(set(inv[target])):
|
||||
lines.append(f"- `{src}`")
|
||||
lines.append("")
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def write_sqlite_fts(path: Path, objects: list[ObjectRec], config: str) -> None:
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE objects (
|
||||
id INTEGER PRIMARY KEY,
|
||||
config TEXT NOT NULL,
|
||||
object_type TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
full_name TEXT NOT NULL,
|
||||
synonym TEXT,
|
||||
comment TEXT,
|
||||
path TEXT,
|
||||
json TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE VIRTUAL TABLE objects_fts USING fts5(
|
||||
full_name,
|
||||
synonym,
|
||||
comment,
|
||||
fields_text,
|
||||
object_id UNINDEXED,
|
||||
tokenize='unicode61'
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE fields (
|
||||
object_id INTEGER NOT NULL,
|
||||
kind TEXT,
|
||||
name TEXT,
|
||||
synonym TEXT,
|
||||
types TEXT,
|
||||
tabular_section TEXT,
|
||||
refs_json TEXT,
|
||||
FOREIGN KEY(object_id) REFERENCES objects(id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE refs (
|
||||
from_full_name TEXT NOT NULL,
|
||||
from_field TEXT NOT NULL,
|
||||
to_kind TEXT NOT NULL,
|
||||
to_name TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
for obj in objects:
|
||||
fields_text = " ".join(
|
||||
" ".join(
|
||||
filter(
|
||||
None,
|
||||
[
|
||||
f.kind,
|
||||
f.name,
|
||||
f.synonym,
|
||||
f.comment,
|
||||
f.tabular_section,
|
||||
" ".join(f.types),
|
||||
],
|
||||
)
|
||||
)
|
||||
for f in obj.fields
|
||||
)
|
||||
payload = json.dumps(obj.to_dict(), ensure_ascii=False)
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO objects
|
||||
(config, object_type, name, full_name, synonym, comment, path, json)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
config,
|
||||
obj.object_type,
|
||||
obj.name,
|
||||
obj.full_name,
|
||||
obj.synonym,
|
||||
obj.comment,
|
||||
obj.path,
|
||||
payload,
|
||||
),
|
||||
)
|
||||
oid = cur.lastrowid
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO objects_fts
|
||||
(full_name, synonym, comment, fields_text, object_id)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(obj.full_name, obj.synonym or "", obj.comment or "", fields_text, oid),
|
||||
)
|
||||
for f in obj.fields:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO fields
|
||||
(object_id, kind, name, synonym, types, tabular_section, refs_json)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
oid,
|
||||
f.kind,
|
||||
f.name,
|
||||
f.synonym,
|
||||
", ".join(f.types),
|
||||
f.tabular_section,
|
||||
json.dumps(f.refs, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
for r in f.refs:
|
||||
loc = f"{f.tabular_section}." if f.tabular_section else ""
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO refs
|
||||
(from_full_name, from_field, to_kind, to_name)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(obj.full_name, f"{loc}{f.name}", r["kind"], r["name"]),
|
||||
)
|
||||
|
||||
conn.execute("CREATE INDEX idx_objects_type ON objects(object_type)")
|
||||
conn.execute("CREATE INDEX idx_objects_name ON objects(name)")
|
||||
conn.execute("CREATE INDEX idx_refs_to ON refs(to_kind, to_name)")
|
||||
conn.execute("CREATE INDEX idx_refs_from ON refs(from_full_name)")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def write_manifest(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {**data, "generated_at": _utcnow()}
|
||||
path.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def write_type_summaries(out_dir: Path, objects: list[ObjectRec]) -> None:
|
||||
by_type: dict[str, list[ObjectRec]] = defaultdict(list)
|
||||
for obj in objects:
|
||||
by_type[obj.object_type].append(obj)
|
||||
types_dir = out_dir / "by_type"
|
||||
types_dir.mkdir(parents=True, exist_ok=True)
|
||||
for otype, items in by_type.items():
|
||||
lines = [f"# {otype}", "", f"Объектов: {len(items)}", ""]
|
||||
for obj in sorted(items, key=lambda o: o.name.casefold()):
|
||||
syn = f" — {obj.synonym}" if obj.synonym else ""
|
||||
n_fields = len([f for f in obj.fields if f.kind != "TabularSection"])
|
||||
lines.append(f"- `{obj.name}`{syn} (полей: {n_fields})")
|
||||
(types_dir / f"{otype}.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def write_global_index_md(cache_root: Path, summary: dict[str, Any]) -> Path:
|
||||
"""Сводка по единому index.sqlite."""
|
||||
cache_root.mkdir(parents=True, exist_ok=True)
|
||||
configs = summary.get("configs") or []
|
||||
lines = [
|
||||
"# Индекс метаданных и модулей 1С (проект)",
|
||||
"",
|
||||
f"Сгенерировано: {_utcnow()}.",
|
||||
f"Режим: `{summary.get('mode', '?')}`, БД: `cache/1c_meta/index.sqlite`.",
|
||||
f"Потоки: {summary.get('workers', '?')}, время: {summary.get('elapsed_sec', '?')} с.",
|
||||
"",
|
||||
"Поиск: `python tools/index/query_1c.py search \"…\"`",
|
||||
"Переиндексация: `python tools/index/index_1c.py reindex` (инкремент) / `--full`.",
|
||||
"",
|
||||
"## Конфигурации",
|
||||
"",
|
||||
"| Конфигурация | Объектов | Полей | Ссылок | Модулей |",
|
||||
"|---|---:|---:|---:|---:|",
|
||||
]
|
||||
for c in sorted(configs, key=lambda x: x.get("name") or x.get("config") or ""):
|
||||
name = c.get("name") or c.get("config") or "?"
|
||||
lines.append(
|
||||
"| `{name}` | {objects} | {fields} | {refs} | {modules} |".format(
|
||||
name=name,
|
||||
objects=c.get("objects", 0),
|
||||
fields=c.get("fields", 0),
|
||||
refs=c.get("refs", 0),
|
||||
modules=c.get("modules", 0),
|
||||
)
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Статистика прогона",
|
||||
"",
|
||||
f"- meta разобрано: {summary.get('meta_parsed', 0)}, пропущено (без изменений): {summary.get('meta_skipped', 0)}",
|
||||
f"- modules разобрано: {summary.get('modules_parsed', 0)}, пропущено: {summary.get('modules_skipped', 0)}",
|
||||
f"- удалено файлов: {summary.get('files_removed', 0)}, ошибок: {summary.get('errors', 0)}",
|
||||
"",
|
||||
"## Артефакты",
|
||||
"",
|
||||
"- `index.sqlite` — единый кэш проекта (объекты, поля, ссылки, модули, FTS)",
|
||||
"- `INDEX.md` — эта сводка",
|
||||
"- `md/<config>/` — опциональный Markdown (`reindex --md`)",
|
||||
"",
|
||||
]
|
||||
)
|
||||
path = cache_root / "INDEX.md"
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
write_manifest(
|
||||
cache_root / "manifest.json",
|
||||
{
|
||||
"db": "index.sqlite",
|
||||
"mode": summary.get("mode"),
|
||||
"configs": configs,
|
||||
"summary": {
|
||||
k: summary.get(k)
|
||||
for k in (
|
||||
"meta_parsed",
|
||||
"meta_skipped",
|
||||
"modules_parsed",
|
||||
"modules_skipped",
|
||||
"files_removed",
|
||||
"errors",
|
||||
"elapsed_sec",
|
||||
"workers",
|
||||
)
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
Reference in New Issue
Block a user