Детализирована документация. Небольшие изменения имен параметров.

index1c/build.py

@@ -0,0 +1,349 @@
+"""Построение единого индекса проекта (метаданные + модули, full/incremental)."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any
+
+from .config import DEFAULT_SKIP_TYPES, OBJECT_TYPE_FOLDERS, ConfigPaths, cache_dir
+from . import db as dbmod
+from .parse_meta import FieldRec, ObjectRec, iter_object_xml_files, parse_metadata_xml
+from .parse_modules import ModuleRec, iter_bsl_files, parse_bsl_file
+from . import writers
+
+
+def default_workers() -> int:
+    return max(4, min(16, (os.cpu_count() or 4)))
+
+
+def _file_stamp(path: Path) -> tuple[int, int]:
+    st = path.stat()
+    mtime_ns = getattr(st, "st_mtime_ns", int(st.st_mtime * 1e9))
+    return mtime_ns, st.st_size
+
+
+def _parse_meta_job(args: tuple[str, str, str]) -> tuple[str, ObjectRec | None, tuple[int, int]]:
+    config, otype, path_s = args
+    path = Path(path_s)
+    stamp = _file_stamp(path)
+    rec = parse_metadata_xml(path, config=config, object_type=otype)
+    return path_s, rec, stamp
+
+
+def _parse_module_job(args: tuple[str, str, str]) -> ModuleRec | None:
+    config, src_s, path_s = args
+    return parse_bsl_file(Path(path_s), config=config, src_root=Path(src_s))
+
+
+def _fields_text(obj: ObjectRec) -> str:
+    return " ".join(
+        " ".join(
+            filter(
+                None,
+                [
+                    f.kind,
+                    f.name,
+                    f.synonym,
+                    f.comment,
+                    f.tabular_section,
+                    " ".join(f.types),
+                ],
+            )
+        )
+        for f in obj.fields
+    )
+
+
+def _run_pool(jobs: list, worker, workers: int) -> list:
+    if not jobs:
+        return []
+    if workers <= 1 or len(jobs) < 8:
+        return [worker(j) for j in jobs]
+    out: list = []
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futs = [pool.submit(worker, j) for j in jobs]
+        for fut in as_completed(futs):
+            try:
+                out.append(fut.result())
+            except Exception:
+                out.append(None)
+    return out
+
+
+def build_project_index(
+    configs: list[ConfigPaths],
+    project_root: Path,
+    *,
+    full: bool = False,
+    wipe_db: bool = False,
+    types: set[str] | None = None,
+    skip_types: set[str] | None = None,
+    index_modules: bool = True,
+    workers: int | None = None,
+    write_md: bool = False,
+) -> dict[str, Any]:
+    """Индексирует конфигурации в единый cache/1c_meta/index.sqlite.
+
+    full=True — для каждой указанной конфигурации удалить её данные и пересобрать.
+    wipe_db=True — удалить весь index.sqlite и создать заново (обычно вместе с full
+    по всем конфигам).
+    Иначе — инкремент по mtime/size файлов.
+    """
+    workers = workers if workers is not None else default_workers()
+    skip = set(skip_types if skip_types is not None else DEFAULT_SKIP_TYPES)
+    cache_root = cache_dir(project_root)
+    cache_root.mkdir(parents=True, exist_ok=True)
+    db_path = dbmod.global_db_path(cache_root)
+
+    t0 = time.perf_counter()
+
+    if wipe_db or not db_path.exists():
+        conn = dbmod.reset_db(db_path)
+        mode = "full"
+        force_config_full = True
+    else:
+        try:
+            conn = dbmod.ensure_db(db_path)
+            mode = "full" if full else "incremental"
+            force_config_full = full
+        except RuntimeError:
+            conn = dbmod.reset_db(db_path)
+            mode = "full"
+            force_config_full = True
+
+    summary: dict[str, Any] = {
+        "mode": mode,
+        "db": str(db_path),
+        "workers": workers,
+        "configs": [],
+        "meta_parsed": 0,
+        "meta_skipped": 0,
+        "modules_parsed": 0,
+        "modules_skipped": 0,
+        "files_removed": 0,
+        "errors": 0,
+    }
+
+    try:
+        for cfg in configs:
+            cfg_stats = _index_one_config(
+                conn,
+                cfg,
+                full=force_config_full,
+                types=types,
+                skip_types=skip,
+                index_modules=index_modules,
+                workers=workers,
+                summary=summary,
+            )
+            summary["configs"].append(cfg_stats)
+            if write_md:
+                _export_md_for_config(conn, cfg, cache_root)
+
+        dbmod.set_meta(conn, "last_reindex_at", dbmod.utcnow())
+        dbmod.set_meta(conn, "last_reindex_mode", mode)
+        conn.commit()
+    finally:
+        conn.close()
+
+    summary["elapsed_sec"] = round(time.perf_counter() - t0, 2)
+    writers.write_global_index_md(cache_root, summary)
+    return summary
+
+
+def _index_one_config(
+    conn,
+    cfg: ConfigPaths,
+    *,
+    full: bool,
+    types: set[str] | None,
+    skip_types: set[str],
+    index_modules: bool,
+    workers: int,
+    summary: dict[str, Any],
+) -> dict[str, Any]:
+    if full:
+        dbmod.delete_config(conn, cfg.name)
+
+    existing = dbmod.get_file_index(conn, cfg.name) if not full else {}
+    seen_paths: set[str] = set()
+
+    meta_files = iter_object_xml_files(cfg.src, types=types, skip_types=skip_types)
+    meta_jobs: list[tuple[str, str, str]] = []
+    meta_skipped = 0
+    for otype, path in meta_files:
+        rel = path.relative_to(cfg.src).as_posix()
+        seen_paths.add(rel)
+        mtime_ns, size = _file_stamp(path)
+        prev = existing.get(rel)
+        if (
+            prev
+            and prev["mtime_ns"] == mtime_ns
+            and prev["size"] == size
+            and prev["kind"] == "meta_xml"
+        ):
+            meta_skipped += 1
+            continue
+        if prev:
+            dbmod.delete_file_cascade(conn, prev["id"])
+        meta_jobs.append((cfg.name, otype, str(path)))
+
+    meta_results = _run_pool(meta_jobs, _parse_meta_job, workers)
+    meta_parsed = 0
+    for item in meta_results:
+        if not item:
+            summary["errors"] += 1
+            continue
+        path_s, rec, stamp = item
+        if rec is None:
+            summary["errors"] += 1
+            continue
+        path = Path(path_s)
+        rel = path.relative_to(cfg.src).as_posix()
+        file_id = dbmod.upsert_file(
+            conn,
+            config=cfg.name,
+            rel_path=rel,
+            kind="meta_xml",
+            mtime_ns=stamp[0],
+            size=stamp[1],
+            object_type=rec.object_type,
+        )
+        dbmod.upsert_object(
+            conn, rec.to_dict(), file_id=file_id, fields_text=_fields_text(rec)
+        )
+        meta_parsed += 1
+
+    mod_parsed = 0
+    mod_skipped = 0
+    if index_modules:
+        mod_jobs: list[tuple[str, str, str]] = []
+        for path in iter_bsl_files(cfg.src):
+            rel = path.relative_to(cfg.src).as_posix()
+            seen_paths.add(rel)
+            mtime_ns, size = _file_stamp(path)
+            prev = existing.get(rel)
+            if (
+                prev
+                and prev["mtime_ns"] == mtime_ns
+                and prev["size"] == size
+                and prev["kind"] == "module_bsl"
+            ):
+                mod_skipped += 1
+                continue
+            if prev:
+                dbmod.delete_file_cascade(conn, prev["id"])
+            mod_jobs.append((cfg.name, str(cfg.src), str(path)))
+
+        mod_results = _run_pool(mod_jobs, _parse_module_job, workers)
+        for rec in mod_results:
+            if rec is None:
+                summary["errors"] += 1
+                continue
+            file_id = dbmod.upsert_file(
+                conn,
+                config=cfg.name,
+                rel_path=rec.rel_path,
+                kind="module_bsl",
+                mtime_ns=rec.mtime_ns,
+                size=rec.size,
+            )
+            dbmod.upsert_module(
+                conn,
+                config=cfg.name,
+                rel_path=rec.rel_path,
+                owner=rec.owner,
+                module_kind=rec.module_kind,
+                symbols=rec.symbols,
+                body=rec.body,
+                line_count=rec.line_count,
+                size=rec.size,
+                file_id=file_id,
+            )
+            mod_parsed += 1
+
+    removed = 0
+    if not full:
+        for rel, info in existing.items():
+            if rel in seen_paths:
+                continue
+            if types is not None and info["kind"] == "meta_xml":
+                folder = rel.split("/", 1)[0]
+                otype = OBJECT_TYPE_FOLDERS.get(folder)
+                if otype and otype not in types:
+                    continue
+            if not index_modules and info["kind"] == "module_bsl":
+                continue
+            dbmod.delete_file_cascade(conn, info["id"])
+            removed += 1
+
+    stats = dbmod.refresh_config_stats(conn, cfg.name, str(cfg.src))
+    summary["meta_parsed"] += meta_parsed
+    summary["meta_skipped"] += meta_skipped
+    summary["modules_parsed"] += mod_parsed
+    summary["modules_skipped"] += mod_skipped
+    summary["files_removed"] += removed
+
+    print(
+        f"→ {cfg.name}: meta +{meta_parsed}/skip {meta_skipped}, "
+        f"modules +{mod_parsed}/skip {mod_skipped}, removed {removed}, "
+        f"total objects={stats['objects']} modules={stats['modules']}"
+    )
+    return stats
+
+
+def _export_md_for_config(conn, cfg: ConfigPaths, cache_root: Path) -> None:
+    rows = conn.execute(
+        "SELECT json FROM objects WHERE config=? ORDER BY object_type, name",
+        (cfg.name,),
+    ).fetchall()
+    objects: list[ObjectRec] = []
+    for r in rows:
+        d = json.loads(r["json"])
+        fields = [FieldRec(**f) for f in d.get("fields") or []]
+        objects.append(
+            ObjectRec(
+                config=d["config"],
+                object_type=d["object_type"],
+                name=d["name"],
+                synonym=d.get("synonym") or "",
+                comment=d.get("comment") or "",
+                tooltip=d.get("tooltip") or "",
+                explanation=d.get("explanation") or "",
+                uuid=d.get("uuid") or "",
+                path=d.get("path") or "",
+                fields=fields,
+                owners=d.get("owners") or [],
+                register_records=d.get("register_records") or [],
+                based_on=d.get("based_on") or [],
+                input_by_string=d.get("input_by_string") or [],
+            )
+        )
+    out = cache_root / "md" / cfg.name
+    out.mkdir(parents=True, exist_ok=True)
+    writers.write_objects_markdown(out / "objects.md", objects, title=f"Метаданные: {cfg.name}")
+    writers.write_refs_markdown(out / "refs.md", objects, title=f"Ссылки: {cfg.name}")
+    writers.write_type_summaries(out, objects)
+
+
+def build_config_index(*args, **kwargs):
+    raise RuntimeError(
+        "Устарело: используйте build_project_index() и единый cache/1c_meta/index.sqlite"
+    )
+
+
+def write_global_index(project_root: Path, manifests: list[dict[str, Any]]) -> Path:
+    cache_root = cache_dir(project_root)
+    summary = {
+        "mode": "legacy",
+        "db": str(dbmod.global_db_path(cache_root)),
+        "configs": manifests,
+        "elapsed_sec": 0,
+        "meta_parsed": 0,
+        "modules_parsed": 0,
+    }
+    return writers.write_global_index_md(cache_root, summary)