"""Построение единого индекса проекта (метаданные + модули, full/incremental).""" from __future__ import annotations import json import os import time from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any from .config import DEFAULT_SKIP_TYPES, OBJECT_TYPE_FOLDERS, ConfigPaths, cache_dir from . import db as dbmod from .parse_meta import FieldRec, ObjectRec, iter_object_xml_files, parse_metadata_xml from .parse_modules import ModuleRec, iter_bsl_files, parse_bsl_file from . import writers def default_workers() -> int: return max(4, min(16, (os.cpu_count() or 4))) def _file_stamp(path: Path) -> tuple[int, int]: st = path.stat() mtime_ns = getattr(st, "st_mtime_ns", int(st.st_mtime * 1e9)) return mtime_ns, st.st_size def _parse_meta_job(args: tuple[str, str, str]) -> tuple[str, ObjectRec | None, tuple[int, int]]: config, otype, path_s = args path = Path(path_s) stamp = _file_stamp(path) rec = parse_metadata_xml(path, config=config, object_type=otype) return path_s, rec, stamp def _parse_module_job(args: tuple[str, str, str]) -> ModuleRec | None: config, src_s, path_s = args return parse_bsl_file(Path(path_s), config=config, src_root=Path(src_s)) def _fields_text(obj: ObjectRec) -> str: return " ".join( " ".join( filter( None, [ f.kind, f.name, f.synonym, f.comment, f.tabular_section, " ".join(f.types), ], ) ) for f in obj.fields ) def _run_pool(jobs: list, worker, workers: int) -> list: if not jobs: return [] if workers <= 1 or len(jobs) < 8: return [worker(j) for j in jobs] out: list = [] with ThreadPoolExecutor(max_workers=workers) as pool: futs = [pool.submit(worker, j) for j in jobs] for fut in as_completed(futs): try: out.append(fut.result()) except Exception: out.append(None) return out def build_project_index( configs: list[ConfigPaths], project_root: Path, *, full: bool = False, wipe_db: bool = False, types: set[str] | None = None, skip_types: set[str] | None = None, index_modules: bool = True, workers: int | None = None, write_md: bool = False, ) -> dict[str, Any]: """Индексирует конфигурации в единый cache/1c_meta/index.sqlite. full=True — для каждой указанной конфигурации удалить её данные и пересобрать. wipe_db=True — удалить весь index.sqlite и создать заново (обычно вместе с full по всем конфигам). Иначе — инкремент по mtime/size файлов. """ workers = workers if workers is not None else default_workers() skip = set(skip_types if skip_types is not None else DEFAULT_SKIP_TYPES) cache_root = cache_dir(project_root) cache_root.mkdir(parents=True, exist_ok=True) db_path = dbmod.global_db_path(cache_root) t0 = time.perf_counter() if wipe_db or not db_path.exists(): conn = dbmod.reset_db(db_path) mode = "full" force_config_full = True else: try: conn = dbmod.ensure_db(db_path) mode = "full" if full else "incremental" force_config_full = full except RuntimeError: conn = dbmod.reset_db(db_path) mode = "full" force_config_full = True summary: dict[str, Any] = { "mode": mode, "db": str(db_path), "workers": workers, "configs": [], "meta_parsed": 0, "meta_skipped": 0, "modules_parsed": 0, "modules_skipped": 0, "files_removed": 0, "errors": 0, } try: for cfg in configs: cfg_stats = _index_one_config( conn, cfg, full=force_config_full, types=types, skip_types=skip, index_modules=index_modules, workers=workers, summary=summary, ) summary["configs"].append(cfg_stats) if write_md: _export_md_for_config(conn, cfg, cache_root) dbmod.set_meta(conn, "last_reindex_at", dbmod.utcnow()) dbmod.set_meta(conn, "last_reindex_mode", mode) conn.commit() finally: conn.close() summary["elapsed_sec"] = round(time.perf_counter() - t0, 2) writers.write_global_index_md(cache_root, summary) return summary def _index_one_config( conn, cfg: ConfigPaths, *, full: bool, types: set[str] | None, skip_types: set[str], index_modules: bool, workers: int, summary: dict[str, Any], ) -> dict[str, Any]: if full: dbmod.delete_config(conn, cfg.name) existing = dbmod.get_file_index(conn, cfg.name) if not full else {} seen_paths: set[str] = set() meta_files = iter_object_xml_files( cfg.src, types=types, skip_types=skip_types, source_kind=cfg.source_kind, external_name=cfg.external_name, ) meta_jobs: list[tuple[str, str, str]] = [] meta_skipped = 0 for otype, path in meta_files: rel = path.relative_to(cfg.src).as_posix() seen_paths.add(rel) mtime_ns, size = _file_stamp(path) prev = existing.get(rel) if ( prev and prev["mtime_ns"] == mtime_ns and prev["size"] == size and prev["kind"] == "meta_xml" ): meta_skipped += 1 continue if prev: dbmod.delete_file_cascade(conn, prev["id"]) meta_jobs.append((cfg.name, otype, str(path))) meta_results = _run_pool(meta_jobs, _parse_meta_job, workers) meta_parsed = 0 for item in meta_results: if not item: summary["errors"] += 1 continue path_s, rec, stamp = item if rec is None: summary["errors"] += 1 continue path = Path(path_s) rel = path.relative_to(cfg.src).as_posix() file_id = dbmod.upsert_file( conn, config=cfg.name, rel_path=rel, kind="meta_xml", mtime_ns=stamp[0], size=stamp[1], object_type=rec.object_type, ) dbmod.upsert_object( conn, rec.to_dict(), file_id=file_id, fields_text=_fields_text(rec) ) meta_parsed += 1 mod_parsed = 0 mod_skipped = 0 if index_modules: mod_jobs: list[tuple[str, str, str]] = [] bsl_paths = ( sorted((cfg.src / cfg.external_name).rglob("*.bsl")) if cfg.source_kind == "external_processor" and cfg.external_name else iter_bsl_files(cfg.src) ) for path in bsl_paths: rel = path.relative_to(cfg.src).as_posix() seen_paths.add(rel) mtime_ns, size = _file_stamp(path) prev = existing.get(rel) if ( prev and prev["mtime_ns"] == mtime_ns and prev["size"] == size and prev["kind"] == "module_bsl" ): mod_skipped += 1 continue if prev: dbmod.delete_file_cascade(conn, prev["id"]) mod_jobs.append((cfg.name, str(cfg.src), str(path))) mod_results = _run_pool(mod_jobs, _parse_module_job, workers) for rec in mod_results: if rec is None: summary["errors"] += 1 continue file_id = dbmod.upsert_file( conn, config=cfg.name, rel_path=rec.rel_path, kind="module_bsl", mtime_ns=rec.mtime_ns, size=rec.size, ) dbmod.upsert_module( conn, config=cfg.name, rel_path=rec.rel_path, owner=rec.owner, module_kind=rec.module_kind, symbols=rec.symbols, body=rec.body, line_count=rec.line_count, size=rec.size, file_id=file_id, ) mod_parsed += 1 removed = 0 if not full: for rel, info in existing.items(): if rel in seen_paths: continue if types is not None and info["kind"] == "meta_xml": folder = rel.split("/", 1)[0] otype = OBJECT_TYPE_FOLDERS.get(folder) if otype and otype not in types: continue if not index_modules and info["kind"] == "module_bsl": continue dbmod.delete_file_cascade(conn, info["id"]) removed += 1 stats = dbmod.refresh_config_stats(conn, cfg.name, str(cfg.src)) summary["meta_parsed"] += meta_parsed summary["meta_skipped"] += meta_skipped summary["modules_parsed"] += mod_parsed summary["modules_skipped"] += mod_skipped summary["files_removed"] += removed print( f"→ {cfg.name}: meta +{meta_parsed}/skip {meta_skipped}, " f"modules +{mod_parsed}/skip {mod_skipped}, removed {removed}, " f"total objects={stats['objects']} modules={stats['modules']}" ) return stats def _export_md_for_config(conn, cfg: ConfigPaths, cache_root: Path) -> None: rows = conn.execute( "SELECT json FROM objects WHERE config=? ORDER BY object_type, name", (cfg.name,), ).fetchall() objects: list[ObjectRec] = [] for r in rows: d = json.loads(r["json"]) fields = [FieldRec(**f) for f in d.get("fields") or []] objects.append( ObjectRec( config=d["config"], object_type=d["object_type"], name=d["name"], synonym=d.get("synonym") or "", comment=d.get("comment") or "", tooltip=d.get("tooltip") or "", explanation=d.get("explanation") or "", uuid=d.get("uuid") or "", path=d.get("path") or "", fields=fields, owners=d.get("owners") or [], register_records=d.get("register_records") or [], based_on=d.get("based_on") or [], input_by_string=d.get("input_by_string") or [], ) ) out = cache_root / "md" / cfg.name out.mkdir(parents=True, exist_ok=True) writers.write_objects_markdown(out / "objects.md", objects, title=f"Метаданные: {cfg.name}") writers.write_refs_markdown(out / "refs.md", objects, title=f"Ссылки: {cfg.name}") writers.write_type_summaries(out, objects) def build_config_index(*args, **kwargs): raise RuntimeError( "Устарело: используйте build_project_index() и единый cache/1c_meta/index.sqlite" ) def write_global_index(project_root: Path, manifests: list[dict[str, Any]]) -> Path: cache_root = cache_dir(project_root) summary = { "mode": "legacy", "db": str(dbmod.global_db_path(cache_root)), "configs": manifests, "elapsed_sec": 0, "meta_parsed": 0, "modules_parsed": 0, } return writers.write_global_index_md(cache_root, summary)