mihailkudravcev
91521e6604
Add virtual baseconf support for external data processors so reindex can discover and index them alongside regular src-based configurations, and update docs to use generic examples without workspace-specific names.
361 lines
12 KiB
Python
361 lines
12 KiB
Python
"""Построение единого индекса проекта (метаданные + модули, full/incremental)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .config import DEFAULT_SKIP_TYPES, OBJECT_TYPE_FOLDERS, ConfigPaths, cache_dir
|
|
from . import db as dbmod
|
|
from .parse_meta import FieldRec, ObjectRec, iter_object_xml_files, parse_metadata_xml
|
|
from .parse_modules import ModuleRec, iter_bsl_files, parse_bsl_file
|
|
from . import writers
|
|
|
|
|
|
def default_workers() -> int:
|
|
return max(4, min(16, (os.cpu_count() or 4)))
|
|
|
|
|
|
def _file_stamp(path: Path) -> tuple[int, int]:
|
|
st = path.stat()
|
|
mtime_ns = getattr(st, "st_mtime_ns", int(st.st_mtime * 1e9))
|
|
return mtime_ns, st.st_size
|
|
|
|
|
|
def _parse_meta_job(args: tuple[str, str, str]) -> tuple[str, ObjectRec | None, tuple[int, int]]:
|
|
config, otype, path_s = args
|
|
path = Path(path_s)
|
|
stamp = _file_stamp(path)
|
|
rec = parse_metadata_xml(path, config=config, object_type=otype)
|
|
return path_s, rec, stamp
|
|
|
|
|
|
def _parse_module_job(args: tuple[str, str, str]) -> ModuleRec | None:
|
|
config, src_s, path_s = args
|
|
return parse_bsl_file(Path(path_s), config=config, src_root=Path(src_s))
|
|
|
|
|
|
def _fields_text(obj: ObjectRec) -> str:
|
|
return " ".join(
|
|
" ".join(
|
|
filter(
|
|
None,
|
|
[
|
|
f.kind,
|
|
f.name,
|
|
f.synonym,
|
|
f.comment,
|
|
f.tabular_section,
|
|
" ".join(f.types),
|
|
],
|
|
)
|
|
)
|
|
for f in obj.fields
|
|
)
|
|
|
|
|
|
def _run_pool(jobs: list, worker, workers: int) -> list:
|
|
if not jobs:
|
|
return []
|
|
if workers <= 1 or len(jobs) < 8:
|
|
return [worker(j) for j in jobs]
|
|
out: list = []
|
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
futs = [pool.submit(worker, j) for j in jobs]
|
|
for fut in as_completed(futs):
|
|
try:
|
|
out.append(fut.result())
|
|
except Exception:
|
|
out.append(None)
|
|
return out
|
|
|
|
|
|
def build_project_index(
|
|
configs: list[ConfigPaths],
|
|
project_root: Path,
|
|
*,
|
|
full: bool = False,
|
|
wipe_db: bool = False,
|
|
types: set[str] | None = None,
|
|
skip_types: set[str] | None = None,
|
|
index_modules: bool = True,
|
|
workers: int | None = None,
|
|
write_md: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Индексирует конфигурации в единый cache/1c_meta/index.sqlite.
|
|
|
|
full=True — для каждой указанной конфигурации удалить её данные и пересобрать.
|
|
wipe_db=True — удалить весь index.sqlite и создать заново (обычно вместе с full
|
|
по всем конфигам).
|
|
Иначе — инкремент по mtime/size файлов.
|
|
"""
|
|
workers = workers if workers is not None else default_workers()
|
|
skip = set(skip_types if skip_types is not None else DEFAULT_SKIP_TYPES)
|
|
cache_root = cache_dir(project_root)
|
|
cache_root.mkdir(parents=True, exist_ok=True)
|
|
db_path = dbmod.global_db_path(cache_root)
|
|
|
|
t0 = time.perf_counter()
|
|
|
|
if wipe_db or not db_path.exists():
|
|
conn = dbmod.reset_db(db_path)
|
|
mode = "full"
|
|
force_config_full = True
|
|
else:
|
|
try:
|
|
conn = dbmod.ensure_db(db_path)
|
|
mode = "full" if full else "incremental"
|
|
force_config_full = full
|
|
except RuntimeError:
|
|
conn = dbmod.reset_db(db_path)
|
|
mode = "full"
|
|
force_config_full = True
|
|
|
|
summary: dict[str, Any] = {
|
|
"mode": mode,
|
|
"db": str(db_path),
|
|
"workers": workers,
|
|
"configs": [],
|
|
"meta_parsed": 0,
|
|
"meta_skipped": 0,
|
|
"modules_parsed": 0,
|
|
"modules_skipped": 0,
|
|
"files_removed": 0,
|
|
"errors": 0,
|
|
}
|
|
|
|
try:
|
|
for cfg in configs:
|
|
cfg_stats = _index_one_config(
|
|
conn,
|
|
cfg,
|
|
full=force_config_full,
|
|
types=types,
|
|
skip_types=skip,
|
|
index_modules=index_modules,
|
|
workers=workers,
|
|
summary=summary,
|
|
)
|
|
summary["configs"].append(cfg_stats)
|
|
if write_md:
|
|
_export_md_for_config(conn, cfg, cache_root)
|
|
|
|
dbmod.set_meta(conn, "last_reindex_at", dbmod.utcnow())
|
|
dbmod.set_meta(conn, "last_reindex_mode", mode)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
summary["elapsed_sec"] = round(time.perf_counter() - t0, 2)
|
|
writers.write_global_index_md(cache_root, summary)
|
|
return summary
|
|
|
|
|
|
def _index_one_config(
|
|
conn,
|
|
cfg: ConfigPaths,
|
|
*,
|
|
full: bool,
|
|
types: set[str] | None,
|
|
skip_types: set[str],
|
|
index_modules: bool,
|
|
workers: int,
|
|
summary: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if full:
|
|
dbmod.delete_config(conn, cfg.name)
|
|
|
|
existing = dbmod.get_file_index(conn, cfg.name) if not full else {}
|
|
seen_paths: set[str] = set()
|
|
|
|
meta_files = iter_object_xml_files(
|
|
cfg.src,
|
|
types=types,
|
|
skip_types=skip_types,
|
|
source_kind=cfg.source_kind,
|
|
external_name=cfg.external_name,
|
|
)
|
|
meta_jobs: list[tuple[str, str, str]] = []
|
|
meta_skipped = 0
|
|
for otype, path in meta_files:
|
|
rel = path.relative_to(cfg.src).as_posix()
|
|
seen_paths.add(rel)
|
|
mtime_ns, size = _file_stamp(path)
|
|
prev = existing.get(rel)
|
|
if (
|
|
prev
|
|
and prev["mtime_ns"] == mtime_ns
|
|
and prev["size"] == size
|
|
and prev["kind"] == "meta_xml"
|
|
):
|
|
meta_skipped += 1
|
|
continue
|
|
if prev:
|
|
dbmod.delete_file_cascade(conn, prev["id"])
|
|
meta_jobs.append((cfg.name, otype, str(path)))
|
|
|
|
meta_results = _run_pool(meta_jobs, _parse_meta_job, workers)
|
|
meta_parsed = 0
|
|
for item in meta_results:
|
|
if not item:
|
|
summary["errors"] += 1
|
|
continue
|
|
path_s, rec, stamp = item
|
|
if rec is None:
|
|
summary["errors"] += 1
|
|
continue
|
|
path = Path(path_s)
|
|
rel = path.relative_to(cfg.src).as_posix()
|
|
file_id = dbmod.upsert_file(
|
|
conn,
|
|
config=cfg.name,
|
|
rel_path=rel,
|
|
kind="meta_xml",
|
|
mtime_ns=stamp[0],
|
|
size=stamp[1],
|
|
object_type=rec.object_type,
|
|
)
|
|
dbmod.upsert_object(
|
|
conn, rec.to_dict(), file_id=file_id, fields_text=_fields_text(rec)
|
|
)
|
|
meta_parsed += 1
|
|
|
|
mod_parsed = 0
|
|
mod_skipped = 0
|
|
if index_modules:
|
|
mod_jobs: list[tuple[str, str, str]] = []
|
|
bsl_paths = (
|
|
sorted((cfg.src / cfg.external_name).rglob("*.bsl"))
|
|
if cfg.source_kind == "external_processor" and cfg.external_name
|
|
else iter_bsl_files(cfg.src)
|
|
)
|
|
for path in bsl_paths:
|
|
rel = path.relative_to(cfg.src).as_posix()
|
|
seen_paths.add(rel)
|
|
mtime_ns, size = _file_stamp(path)
|
|
prev = existing.get(rel)
|
|
if (
|
|
prev
|
|
and prev["mtime_ns"] == mtime_ns
|
|
and prev["size"] == size
|
|
and prev["kind"] == "module_bsl"
|
|
):
|
|
mod_skipped += 1
|
|
continue
|
|
if prev:
|
|
dbmod.delete_file_cascade(conn, prev["id"])
|
|
mod_jobs.append((cfg.name, str(cfg.src), str(path)))
|
|
|
|
mod_results = _run_pool(mod_jobs, _parse_module_job, workers)
|
|
for rec in mod_results:
|
|
if rec is None:
|
|
summary["errors"] += 1
|
|
continue
|
|
file_id = dbmod.upsert_file(
|
|
conn,
|
|
config=cfg.name,
|
|
rel_path=rec.rel_path,
|
|
kind="module_bsl",
|
|
mtime_ns=rec.mtime_ns,
|
|
size=rec.size,
|
|
)
|
|
dbmod.upsert_module(
|
|
conn,
|
|
config=cfg.name,
|
|
rel_path=rec.rel_path,
|
|
owner=rec.owner,
|
|
module_kind=rec.module_kind,
|
|
symbols=rec.symbols,
|
|
body=rec.body,
|
|
line_count=rec.line_count,
|
|
size=rec.size,
|
|
file_id=file_id,
|
|
)
|
|
mod_parsed += 1
|
|
|
|
removed = 0
|
|
if not full:
|
|
for rel, info in existing.items():
|
|
if rel in seen_paths:
|
|
continue
|
|
if types is not None and info["kind"] == "meta_xml":
|
|
folder = rel.split("/", 1)[0]
|
|
otype = OBJECT_TYPE_FOLDERS.get(folder)
|
|
if otype and otype not in types:
|
|
continue
|
|
if not index_modules and info["kind"] == "module_bsl":
|
|
continue
|
|
dbmod.delete_file_cascade(conn, info["id"])
|
|
removed += 1
|
|
|
|
stats = dbmod.refresh_config_stats(conn, cfg.name, str(cfg.src))
|
|
summary["meta_parsed"] += meta_parsed
|
|
summary["meta_skipped"] += meta_skipped
|
|
summary["modules_parsed"] += mod_parsed
|
|
summary["modules_skipped"] += mod_skipped
|
|
summary["files_removed"] += removed
|
|
|
|
print(
|
|
f"→ {cfg.name}: meta +{meta_parsed}/skip {meta_skipped}, "
|
|
f"modules +{mod_parsed}/skip {mod_skipped}, removed {removed}, "
|
|
f"total objects={stats['objects']} modules={stats['modules']}"
|
|
)
|
|
return stats
|
|
|
|
|
|
def _export_md_for_config(conn, cfg: ConfigPaths, cache_root: Path) -> None:
|
|
rows = conn.execute(
|
|
"SELECT json FROM objects WHERE config=? ORDER BY object_type, name",
|
|
(cfg.name,),
|
|
).fetchall()
|
|
objects: list[ObjectRec] = []
|
|
for r in rows:
|
|
d = json.loads(r["json"])
|
|
fields = [FieldRec(**f) for f in d.get("fields") or []]
|
|
objects.append(
|
|
ObjectRec(
|
|
config=d["config"],
|
|
object_type=d["object_type"],
|
|
name=d["name"],
|
|
synonym=d.get("synonym") or "",
|
|
comment=d.get("comment") or "",
|
|
tooltip=d.get("tooltip") or "",
|
|
explanation=d.get("explanation") or "",
|
|
uuid=d.get("uuid") or "",
|
|
path=d.get("path") or "",
|
|
fields=fields,
|
|
owners=d.get("owners") or [],
|
|
register_records=d.get("register_records") or [],
|
|
based_on=d.get("based_on") or [],
|
|
input_by_string=d.get("input_by_string") or [],
|
|
)
|
|
)
|
|
out = cache_root / "md" / cfg.name
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
writers.write_objects_markdown(out / "objects.md", objects, title=f"Метаданные: {cfg.name}")
|
|
writers.write_refs_markdown(out / "refs.md", objects, title=f"Ссылки: {cfg.name}")
|
|
writers.write_type_summaries(out, objects)
|
|
|
|
|
|
def build_config_index(*args, **kwargs):
|
|
raise RuntimeError(
|
|
"Устарело: используйте build_project_index() и единый cache/1c_meta/index.sqlite"
|
|
)
|
|
|
|
|
|
def write_global_index(project_root: Path, manifests: list[dict[str, Any]]) -> Path:
|
|
cache_root = cache_dir(project_root)
|
|
summary = {
|
|
"mode": "legacy",
|
|
"db": str(dbmod.global_db_path(cache_root)),
|
|
"configs": manifests,
|
|
"elapsed_sec": 0,
|
|
"meta_parsed": 0,
|
|
"modules_parsed": 0,
|
|
}
|
|
return writers.write_global_index_md(cache_root, summary)
|