Детализирована документация. Небольшие изменения имен параметров.
This commit is contained in:
mihailkudravcev
+520
@@ -0,0 +1,520 @@
|
||||
"""Единый SQLite-индекс проекта (все конфигурации 1С)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
SCHEMA_VERSION = 2
|
||||
|
||||
DDL = """
|
||||
PRAGMA journal_mode=WAL;
|
||||
PRAGMA synchronous=NORMAL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS configs (
|
||||
name TEXT PRIMARY KEY,
|
||||
src TEXT NOT NULL,
|
||||
objects INTEGER NOT NULL DEFAULT 0,
|
||||
fields INTEGER NOT NULL DEFAULT 0,
|
||||
refs INTEGER NOT NULL DEFAULT 0,
|
||||
modules INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
id INTEGER PRIMARY KEY,
|
||||
config TEXT NOT NULL,
|
||||
rel_path TEXT NOT NULL,
|
||||
kind TEXT NOT NULL, -- meta_xml | module_bsl
|
||||
mtime_ns INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
sha1 TEXT,
|
||||
object_type TEXT,
|
||||
UNIQUE(config, rel_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS objects (
|
||||
id INTEGER PRIMARY KEY,
|
||||
config TEXT NOT NULL,
|
||||
object_type TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
full_name TEXT NOT NULL,
|
||||
synonym TEXT,
|
||||
comment TEXT,
|
||||
tooltip TEXT,
|
||||
explanation TEXT,
|
||||
uuid TEXT,
|
||||
path TEXT,
|
||||
file_id INTEGER,
|
||||
json TEXT NOT NULL,
|
||||
UNIQUE(config, full_name),
|
||||
FOREIGN KEY(file_id) REFERENCES files(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fields (
|
||||
id INTEGER PRIMARY KEY,
|
||||
object_id INTEGER NOT NULL,
|
||||
config TEXT NOT NULL,
|
||||
kind TEXT,
|
||||
name TEXT,
|
||||
synonym TEXT,
|
||||
comment TEXT,
|
||||
types TEXT,
|
||||
tabular_section TEXT,
|
||||
refs_json TEXT,
|
||||
FOREIGN KEY(object_id) REFERENCES objects(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS refs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
config TEXT NOT NULL,
|
||||
from_full_name TEXT NOT NULL,
|
||||
from_field TEXT NOT NULL,
|
||||
to_kind TEXT NOT NULL,
|
||||
to_name TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS modules (
|
||||
id INTEGER PRIMARY KEY,
|
||||
config TEXT NOT NULL,
|
||||
rel_path TEXT NOT NULL,
|
||||
owner TEXT,
|
||||
module_kind TEXT,
|
||||
symbols TEXT,
|
||||
line_count INTEGER,
|
||||
size INTEGER,
|
||||
file_id INTEGER,
|
||||
UNIQUE(config, rel_path),
|
||||
FOREIGN KEY(file_id) REFERENCES files(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS objects_fts USING fts5(
|
||||
full_name,
|
||||
synonym,
|
||||
comment,
|
||||
fields_text,
|
||||
object_id UNINDEXED,
|
||||
config UNINDEXED,
|
||||
tokenize='unicode61'
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS modules_fts USING fts5(
|
||||
rel_path,
|
||||
owner,
|
||||
module_kind,
|
||||
symbols,
|
||||
body,
|
||||
module_id UNINDEXED,
|
||||
config UNINDEXED,
|
||||
tokenize='unicode61'
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_objects_config_type ON objects(config, object_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_objects_name ON objects(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_fields_object ON fields(object_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_refs_to ON refs(to_kind, to_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_refs_from ON refs(from_full_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_refs_config ON refs(config);
|
||||
CREATE INDEX IF NOT EXISTS idx_modules_config ON modules(config);
|
||||
CREATE INDEX IF NOT EXISTS idx_modules_owner ON modules(owner);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_config ON files(config);
|
||||
"""
|
||||
|
||||
|
||||
def utcnow() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def global_db_path(cache_root: Path) -> Path:
|
||||
return cache_root / "index.sqlite"
|
||||
|
||||
|
||||
def connect(db_path: Path, *, readonly: bool = False) -> sqlite3.Connection:
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if readonly and db_path.is_file():
|
||||
uri = f"file:{db_path.resolve()}?mode=ro"
|
||||
conn = sqlite3.connect(uri, uri=True)
|
||||
else:
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(DDL)
|
||||
row = conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone()
|
||||
if row is None:
|
||||
conn.execute(
|
||||
"INSERT INTO meta(key, value) VALUES('schema_version', ?)",
|
||||
(str(SCHEMA_VERSION),),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO meta(key, value) VALUES('created_at', ?)",
|
||||
(utcnow(),),
|
||||
)
|
||||
elif int(row["value"]) != SCHEMA_VERSION:
|
||||
raise RuntimeError(
|
||||
f"Несовместимая схема индекса (есть {row['value']}, нужна {SCHEMA_VERSION}). "
|
||||
"Запустите: python tools/index/index_1c.py reindex --full"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def reset_db(db_path: Path) -> sqlite3.Connection:
|
||||
if db_path.exists():
|
||||
db_path.unlink()
|
||||
for suffix in ("-wal", "-shm"):
|
||||
p = Path(str(db_path) + suffix)
|
||||
if p.exists():
|
||||
p.unlink()
|
||||
conn = connect(db_path)
|
||||
init_db(conn)
|
||||
return conn
|
||||
|
||||
|
||||
def ensure_db(db_path: Path) -> sqlite3.Connection:
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
init_db(conn)
|
||||
except RuntimeError:
|
||||
conn.close()
|
||||
raise
|
||||
return conn
|
||||
|
||||
|
||||
def get_file_index(conn: sqlite3.Connection, config: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
"""rel_path → {mtime_ns, size, id, kind} для инкремента."""
|
||||
if config:
|
||||
rows = conn.execute(
|
||||
"SELECT id, rel_path, kind, mtime_ns, size FROM files WHERE config=?",
|
||||
(config,),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT id, config, rel_path, kind, mtime_ns, size FROM files"
|
||||
).fetchall()
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for r in rows:
|
||||
key = r["rel_path"] if config else f"{r['config']}:{r['rel_path']}"
|
||||
out[key] = {
|
||||
"id": r["id"],
|
||||
"kind": r["kind"],
|
||||
"mtime_ns": r["mtime_ns"],
|
||||
"size": r["size"],
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def delete_file_cascade(conn: sqlite3.Connection, file_id: int) -> None:
|
||||
"""Удалить файл и связанные объекты/модули/FTS."""
|
||||
obj = conn.execute(
|
||||
"SELECT id FROM objects WHERE file_id=?", (file_id,)
|
||||
).fetchone()
|
||||
if obj:
|
||||
oid = obj["id"]
|
||||
conn.execute("DELETE FROM objects_fts WHERE object_id=?", (oid,))
|
||||
conn.execute("DELETE FROM fields WHERE object_id=?", (oid,))
|
||||
row = conn.execute(
|
||||
"SELECT config, full_name FROM objects WHERE id=?", (oid,)
|
||||
).fetchone()
|
||||
if row:
|
||||
conn.execute(
|
||||
"DELETE FROM refs WHERE config=? AND from_full_name=?",
|
||||
(row["config"], row["full_name"]),
|
||||
)
|
||||
conn.execute("DELETE FROM objects WHERE id=?", (oid,))
|
||||
|
||||
mod = conn.execute(
|
||||
"SELECT id FROM modules WHERE file_id=?", (file_id,)
|
||||
).fetchone()
|
||||
if mod:
|
||||
mid = mod["id"]
|
||||
conn.execute("DELETE FROM modules_fts WHERE module_id=?", (mid,))
|
||||
conn.execute("DELETE FROM modules WHERE id=?", (mid,))
|
||||
|
||||
conn.execute("DELETE FROM files WHERE id=?", (file_id,))
|
||||
|
||||
|
||||
def delete_config(conn: sqlite3.Connection, config: str) -> None:
|
||||
oids = [
|
||||
r["id"]
|
||||
for r in conn.execute("SELECT id FROM objects WHERE config=?", (config,))
|
||||
]
|
||||
for oid in oids:
|
||||
conn.execute("DELETE FROM objects_fts WHERE object_id=?", (oid,))
|
||||
mids = [
|
||||
r["id"]
|
||||
for r in conn.execute("SELECT id FROM modules WHERE config=?", (config,))
|
||||
]
|
||||
for mid in mids:
|
||||
conn.execute("DELETE FROM modules_fts WHERE module_id=?", (mid,))
|
||||
conn.execute("DELETE FROM fields WHERE config=?", (config,))
|
||||
conn.execute("DELETE FROM refs WHERE config=?", (config,))
|
||||
conn.execute("DELETE FROM objects WHERE config=?", (config,))
|
||||
conn.execute("DELETE FROM modules WHERE config=?", (config,))
|
||||
conn.execute("DELETE FROM files WHERE config=?", (config,))
|
||||
conn.execute("DELETE FROM configs WHERE name=?", (config,))
|
||||
|
||||
|
||||
def upsert_file(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
config: str,
|
||||
rel_path: str,
|
||||
kind: str,
|
||||
mtime_ns: int,
|
||||
size: int,
|
||||
object_type: str | None = None,
|
||||
sha1: str | None = None,
|
||||
) -> int:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO files(config, rel_path, kind, mtime_ns, size, sha1, object_type)
|
||||
VALUES(?,?,?,?,?,?,?)
|
||||
ON CONFLICT(config, rel_path) DO UPDATE SET
|
||||
kind=excluded.kind,
|
||||
mtime_ns=excluded.mtime_ns,
|
||||
size=excluded.size,
|
||||
sha1=excluded.sha1,
|
||||
object_type=excluded.object_type
|
||||
""",
|
||||
(config, rel_path, kind, mtime_ns, size, sha1, object_type),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT id FROM files WHERE config=? AND rel_path=?",
|
||||
(config, rel_path),
|
||||
).fetchone()
|
||||
assert row is not None
|
||||
return int(row["id"])
|
||||
|
||||
|
||||
def upsert_object(
|
||||
conn: sqlite3.Connection,
|
||||
obj: dict[str, Any],
|
||||
*,
|
||||
file_id: int,
|
||||
fields_text: str,
|
||||
) -> int:
|
||||
config = obj["config"]
|
||||
full_name = obj["full_name"]
|
||||
old = conn.execute(
|
||||
"SELECT id FROM objects WHERE config=? AND full_name=?",
|
||||
(config, full_name),
|
||||
).fetchone()
|
||||
if old:
|
||||
oid = int(old["id"])
|
||||
conn.execute("DELETE FROM objects_fts WHERE object_id=?", (oid,))
|
||||
conn.execute("DELETE FROM fields WHERE object_id=?", (oid,))
|
||||
conn.execute(
|
||||
"DELETE FROM refs WHERE config=? AND from_full_name=?",
|
||||
(config, full_name),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE objects SET object_type=?, name=?, synonym=?, comment=?, tooltip=?,
|
||||
explanation=?, uuid=?, path=?, file_id=?, json=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
obj["object_type"],
|
||||
obj["name"],
|
||||
obj.get("synonym"),
|
||||
obj.get("comment"),
|
||||
obj.get("tooltip"),
|
||||
obj.get("explanation"),
|
||||
obj.get("uuid"),
|
||||
obj.get("path"),
|
||||
file_id,
|
||||
json.dumps(obj, ensure_ascii=False),
|
||||
oid,
|
||||
),
|
||||
)
|
||||
else:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO objects(config, object_type, name, full_name, synonym, comment,
|
||||
tooltip, explanation, uuid, path, file_id, json)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
""",
|
||||
(
|
||||
config,
|
||||
obj["object_type"],
|
||||
obj["name"],
|
||||
full_name,
|
||||
obj.get("synonym"),
|
||||
obj.get("comment"),
|
||||
obj.get("tooltip"),
|
||||
obj.get("explanation"),
|
||||
obj.get("uuid"),
|
||||
obj.get("path"),
|
||||
file_id,
|
||||
json.dumps(obj, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
oid = int(cur.lastrowid)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO objects_fts(full_name, synonym, comment, fields_text, object_id, config)
|
||||
VALUES(?,?,?,?,?,?)
|
||||
""",
|
||||
(
|
||||
full_name,
|
||||
obj.get("synonym") or "",
|
||||
obj.get("comment") or "",
|
||||
fields_text,
|
||||
oid,
|
||||
config,
|
||||
),
|
||||
)
|
||||
|
||||
for f in obj.get("fields") or []:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO fields(object_id, config, kind, name, synonym, comment, types,
|
||||
tabular_section, refs_json)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)
|
||||
""",
|
||||
(
|
||||
oid,
|
||||
config,
|
||||
f.get("kind"),
|
||||
f.get("name"),
|
||||
f.get("synonym"),
|
||||
f.get("comment"),
|
||||
", ".join(f.get("types") or []),
|
||||
f.get("tabular_section") or "",
|
||||
json.dumps(f.get("refs") or [], ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
for r in f.get("refs") or []:
|
||||
loc = f"{f['tabular_section']}." if f.get("tabular_section") else ""
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO refs(config, from_full_name, from_field, to_kind, to_name)
|
||||
VALUES(?,?,?,?,?)
|
||||
""",
|
||||
(
|
||||
config,
|
||||
full_name,
|
||||
f"{loc}{f.get('name')}",
|
||||
r.get("kind"),
|
||||
r.get("name"),
|
||||
),
|
||||
)
|
||||
return oid
|
||||
|
||||
|
||||
def upsert_module(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
config: str,
|
||||
rel_path: str,
|
||||
owner: str,
|
||||
module_kind: str,
|
||||
symbols: list[str],
|
||||
body: str,
|
||||
line_count: int,
|
||||
size: int,
|
||||
file_id: int,
|
||||
) -> int:
|
||||
sym_text = "\n".join(symbols)
|
||||
old = conn.execute(
|
||||
"SELECT id FROM modules WHERE config=? AND rel_path=?",
|
||||
(config, rel_path),
|
||||
).fetchone()
|
||||
if old:
|
||||
mid = int(old["id"])
|
||||
conn.execute("DELETE FROM modules_fts WHERE module_id=?", (mid,))
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE modules SET owner=?, module_kind=?, symbols=?, line_count=?, size=?, file_id=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(owner, module_kind, sym_text, line_count, size, file_id, mid),
|
||||
)
|
||||
else:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO modules(config, rel_path, owner, module_kind, symbols, line_count, size, file_id)
|
||||
VALUES(?,?,?,?,?,?,?,?)
|
||||
""",
|
||||
(config, rel_path, owner, module_kind, sym_text, line_count, size, file_id),
|
||||
)
|
||||
mid = int(cur.lastrowid)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO modules_fts(rel_path, owner, module_kind, symbols, body, module_id, config)
|
||||
VALUES(?,?,?,?,?,?,?)
|
||||
""",
|
||||
(rel_path, owner, module_kind, sym_text, body, mid, config),
|
||||
)
|
||||
return mid
|
||||
|
||||
|
||||
def refresh_config_stats(conn: sqlite3.Connection, config: str, src: str) -> dict[str, Any]:
|
||||
objects = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM objects WHERE config=?", (config,)
|
||||
).fetchone()["n"]
|
||||
fields = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM fields WHERE config=?", (config,)
|
||||
).fetchone()["n"]
|
||||
refs = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM refs WHERE config=?", (config,)
|
||||
).fetchone()["n"]
|
||||
modules = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM modules WHERE config=?", (config,)
|
||||
).fetchone()["n"]
|
||||
stats = {
|
||||
"name": config,
|
||||
"src": src,
|
||||
"objects": objects,
|
||||
"fields": fields,
|
||||
"refs": refs,
|
||||
"modules": modules,
|
||||
"updated_at": utcnow(),
|
||||
}
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO configs(name, src, objects, fields, refs, modules, updated_at)
|
||||
VALUES(?,?,?,?,?,?,?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
src=excluded.src,
|
||||
objects=excluded.objects,
|
||||
fields=excluded.fields,
|
||||
refs=excluded.refs,
|
||||
modules=excluded.modules,
|
||||
updated_at=excluded.updated_at
|
||||
""",
|
||||
(
|
||||
stats["name"],
|
||||
stats["src"],
|
||||
stats["objects"],
|
||||
stats["fields"],
|
||||
stats["refs"],
|
||||
stats["modules"],
|
||||
stats["updated_at"],
|
||||
),
|
||||
)
|
||||
return stats
|
||||
|
||||
|
||||
def list_configs(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||
return [dict(r) for r in conn.execute("SELECT * FROM configs ORDER BY name")]
|
||||
|
||||
|
||||
def set_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO meta(key, value) VALUES(?, ?)",
|
||||
(key, value),
|
||||
)
|
||||
Reference in New Issue
Block a user