Integrate Content Editor and Publication workflow
This commit is contained in:
@@ -0,0 +1,522 @@
|
||||
"""Real release workflow: private provenance, immutable candidates and guarded activation.
|
||||
|
||||
Workspace writes still belong to ContentService. This service never edits source files.
|
||||
All cooperating builds/writes/activation use SiteLock; external filesystem writers must
|
||||
honour that lock. Private source snapshots are never copied to public dist.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from ..build.pipeline import (
|
||||
_SKIP_DIRS,
|
||||
BUILD_ID_RE,
|
||||
build_site,
|
||||
hash_tree,
|
||||
render_fingerprint,
|
||||
switch_current,
|
||||
)
|
||||
from ..core.registry import load_site
|
||||
from ..core.routes import build_route_table, route_to_output
|
||||
from ..core.validate import validate_release, validate_workspace
|
||||
from ..errors import ConflictError, EngineError, Report, UnsafePathError
|
||||
from ..parsing.frontmatter import parse
|
||||
from ..services.preview import asset_path
|
||||
from ..storage.fs import atomic_write, safe_join
|
||||
from ..storage.locks import SiteLock
|
||||
|
||||
log = logging.getLogger("uvicorn.error")
|
||||
ROOT_FILES = {"site.yaml", "theme.yaml", "redirects.yaml"}
|
||||
TYPES = {
|
||||
"pages": "Page",
|
||||
"content": "Entry",
|
||||
"menus": "Menu",
|
||||
"assets": "Asset",
|
||||
"layouts": "Layout",
|
||||
"fragments": "Fragment",
|
||||
"collections": "Schema",
|
||||
"display": "Display",
|
||||
"data": "Data",
|
||||
"forms": "Form",
|
||||
}
|
||||
|
||||
|
||||
class ReleaseMissing(EngineError):
|
||||
pass
|
||||
|
||||
|
||||
class CandidateCommitError(EngineError):
|
||||
def __init__(self, build_id):
|
||||
self.build_id = build_id
|
||||
super().__init__(
|
||||
"Кандидат собран, но запись служебных данных не завершена. Live не переключён. Обновите состояние перед повтором."
|
||||
)
|
||||
|
||||
|
||||
class ReleaseCommitError(EngineError):
|
||||
def __init__(self, build_id):
|
||||
self.build_id = build_id
|
||||
super().__init__(
|
||||
"Live переключён, но запись служебных данных не завершена. Обновите состояние перед повтором."
|
||||
)
|
||||
|
||||
|
||||
class PublicationService:
|
||||
def __init__(self, root, dist, state, audit):
|
||||
self.root, self.dist = Path(root), Path(dist)
|
||||
self.directory, self.audit = Path(state) / "publication", audit
|
||||
|
||||
def current(self):
|
||||
path = self.dist / "current"
|
||||
if not path.exists():
|
||||
return None
|
||||
resolved = path.resolve()
|
||||
if resolved.parent != self.dist.resolve() or not BUILD_ID_RE.fullmatch(resolved.name):
|
||||
raise UnsafePathError("Некорректный указатель текущего релиза")
|
||||
return resolved.name
|
||||
|
||||
def folder(self, identity):
|
||||
if not BUILD_ID_RE.fullmatch(identity):
|
||||
raise ReleaseMissing("Недопустимый ID релиза")
|
||||
path = self.dist / identity
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise ReleaseMissing("Релиз отсутствует или уже удалён")
|
||||
return path
|
||||
|
||||
def manifest(self, identity):
|
||||
return json.loads((self.folder(identity) / "manifest.json").read_text())
|
||||
|
||||
def read(self, name, default=None):
|
||||
path = self.directory / name
|
||||
return json.loads(path.read_text()) if path.is_file() else default
|
||||
|
||||
def write(self, name, data):
|
||||
self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
atomic_write(self.directory / name, json.dumps(data, ensure_ascii=False, indent=2))
|
||||
|
||||
def provenance(self, identity):
|
||||
return self.read(f"releases/{identity}.json") if identity else None
|
||||
|
||||
def sources(self):
|
||||
result = {}
|
||||
theme_file = load_site(self.root).site.theme_file
|
||||
root_files = ROOT_FILES | ({theme_file} if theme_file else set())
|
||||
for path in sorted(self.root.rglob("*")):
|
||||
rel = path.relative_to(self.root)
|
||||
if _SKIP_DIRS & set(rel.parts) or any(x.startswith(".") for x in rel.parts):
|
||||
continue
|
||||
if path.is_symlink():
|
||||
raise UnsafePathError(f"Symlink не допускается в исходниках релиза: {rel}")
|
||||
if not path.is_file() or (
|
||||
rel.parts[0] not in TYPES and rel.as_posix() not in root_files
|
||||
):
|
||||
continue
|
||||
raw = path.read_bytes()
|
||||
kind = TYPES.get(rel.parts[0], "Config")
|
||||
title, status, text = rel.name, "configuration", None
|
||||
if kind in {"Page", "Entry"} and path.suffix == ".md":
|
||||
try:
|
||||
meta, _ = parse(raw.decode())
|
||||
title, status = (
|
||||
str(meta.get("title") or path.stem),
|
||||
str(meta.get("status", "draft")),
|
||||
)
|
||||
except (ValueError, UnicodeError):
|
||||
status = "invalid"
|
||||
if len(raw) <= 524288 and path.suffix in {
|
||||
".md",
|
||||
".yaml",
|
||||
".yml",
|
||||
".css",
|
||||
".js",
|
||||
".j2",
|
||||
".txt",
|
||||
}:
|
||||
try:
|
||||
text = raw.decode()
|
||||
except UnicodeError:
|
||||
pass
|
||||
result[rel.as_posix()] = {
|
||||
"hash": hashlib.sha256(raw).hexdigest(),
|
||||
"size": len(raw),
|
||||
"title": title,
|
||||
"status": status,
|
||||
"type": kind,
|
||||
"source": text,
|
||||
}
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def delta(now, before, unknown=False):
|
||||
rows = []
|
||||
for path in sorted(set(now) | set(before)):
|
||||
a, b = now.get(path), before.get(path)
|
||||
if a and b and a["hash"] == b["hash"]:
|
||||
continue
|
||||
row = a or b
|
||||
rows.append(
|
||||
{
|
||||
"path": path,
|
||||
"title": row["title"],
|
||||
"type": row["type"],
|
||||
"status": a["status"] if a else "deleted",
|
||||
"kind": "unknown"
|
||||
if unknown
|
||||
else "new"
|
||||
if not b
|
||||
else "deleted"
|
||||
if not a
|
||||
else "modified",
|
||||
"before": b,
|
||||
"after": a,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
def event(self, event, ctx, **fields):
|
||||
self.audit.write(
|
||||
"release_" + event, actor=ctx.principal.login, ip=ctx.ip, source=ctx.source, **fields
|
||||
)
|
||||
|
||||
def activity(self):
|
||||
path = self.audit.path
|
||||
if not path.is_file():
|
||||
return []
|
||||
result = []
|
||||
# Bounded tail; old events remain in the actual audit log.
|
||||
with path.open("rb") as stream:
|
||||
stream.seek(0, 2)
|
||||
length = stream.tell()
|
||||
stream.seek(max(0, length - 262144))
|
||||
if length > 262144:
|
||||
stream.readline()
|
||||
for line in stream:
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except (ValueError, UnicodeError):
|
||||
continue
|
||||
if item.get("event", "").startswith("release_") or item.get("event") in {
|
||||
"publish",
|
||||
"rollback",
|
||||
}:
|
||||
result.append(
|
||||
{
|
||||
k: item[k]
|
||||
for k in (
|
||||
"ts",
|
||||
"event",
|
||||
"actor",
|
||||
"object",
|
||||
"errors",
|
||||
"warnings",
|
||||
"request_id",
|
||||
)
|
||||
if k in item
|
||||
}
|
||||
)
|
||||
return result[-80:][::-1]
|
||||
|
||||
def state(self):
|
||||
with SiteLock(self.root):
|
||||
return self._state()
|
||||
|
||||
def _state(self):
|
||||
live, current_sources = self.current(), self.sources()
|
||||
baseline = self.provenance(live)
|
||||
comparison = not live or baseline is not None
|
||||
changes = self.delta(current_sources, (baseline or {}).get("sources", {}), not comparison)
|
||||
candidate = self.read("candidate.json")
|
||||
fingerprint = hash_tree(self.root)
|
||||
renderer = render_fingerprint(load_site(self.root))
|
||||
if candidate:
|
||||
candidate = dict(candidate)
|
||||
candidate["fresh"] = (
|
||||
candidate["source_hash"] == fingerprint
|
||||
and candidate["render_hash"] == renderer
|
||||
and candidate["base"] == live
|
||||
)
|
||||
if not (self.dist / candidate["id"]).is_dir():
|
||||
candidate["fresh"] = False
|
||||
validation = self.read("validation.json")
|
||||
if validation:
|
||||
validation["fresh"] = (
|
||||
validation["source_hash"] == fingerprint and validation["render_hash"] == renderer
|
||||
)
|
||||
history = []
|
||||
if self.dist.is_dir():
|
||||
for p in sorted(self.dist.iterdir(), reverse=True):
|
||||
if BUILD_ID_RE.fullmatch(p.name) and p.is_dir() and not p.is_symlink():
|
||||
m, provenance = self.manifest(p.name), self.provenance(p.name)
|
||||
history.append(
|
||||
{
|
||||
"id": p.name,
|
||||
"current": p.name == live,
|
||||
"built_at": m.get("built_at"),
|
||||
"pages": len(m.get("routes", [])),
|
||||
"routes": m.get("routes", []),
|
||||
"forms": self.forms(p.name),
|
||||
"author": (provenance or {}).get("author"),
|
||||
"comment": (provenance or {}).get("comment", ""),
|
||||
"candidate": provenance is not None
|
||||
and not provenance.get("published_at")
|
||||
and p.name != live,
|
||||
"comparison_available": provenance is not None,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"current": live,
|
||||
"changes": changes,
|
||||
"comparison_available": comparison,
|
||||
"source_hash": fingerprint,
|
||||
"candidate": candidate,
|
||||
"validation": validation,
|
||||
"history": history,
|
||||
"builds": self.read("builds.json", []),
|
||||
"activity": self.activity(),
|
||||
}
|
||||
|
||||
def forms(self, identity):
|
||||
folder = self.folder(identity) / "_forms"
|
||||
return sorted(p.stem for p in folder.glob("*.json"))
|
||||
|
||||
def validate(self, ctx):
|
||||
with SiteLock(self.root):
|
||||
reg, report = load_site(self.root), Report()
|
||||
table = build_route_table(reg, report)
|
||||
report.extend(validate_workspace(reg, table))
|
||||
report.extend(validate_release(reg, table))
|
||||
data = {
|
||||
"source_hash": hash_tree(self.root),
|
||||
"render_hash": render_fingerprint(reg),
|
||||
"at": dt.datetime.now(dt.UTC).isoformat(),
|
||||
"issues": [i.to_dict() for i in report.issues],
|
||||
"errors": len(report.errors),
|
||||
"warnings": len(report.warnings),
|
||||
}
|
||||
self.write("validation.json", data)
|
||||
self.event("validate", ctx, errors=data["errors"], warnings=data["warnings"])
|
||||
return data
|
||||
|
||||
def build(self, ctx, request_id=""):
|
||||
with SiteLock(self.root):
|
||||
started, base = dt.datetime.now(dt.UTC).isoformat(), self.current()
|
||||
builds = self.read("builds.json", [])
|
||||
completed = None
|
||||
try:
|
||||
sources = self.sources()
|
||||
result = build_site(self.root, self.dist, activate=False)
|
||||
completed = result.build_id
|
||||
manifest = self.manifest(result.build_id)
|
||||
candidate = {
|
||||
"id": result.build_id,
|
||||
"base": base,
|
||||
"source_hash": result.source_hash,
|
||||
"render_hash": manifest["render_hash"],
|
||||
"at": started,
|
||||
"author": ctx.principal.login,
|
||||
"pages": result.pages,
|
||||
"output_hash": hash_tree(result.output_dir),
|
||||
"routes": manifest["routes"],
|
||||
"issues": [i.to_dict() for i in result.report.issues],
|
||||
}
|
||||
prior = self.provenance(base)
|
||||
changes = self.delta(
|
||||
sources, (prior or {}).get("sources", {}), bool(base and prior is None)
|
||||
)
|
||||
candidate["changes"] = len(changes)
|
||||
provenance = self.provenance(result.build_id)
|
||||
# Identical build within one second can reuse an existing immutable release.
|
||||
if provenance is None:
|
||||
self.write(
|
||||
f"releases/{result.build_id}.json", {"sources": sources, **candidate}
|
||||
)
|
||||
self.write("candidate.json", candidate)
|
||||
builds.insert(
|
||||
0,
|
||||
{
|
||||
"id": result.build_id,
|
||||
"at": started,
|
||||
"base": base,
|
||||
"status": "ready",
|
||||
"pages": result.pages,
|
||||
"request_id": request_id,
|
||||
"log": f"validate: OK\nrender: {result.pages} страниц\nsnapshot: {result.build_id}\nlive: не переключён",
|
||||
},
|
||||
)
|
||||
self.write("builds.json", builds[:40])
|
||||
self.event("build", ctx, object=result.build_id, request_id=request_id)
|
||||
return candidate
|
||||
except Exception as exc:
|
||||
if completed is not None:
|
||||
raise CandidateCommitError(completed) from exc
|
||||
# Keep previous candidate and live intact; diagnostics are real, no simulated progress.
|
||||
issues = (
|
||||
[i.to_dict() for i in exc.report.issues] if isinstance(exc, EngineError) else []
|
||||
)
|
||||
builds.insert(
|
||||
0,
|
||||
{
|
||||
"id": "",
|
||||
"at": started,
|
||||
"base": base,
|
||||
"status": "failed",
|
||||
"request_id": request_id,
|
||||
"issues": issues,
|
||||
"log": "Сборка не завершена. Live не переключён.\n"
|
||||
+ (str(exc) if issues else "См. серверный traceback по ID запроса."),
|
||||
},
|
||||
)
|
||||
try:
|
||||
self.write("builds.json", builds[:40])
|
||||
self.event("failure", ctx, request_id=request_id)
|
||||
except OSError:
|
||||
log.exception("release_failure_record_failed request_id=%s", request_id)
|
||||
raise
|
||||
|
||||
def activate(self, identity, expected_live, comment, ctx, *, rollback=False):
|
||||
with SiteLock(self.root):
|
||||
actual = self.current()
|
||||
if actual != expected_live:
|
||||
raise ConflictError(
|
||||
"Live изменился в другой вкладке. Обновите данные перед подтверждением.",
|
||||
actual or "absent",
|
||||
)
|
||||
manifest = self.manifest(identity)
|
||||
provenance = self.provenance(identity)
|
||||
if (
|
||||
rollback
|
||||
and provenance
|
||||
and not provenance.get("published_at")
|
||||
and identity != actual
|
||||
):
|
||||
raise EngineError("Неопубликованный кандидат нельзя выбрать через откат")
|
||||
if not rollback:
|
||||
candidate = self.read("candidate.json")
|
||||
if not candidate or candidate["id"] != identity:
|
||||
raise ConflictError("Кандидат заменён. Проверьте актуальную сборку.", identity)
|
||||
if (
|
||||
candidate["base"] != actual
|
||||
or candidate["source_hash"] != hash_tree(self.root)
|
||||
or candidate["render_hash"] != render_fingerprint(load_site(self.root))
|
||||
):
|
||||
raise ConflictError(
|
||||
"Кандидат устарел. Проверьте и пересоберите workspace.", identity
|
||||
)
|
||||
if not comment.strip():
|
||||
raise ValueError("Добавьте комментарий к релизу")
|
||||
if manifest.get("build_id") != identity:
|
||||
raise EngineError("Manifest не соответствует выбранному релизу")
|
||||
if provenance and provenance.get("output_hash") != hash_tree(self.folder(identity)):
|
||||
raise EngineError("Файлы снимка изменились после сборки. Создайте новый кандидат.")
|
||||
switch_current(self.dist, identity)
|
||||
try:
|
||||
if provenance:
|
||||
provenance.update(published_at=dt.datetime.now(dt.UTC).isoformat())
|
||||
if not rollback:
|
||||
provenance.update(comment=comment.strip(), author=ctx.principal.login)
|
||||
self.write(f"releases/{identity}.json", provenance)
|
||||
if (self.read("candidate.json") or {}).get("id") == identity:
|
||||
self.write("candidate.json", None)
|
||||
self.event("rollback" if rollback else "publish", ctx, object=identity)
|
||||
except OSError as exc:
|
||||
raise ReleaseCommitError(identity) from exc
|
||||
return {"current": identity, "committed": True}
|
||||
|
||||
def release_details(self, identity):
|
||||
with SiteLock(self.root):
|
||||
live = self.current()
|
||||
target = self.folder(identity)
|
||||
base = self.folder(live) if live else None
|
||||
|
||||
def outputs(folder):
|
||||
if folder is None:
|
||||
return {}
|
||||
result = {}
|
||||
for p in folder.rglob("*"):
|
||||
if p.is_file():
|
||||
rel = p.relative_to(folder).as_posix()
|
||||
result[rel] = hashlib.sha256(
|
||||
asset_path(folder, rel).read_bytes()
|
||||
).hexdigest()
|
||||
return result
|
||||
|
||||
before, after = outputs(base), outputs(target)
|
||||
changes = [
|
||||
{
|
||||
"path": p,
|
||||
"kind": "new"
|
||||
if p not in before
|
||||
else "deleted"
|
||||
if p not in after
|
||||
else "modified",
|
||||
}
|
||||
for p in sorted(before.keys() | after.keys())
|
||||
if before.get(p) != after.get(p)
|
||||
]
|
||||
return {"id": identity, "base": live, "changes": changes}
|
||||
|
||||
def candidate_diff(self):
|
||||
with SiteLock(self.root):
|
||||
candidate = self.read("candidate.json")
|
||||
if not candidate:
|
||||
raise ReleaseMissing("Кандидат пока не создан")
|
||||
snapshot = self.provenance(candidate["id"])
|
||||
base = self.provenance(candidate["base"])
|
||||
available = not candidate["base"] or base is not None
|
||||
return {
|
||||
"id": candidate["id"],
|
||||
"base": candidate["base"],
|
||||
"comparison_available": available,
|
||||
"changes": self.delta(
|
||||
(snapshot or {}).get("sources", {}),
|
||||
(base or {}).get("sources", {}),
|
||||
not available,
|
||||
),
|
||||
}
|
||||
|
||||
def compare(self, path):
|
||||
safe_join(self.root, path)
|
||||
with SiteLock(self.root):
|
||||
live = self.current()
|
||||
baseline = self.provenance(live)
|
||||
now = self.sources().get(path)
|
||||
before = (baseline or {}).get("sources", {}).get(path)
|
||||
return {
|
||||
"path": path,
|
||||
"current": live,
|
||||
"comparison_available": not live or baseline is not None,
|
||||
"before": before,
|
||||
"after": now,
|
||||
}
|
||||
|
||||
def document_history(self, path):
|
||||
safe_join(self.root, path)
|
||||
if path.split("/", 1)[0] not in TYPES or any(p.startswith(".") for p in Path(path).parts):
|
||||
raise UnsafePathError("История недоступна для этого пути")
|
||||
folder = safe_join(self.root, ".engine/history/" + path)
|
||||
versions = []
|
||||
for p in sorted(folder.iterdir(), reverse=True) if folder.is_dir() else []:
|
||||
if p.is_file() and not p.is_symlink():
|
||||
versions.append({"id": p.name, "size": p.stat().st_size})
|
||||
return {"path": path, "versions": versions[:20]}
|
||||
|
||||
def preview(self, identity, route, previews, principal):
|
||||
with SiteLock(self.root):
|
||||
manifest, folder = self.manifest(identity), self.folder(identity)
|
||||
if route not in manifest.get("routes", []):
|
||||
raise ReleaseMissing("Маршрут отсутствует в этом снимке")
|
||||
html = asset_path(folder, route_to_output(route)).read_text()
|
||||
return previews.store(
|
||||
SimpleNamespace(root=folder),
|
||||
html,
|
||||
0,
|
||||
principal,
|
||||
f"release:{identity}:{route}",
|
||||
passive=True,
|
||||
)
|
||||
Reference in New Issue
Block a user