166 lines
8.2 KiB
Python
166 lines
8.2 KiB
Python
"""Commit queued submissions to GitHub; GitHub Actions builds and deploys."""
|
|
import fcntl
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from app import atomic_json
|
|
from posts import publish_post
|
|
|
|
log = logging.getLogger("publisher")
|
|
|
|
|
|
def entry_date(path):
|
|
match = re.search(r'^date: "(\d{4}-\d{2}-\d{2})', path.read_text(), re.MULTILINE)
|
|
if not match:
|
|
raise ValueError(f"Photo entry has no valid date: {path.name}")
|
|
return match.group(1)
|
|
|
|
|
|
class Publisher:
|
|
def __init__(self):
|
|
self.data = Path(os.environ.get("PHOTO_DATA", "/data"))
|
|
self.work = Path(os.environ.get("PHOTO_WORK", "/work"))
|
|
self.repo = self.work / "repo"
|
|
self.remote = os.environ["PHOTO_REPOSITORY"]
|
|
self.branch = os.environ.get("PHOTO_BRANCH", "master")
|
|
for path in (self.work, self.data / "submissions", self.data / "status"):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
|
|
def git(self, *arguments, directory=None):
|
|
result = subprocess.run(["git", *arguments], cwd=directory or self.repo,
|
|
text=True, capture_output=True, timeout=300, check=True)
|
|
return result.stdout.strip()
|
|
|
|
def state(self, job, status, **extra):
|
|
atomic_json(self.data / "status" / f"{job}.json", dict(status=status, **extra))
|
|
|
|
def pending(self):
|
|
jobs = []
|
|
for path in sorted((self.data / "submissions").iterdir()):
|
|
status_path = self.data / "status" / f"{path.name}.json"
|
|
status = json.loads(status_path.read_text())["status"] if status_path.exists() else "queued"
|
|
if status in {"queued", "publishing"}:
|
|
jobs.append(path.name)
|
|
return jobs
|
|
|
|
def publish(self):
|
|
jobs = self.pending()
|
|
if not jobs:
|
|
return
|
|
for job in jobs:
|
|
self.state(job, "publishing")
|
|
try:
|
|
outcomes = {}
|
|
replaced_photos = {}
|
|
if not (self.repo / ".git").is_dir():
|
|
self.git("clone", "--branch", self.branch, "--single-branch", "--",
|
|
self.remote, str(self.repo), directory=self.work)
|
|
self.git("fetch", "origin", self.branch)
|
|
# Only reset this service-owned disposable checkout, never the user's checkout.
|
|
self.git("reset", "--hard", f"origin/{self.branch}")
|
|
self.git("config", "user.name", os.environ.get("PHOTO_GIT_NAME", "Photo publisher"))
|
|
self.git("config", "user.email", os.environ.get("PHOTO_GIT_EMAIL", "photos@theread.me"))
|
|
# Prepending oldest first leaves the newest snippet at the top.
|
|
jobs.sort(key=lambda job: (json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"], job))
|
|
for job in jobs:
|
|
submission = self.data / "submissions" / job
|
|
manifest = json.loads((submission / "manifest.json").read_text())
|
|
if manifest.get("kind") == "post":
|
|
entry, url, action, identity = publish_post(self.repo, submission, manifest)
|
|
self.git("add", "--", str(entry))
|
|
outcomes[job] = dict(url=url, action=action, post_id=identity)
|
|
continue
|
|
if manifest.get("kind") == "snippet":
|
|
entry = self.repo / "snippets.md"
|
|
existing = entry.read_text()
|
|
marker = "<!-- uploaded-snippets -->"
|
|
if marker not in existing:
|
|
raise OSError("snippets.md is missing its upload insertion marker")
|
|
if f'<div id="snippet-{job}"></div>' not in existing:
|
|
entry.write_text(existing.replace(marker, marker + "\n\n" +
|
|
(submission / "entry.md").read_text(), 1))
|
|
images = Path("img/snippets/uploads") / job
|
|
if manifest["count"]:
|
|
shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True)
|
|
self.git("add", "--", str(images))
|
|
self.git("add", "--", "snippets.md")
|
|
continue
|
|
entry = Path("_art") / f"{manifest['date'][:10]}-photo-{job}.md"
|
|
images = Path("img/arts/uploads") / job
|
|
replaces = manifest.get("replaces", "")
|
|
if re.fullmatch(r"[a-f0-9]{64}", replaces) and replaces != job:
|
|
for previous in (self.repo / "_art").glob(f"*-photo-{replaces}*.md"):
|
|
self.git("rm", "--ignore-unmatch", "--",
|
|
str(previous.relative_to(self.repo)))
|
|
previous_images = Path("img/arts/uploads") / replaces
|
|
self.git("rm", "-r", "--ignore-unmatch", "--", str(previous_images))
|
|
replaced_photos[job] = replaces
|
|
(self.repo / entry).parent.mkdir(parents=True, exist_ok=True)
|
|
entries = submission / "entries"
|
|
if entries.is_dir():
|
|
# Remove the former batch document when a queued submission
|
|
# is republished by a newer service version.
|
|
self.git("rm", "--ignore-unmatch", "--", str(entry))
|
|
for source in sorted(entries.glob("*.md")):
|
|
destination = Path("_art") / f"{entry_date(source)}-photo-{job}-{source.stem}.md"
|
|
shutil.copyfile(source, self.repo / destination)
|
|
self.git("add", "--", str(destination))
|
|
else:
|
|
shutil.copyfile(submission / "entry.md", self.repo / entry)
|
|
self.git("add", "--", str(entry))
|
|
shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True)
|
|
self.git("add", "--", str(images))
|
|
if self.git("diff", "--cached", "--name-only"):
|
|
self.git("commit", "-m", f"Publish {len(jobs)} submission(s)")
|
|
self.git("push", "origin", f"HEAD:refs/heads/{self.branch}")
|
|
commit = self.git("rev-parse", "HEAD")
|
|
for job in jobs:
|
|
self.state(job, "pushed", commit=commit, **outcomes.get(job, {}))
|
|
if job in replaced_photos:
|
|
self.state(replaced_photos[job], "superseded", replaced_by=job)
|
|
# A subsequent revision replaces the old one in the rendered site.
|
|
for job, outcome in outcomes.items():
|
|
job_date = json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"]
|
|
for status_path in (self.data / "status").glob("*.json"):
|
|
previous = json.loads(status_path.read_text())
|
|
if status_path.stem == job or previous.get("post_id") != outcome["post_id"]:
|
|
continue
|
|
old_manifest = self.data / "submissions" / status_path.stem / "manifest.json"
|
|
if old_manifest.exists() and json.loads(old_manifest.read_text())["date"] < job_date:
|
|
previous.update(status="superseded", replaced_by=job)
|
|
atomic_json(status_path, previous)
|
|
log.info("Pushed %s submission(s) at %s", len(jobs), commit)
|
|
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
|
log.error("Git publication failed: %s", error)
|
|
if isinstance(error, subprocess.CalledProcessError):
|
|
log.error("%s", error.stderr)
|
|
for job in jobs:
|
|
self.state(job, "failed", error=str(error) if isinstance(error, ValueError) else
|
|
"Git push failed; check publisher logs and retry.")
|
|
raise
|
|
|
|
def run(self, once=False):
|
|
with (self.work / "publisher.lock").open("w") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
while True:
|
|
try:
|
|
self.publish()
|
|
except (OSError, ValueError, subprocess.SubprocessError):
|
|
if once:
|
|
raise
|
|
if once:
|
|
return
|
|
time.sleep(3)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO)
|
|
Publisher().run(once="--once" in sys.argv)
|