"""Commit queued submissions to GitHub; GitHub Actions builds and deploys.""" import fcntl import json import logging import os from pathlib import Path import re import shutil import subprocess import sys import time from app import atomic_json from posts import publish_post log = logging.getLogger("publisher") def entry_date(path): match = re.search(r'^date: "(\d{4}-\d{2}-\d{2})', path.read_text(), re.MULTILINE) if not match: raise ValueError(f"Photo entry has no valid date: {path.name}") return match.group(1) class Publisher: def __init__(self): self.data = Path(os.environ.get("PHOTO_DATA", "/data")) self.work = Path(os.environ.get("PHOTO_WORK", "/work")) self.repo = self.work / "repo" self.remote = os.environ["PHOTO_REPOSITORY"] self.branch = os.environ.get("PHOTO_BRANCH", "master") for path in (self.work, self.data / "submissions", self.data / "status"): path.mkdir(parents=True, exist_ok=True) def git(self, *arguments, directory=None): result = subprocess.run(["git", *arguments], cwd=directory or self.repo, text=True, capture_output=True, timeout=300, check=True) return result.stdout.strip() def state(self, job, status, **extra): atomic_json(self.data / "status" / f"{job}.json", dict(status=status, **extra)) def pending(self): jobs = [] for path in sorted((self.data / "submissions").iterdir()): status_path = self.data / "status" / f"{path.name}.json" status = json.loads(status_path.read_text())["status"] if status_path.exists() else "queued" if status in {"queued", "publishing"}: jobs.append(path.name) return jobs def publish(self): jobs = self.pending() if not jobs: return for job in jobs: self.state(job, "publishing") try: outcomes = {} replaced_photos = {} if not (self.repo / ".git").is_dir(): self.git("clone", "--branch", self.branch, "--single-branch", "--", self.remote, str(self.repo), directory=self.work) self.git("fetch", "origin", self.branch) # Only reset this service-owned disposable checkout, never the user's checkout. self.git("reset", "--hard", f"origin/{self.branch}") self.git("config", "user.name", os.environ.get("PHOTO_GIT_NAME", "Photo publisher")) self.git("config", "user.email", os.environ.get("PHOTO_GIT_EMAIL", "photos@theread.me")) # Prepending oldest first leaves the newest snippet at the top. jobs.sort(key=lambda job: (json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"], job)) for job in jobs: submission = self.data / "submissions" / job manifest = json.loads((submission / "manifest.json").read_text()) if manifest.get("kind") == "post": entry, url, action, identity = publish_post(self.repo, submission, manifest) self.git("add", "--", str(entry)) outcomes[job] = dict(url=url, action=action, post_id=identity) continue if manifest.get("kind") == "snippet": entry = self.repo / "snippets.md" existing = entry.read_text() marker = "" if marker not in existing: raise OSError("snippets.md is missing its upload insertion marker") if f'
' not in existing: entry.write_text(existing.replace(marker, marker + "\n\n" + (submission / "entry.md").read_text(), 1)) images = Path("img/snippets/uploads") / job if manifest["count"]: shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True) self.git("add", "--", str(images)) self.git("add", "--", "snippets.md") continue entry = Path("_art") / f"{manifest['date'][:10]}-photo-{job}.md" images = Path("img/arts/uploads") / job replaces = manifest.get("replaces", "") if re.fullmatch(r"[a-f0-9]{64}", replaces) and replaces != job: for previous in (self.repo / "_art").glob(f"*-photo-{replaces}*.md"): self.git("rm", "--ignore-unmatch", "--", str(previous.relative_to(self.repo))) previous_images = Path("img/arts/uploads") / replaces self.git("rm", "-r", "--ignore-unmatch", "--", str(previous_images)) replaced_photos[job] = replaces (self.repo / entry).parent.mkdir(parents=True, exist_ok=True) entries = submission / "entries" if entries.is_dir(): # Remove the former batch document when a queued submission # is republished by a newer service version. self.git("rm", "--ignore-unmatch", "--", str(entry)) for source in sorted(entries.glob("*.md")): destination = Path("_art") / f"{entry_date(source)}-photo-{job}-{source.stem}.md" shutil.copyfile(source, self.repo / destination) self.git("add", "--", str(destination)) else: shutil.copyfile(submission / "entry.md", self.repo / entry) self.git("add", "--", str(entry)) shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True) self.git("add", "--", str(images)) if self.git("diff", "--cached", "--name-only"): self.git("commit", "-m", f"Publish {len(jobs)} submission(s)") self.git("push", "origin", f"HEAD:refs/heads/{self.branch}") commit = self.git("rev-parse", "HEAD") for job in jobs: self.state(job, "pushed", commit=commit, **outcomes.get(job, {})) if job in replaced_photos: self.state(replaced_photos[job], "superseded", replaced_by=job) # A subsequent revision replaces the old one in the rendered site. for job, outcome in outcomes.items(): job_date = json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"] for status_path in (self.data / "status").glob("*.json"): previous = json.loads(status_path.read_text()) if status_path.stem == job or previous.get("post_id") != outcome["post_id"]: continue old_manifest = self.data / "submissions" / status_path.stem / "manifest.json" if old_manifest.exists() and json.loads(old_manifest.read_text())["date"] < job_date: previous.update(status="superseded", replaced_by=job) atomic_json(status_path, previous) log.info("Pushed %s submission(s) at %s", len(jobs), commit) except (OSError, ValueError, subprocess.SubprocessError) as error: log.error("Git publication failed: %s", error) if isinstance(error, subprocess.CalledProcessError): log.error("%s", error.stderr) for job in jobs: self.state(job, "failed", error=str(error) if isinstance(error, ValueError) else "Git push failed; check publisher logs and retry.") raise def run(self, once=False): with (self.work / "publisher.lock").open("w") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) while True: try: self.publish() except (OSError, ValueError, subprocess.SubprocessError): if once: raise if once: return time.sleep(3) if __name__ == "__main__": logging.basicConfig(level=logging.INFO) Publisher().run(once="--once" in sys.argv)