"""Commit queued submissions to GitHub; GitHub Actions builds and deploys.""" import fcntl import json import logging import os from pathlib import Path import shutil import subprocess import sys import time from app import atomic_json from posts import publish_post log = logging.getLogger("publisher") class Publisher: def __init__(self): self.data = Path(os.environ.get("PHOTO_DATA", "/data")) self.work = Path(os.environ.get("PHOTO_WORK", "/work")) self.repo = self.work / "repo" self.remote = os.environ["PHOTO_REPOSITORY"] self.branch = os.environ.get("PHOTO_BRANCH", "master") for path in (self.work, self.data / "submissions", self.data / "status"): path.mkdir(parents=True, exist_ok=True) def git(self, *arguments, directory=None): result = subprocess.run(["git", *arguments], cwd=directory or self.repo, text=True, capture_output=True, timeout=300, check=True) return result.stdout.strip() def state(self, job, status, **extra): atomic_json(self.data / "status" / f"{job}.json", dict(status=status, **extra)) def pending(self): jobs = [] for path in sorted((self.data / "submissions").iterdir()): status_path = self.data / "status" / f"{path.name}.json" status = json.loads(status_path.read_text())["status"] if status_path.exists() else "queued" if status in {"queued", "publishing"}: jobs.append(path.name) return jobs def publish(self): jobs = self.pending() if not jobs: return for job in jobs: self.state(job, "publishing") try: outcomes = {} if not (self.repo / ".git").is_dir(): self.git("clone", "--branch", self.branch, "--single-branch", "--", self.remote, str(self.repo), directory=self.work) self.git("fetch", "origin", self.branch) # Only reset this service-owned disposable checkout, never the user's checkout. self.git("reset", "--hard", f"origin/{self.branch}") self.git("config", "user.name", os.environ.get("PHOTO_GIT_NAME", "Photo publisher")) self.git("config", "user.email", os.environ.get("PHOTO_GIT_EMAIL", "photos@theread.me")) # Prepending oldest first leaves the newest snippet at the top. jobs.sort(key=lambda job: (json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"], job)) for job in jobs: submission = self.data / "submissions" / job manifest = json.loads((submission / "manifest.json").read_text()) if manifest.get("kind") == "post": entry, url, action, identity = publish_post(self.repo, submission, manifest) self.git("add", "--", str(entry)) outcomes[job] = dict(url=url, action=action, post_id=identity) continue if manifest.get("kind") == "snippet": entry = self.repo / "snippets.md" existing = entry.read_text() marker = "" if marker not in existing: raise OSError("snippets.md is missing its upload insertion marker") if f'
' not in existing: entry.write_text(existing.replace(marker, marker + "\n\n" + (submission / "entry.md").read_text(), 1)) images = Path("img/snippets/uploads") / job if manifest["count"]: shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True) self.git("add", "--", str(images)) self.git("add", "--", "snippets.md") continue entry = Path("_art") / f"{manifest['date'][:10]}-photo-{job}.md" images = Path("img/arts/uploads") / job (self.repo / entry).parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(submission / "entry.md", self.repo / entry) shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True) self.git("add", "--", str(entry), str(images)) if self.git("diff", "--cached", "--name-only"): self.git("commit", "-m", f"Publish {len(jobs)} submission(s)") self.git("push", "origin", f"HEAD:refs/heads/{self.branch}") commit = self.git("rev-parse", "HEAD") for job in jobs: self.state(job, "pushed", commit=commit, **outcomes.get(job, {})) # A subsequent revision replaces the old one in the rendered site. for job, outcome in outcomes.items(): job_date = json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"] for status_path in (self.data / "status").glob("*.json"): previous = json.loads(status_path.read_text()) if status_path.stem == job or previous.get("post_id") != outcome["post_id"]: continue old_manifest = self.data / "submissions" / status_path.stem / "manifest.json" if old_manifest.exists() and json.loads(old_manifest.read_text())["date"] < job_date: previous.update(status="superseded", replaced_by=job) atomic_json(status_path, previous) log.info("Pushed %s submission(s) at %s", len(jobs), commit) except (OSError, ValueError, subprocess.SubprocessError) as error: log.error("Git publication failed: %s", error) if isinstance(error, subprocess.CalledProcessError): log.error("%s", error.stderr) for job in jobs: self.state(job, "failed", error=str(error) if isinstance(error, ValueError) else "Git push failed; check publisher logs and retry.") raise def run(self, once=False): with (self.work / "publisher.lock").open("w") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) while True: try: self.publish() except (OSError, ValueError, subprocess.SubprocessError): if once: raise if once: return time.sleep(3) if __name__ == "__main__": logging.basicConfig(level=logging.INFO) Publisher().run(once="--once" in sys.argv)