diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..4d59ad6 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,44 @@ +name: Build and publish blog + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: blog-production + cancel-in-progress: false + +jobs: + publish: + if: github.ref == 'refs/heads/master' + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Test deployment scripts + run: python3 -m unittest discover -s scripts/tests -v + - uses: ruby/setup-ruby@a0102e0972be65f351c307e2d64b9314a57c8073 # v1 + with: + ruby-version: '3.1.2' + bundler-cache: true + - name: Build Jekyll + env: + JEKYLL_ENV: production + run: bundle exec jekyll build --trace + - name: Record deployed photos + run: python3 scripts/photo-manifest.py + - name: Deploy to nginx server + env: + DEPLOY_HOST: ${{ vars.DEPLOY_HOST || 'theread.me' }} + DEPLOY_USER: ${{ vars.DEPLOY_USER || 'mahdi' }} + DEPLOY_PORT: ${{ vars.DEPLOY_PORT || '22' }} + DEPLOY_PATH: ${{ vars.DEPLOY_PATH || '/home/mahdi/blog-published' }} + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + run: bash scripts/deploy-site.sh diff --git a/.gitignore b/.gitignore index 76b3afa..4a6cec1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ _site img/.DS_Store vendor .jekyll-cache +photo-upload/.env +__pycache__/ +.pytest_cache/ diff --git a/Gemfile.lock b/Gemfile.lock index 1a9f9ce..c6f1ec5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -41,8 +41,6 @@ GEM rouge (~> 3.0) safe_yaml (~> 1.0) terminal-table (~> 2.0) - jekyll-latex (1.1.0) - polytexnic (~> 1.6) jekyll-sass-converter (2.2.0) sassc (> 2.0.1, < 3.0) jekyll-scholar (7.1.0) @@ -52,7 +50,6 @@ GEM jekyll (~> 4.0) jekyll-watch (2.2.1) listen (~> 3.0) - json (2.3.1) kramdown (2.4.0) rexml kramdown-parser-gfm (1.1.0) @@ -64,21 +61,13 @@ GEM rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) mini_portile2 (2.8.0) - msgpack (1.2.10) namae (1.1.1) nokogiri (1.13.6) mini_portile2 (~> 2.8.0) racc (~> 1.4) pathutil (0.16.2) forwardable-extended (~> 2.6) - polytexnic (1.7.3) - json (~> 2.3.0) - kramdown (>= 2.0, < 3.0) - msgpack (~> 1.2.0) - nokogiri (>= 1.6.0, < 2.0) - pygments.rb (~> 2.1) public_suffix (4.0.7) - pygments.rb (2.3.0) racc (1.6.0) rb-fsevent (0.11.1) rb-inotify (0.10.1) @@ -98,7 +87,6 @@ PLATFORMS DEPENDENCIES jekyll - jekyll-latex jekyll-scholar nokogiri webrick (~> 1.7) diff --git a/_config.yml b/_config.yml index 15bbc51..99956b0 100644 --- a/_config.yml +++ b/_config.yml @@ -24,3 +24,6 @@ showToggleButton: true scholar: style: apa +exclude: + - photo-upload + - scripts diff --git a/photo-upload/.dockerignore b/photo-upload/.dockerignore new file mode 100644 index 0000000..66c9758 --- /dev/null +++ b/photo-upload/.dockerignore @@ -0,0 +1,4 @@ +.env +__pycache__ +.pytest_cache +tests diff --git a/photo-upload/.env.example b/photo-upload/.env.example new file mode 100644 index 0000000..1d9e6e8 --- /dev/null +++ b/photo-upload/.env.example @@ -0,0 +1,7 @@ +PHOTO_TOKEN_FILE=/home/mahdi/.config/blog-photos/token +PHOTO_GIT_KEY_FILE=/home/mahdi/.config/blog-photos/git_key +PHOTO_KNOWN_HOSTS_FILE=/home/mahdi/.config/blog-photos/known_hosts +PHOTO_REPOSITORY=git@github.com:mdibaiee/mahdi.blog.git +PHOTO_PUBLIC_DIR=/home/mahdi/blog-published +PHOTO_BRANCH=master +PHOTO_PORT=8088 diff --git a/photo-upload/Containerfile b/photo-upload/Containerfile new file mode 100644 index 0000000..80fe5e4 --- /dev/null +++ b/photo-upload/Containerfile @@ -0,0 +1,8 @@ +FROM docker.io/library/python:3.13-slim +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends git openssh-client && rm -rf /var/lib/apt/lists/* +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY app.py publisher.py ./ +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 GIT_TERMINAL_PROMPT=0 +CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--workers", "1", "--threads", "2", "--timeout", "180", "app:create_app()"] diff --git a/photo-upload/README.md b/photo-upload/README.md new file mode 100644 index 0000000..f8df640 --- /dev/null +++ b/photo-upload/README.md @@ -0,0 +1,188 @@ +# iPhone → blog, with GitHub Actions + +Photos → Share → **Publish to blog** → optional caption → upload. + +The authenticated upload service converts photos to JPEG (maximum 2560 px), applies +orientation, converts embedded color profiles to sRGB, and strips public metadata. +Originals stay in a private volume. A worker commits optimized images and an `_art` +entry to **github.com/mdibaiee/mahdi.blog**, on `master`. + +Every push to `master` triggers `.github/workflows/publish.yml`. GitHub Actions builds +Jekyll and uploads the static output over SSH. nginx switches to the new release +only after the transfer completes. Other blog edits use the same workflow. +No Jekyll build runs on the server, and Nextcloud is not involved. + +## GitHub repository configuration + +Actions is already enabled; there were no publishing workflows or deployment secrets +when inspected. Add these at Settings → Secrets and variables → Actions: + +| Repository secret | Value | +| --- | --- | +| `DEPLOY_SSH_KEY` | Private SSH key whose public key is authorized for deployment on theread.me | +| `DEPLOY_KNOWN_HOSTS` | Verified SSH known_hosts entry for theread.me (include `[host]:port` if nonstandard) | + +The workflow has these defaults; override with repository **variables** if needed: + +| Variable | Default | +| --- | --- | +| `DEPLOY_HOST` | `theread.me` | +| `DEPLOY_USER` | `mahdi` | +| `DEPLOY_PORT` | `22` | +| `DEPLOY_PATH` | `/home/mahdi/blog-published` | + +The server login needs write access to `DEPLOY_PATH`, plus `bash`, `rsync`, and +`flock`. Install its public key in that account's `~/.ssh/authorized_keys`; use a +separate key for this workflow. Verify the SSH host fingerprint independently, +not by blindly trusting an unauthenticated scan. The server must accept SSH from +GitHub-hosted runners. Deployment itself needs no sudo or nginx reload. + +Separately, add the photo publisher's public SSH key at Settings → Deploy keys, +with **Allow write access** enabled for this repository. This key stays on your +server and allows photo pushes to trigger Actions; it is not a workflow secret or +`GITHUB_TOKEN`. GitHub is the publishing source of truth. Push ordinary edits to +the `github` remote too; pushing only to git.theread.me does not trigger this workflow. + +The workflow uses read-only repository permissions and pinned action commits. It +runs only for `master`, and serialized deployments plus a server-side run-number +check prevent an older workflow from replacing a newer deployment. Ruby matches +this blog's existing 3.1.2 requirement. The lockfile removes the obsolete +jekyll-latex dependency that was already absent from Gemfile, allowing frozen CI +installs. Runtime upgrades are separate from this change. + +## Server setup + +The live server is Ubuntu 24.04. nginx currently serves +`/home/mahdi/theread.me/_site`. `/home/mahdi/update-server/update-server.js` exists, +was not running, and uses a different port from nginx's old `/_update` route. +Podman is not installed. The new workflow replaces that rebuild hook. + +1. Install Podman and a Compose provider (`sudo apt-get install podman podman-compose`). +2. Keep this service in its own checkout, e.g. `/home/mahdi/blog-photo-service`. +3. Prepare private credentials and the public output directory: + +```sh +install -d -m 700 "$HOME/.config/blog-photos" +install -d -m 755 /home/mahdi/blog-published +umask 077 +openssl rand -hex 32 > "$HOME/.config/blog-photos/token" +ssh-keygen -t ed25519 -N '' -f "$HOME/.config/blog-photos/git_key" -C blog-photo-publisher +``` + +Register `git_key.pub` as the repository's write-enabled deploy key. Put GitHub's +verified SSH host key in `~/.config/blog-photos/known_hosts`. Keep the private key +readable only by its owner. This key is separate from Actions' server-login key. + +```sh +cd /home/mahdi/blog-photo-service/photo-upload +cp .env.example .env +# Adjust absolute paths if needed. +podman compose config +podman compose up -d --build +podman compose logs -f publisher +``` + +On SELinux hosts add shared `:z` labels to bind mounts as appropriate. + +Commit and push this change to GitHub, and wait for the first Actions deployment. +Then back up nginx's site config and use `host-nginx.conf.example`: change the +HTTPS server's root to `/home/mahdi/blog-published/current` and add `/api/photos`. +Preserve the existing `/`, `/raw/`, TLS, `/_xpanel`, and unrelated routes. Ensure +nginx can traverse parent directories and read the output. + +```sh +sudo nginx -t +sudo systemctl reload nginx +``` + +The API listens on loopback port 8088; HTTPS remains on the existing nginx. +Rollback by restoring the original nginx root and reloading. The old checkout and +site are untouched. Once verified, retire the old rebuild route. + +For reboot persistence, create `~/.config/systemd/user/blog-photos.service`: + +```ini +[Unit] +Description=Blog photo uploads +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/home/mahdi/blog-photo-service/photo-upload +ExecStart=/usr/bin/podman compose up -d +ExecStop=/usr/bin/podman compose stop +TimeoutStartSec=180 + +[Install] +WantedBy=default.target +``` + +Then run `systemctl --user daemon-reload`, `systemctl --user enable --now blog-photos.service`, +and `sudo loginctl enable-linger mahdi`. + +## iPhone shortcut + +Create **Publish to blog**: + +1. Enable **Show in Share Sheet**, accepting **Images**. +2. **Ask for Input** (Text): “Caption (optional)”. +3. **Get Contents of URL**: `https://theread.me/api/photos`. + - Method: **POST**. + - Header: `Authorization` = `Bearer YOUR_TOKEN`. + - Request Body: **Form**. + - `photos`: type **File**, value **Shortcut Input** (the selected images). + - `caption`: type **Text**, value the caption response. + - Do not set Content-Type; Shortcuts supplies the multipart boundary. +4. Show “Uploaded; publishing” on success. Optionally poll the returned `status_url` + with the same header. `pushed` means “GitHub Actions is building/deploying”; + only `published` means the deployed site's receipt contains this album. + +Multi-photo albums require repeated `photos` parts in one request. Verify that +file-list behavior on the target iPhone, starting with one photo and then two. +Select still photos; videos/Live Photo video components aren't supported. Dates +are upload time in UTC; captions are plain text. A caption is optional. + +The token is visible in the shortcut editor; remove it before sharing the shortcut. +Rotate it by replacing the token file and recreating the upload container with +`podman compose up -d --force-recreate upload`. + +## API, failures, and backups + +All API routes require the bearer token: + +- `POST /api/photos`: 1–20 multipart `photos`, optional `caption` (4000 characters). + Maximum request 100 MiB; each photo at most 60 megapixels. +- `GET /api/photos/`: `queued`, `publishing`, `pushed`, `published`, or `failed`. +- `POST /api/photos//retry`: retry a failed Git publication. + +Identical original bytes, ordering, and caption produce the same ID. Retrying does +not duplicate entries or commits. Re-encoding or changing the caption creates a new +submission; edit an existing entry through Git. Keep the upload response/status URL. + +For Git failures, check `podman compose logs publisher`, fix access/conflicts, and +POST to the status URL plus `/retry`. No force push is used. If Actions fails, +status stays `pushed`: inspect and rerun the failed workflow on GitHub, rather than +uploading again. Deployment only switches nginx after a complete transfer; the old +site stays available during builds and failures. The receipt is built from rendered +photo anchors so the service does not mistake a successful push for publication. + +Back up the private `photo-data` volume (originals and job state) and GitHub's +repository. `photo-work` is disposable. Never expose private data through nginx or +run `compose down -v`. Originals and deployment releases are retained, so monitor +disk usage and periodically remove old releases (never `current` or an in-progress +release). Interrupted uploads may leave files in `photo-data/temporary`; clean that +folder only while the API is stopped. Public Git images are optimized copies. + +## Tests + +```sh +python3 -m venv /tmp/blog-photo-tests +/tmp/blog-photo-tests/bin/pip install -r requirements.txt pytest +/tmp/blog-photo-tests/bin/pytest -q tests +``` + +Tests use a temporary local bare Git repository; no test pushes go to GitHub or the +live server. The iPhone shortcut and production deployment need an end-to-end check +after credentials are configured. diff --git a/photo-upload/app.py b/photo-upload/app.py new file mode 100644 index 0000000..b137fcc --- /dev/null +++ b/photo-upload/app.py @@ -0,0 +1,173 @@ +"""Authenticated photo intake. Only completed directories are visible to the builder.""" + +import hashlib +import hmac +import html +import io +import json +import os +from pathlib import Path +import re +import shutil +import tempfile +from datetime import datetime, timezone +import warnings + +from flask import Flask, abort, jsonify, request +from PIL import Image, ImageCms, ImageOps, UnidentifiedImageError +from pillow_heif import register_heif_opener +from werkzeug.exceptions import HTTPException + +register_heif_opener() +Image.MAX_IMAGE_PIXELS = 60_000_000 +warnings.simplefilter("error", Image.DecompressionBombWarning) + + +def atomic_json(path, value): + fd, temporary = tempfile.mkstemp(dir=path.parent) + try: + with os.fdopen(fd, "w") as output: + json.dump(value, output) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + + +def public_text(text): + # Escape both HTML and Liquid: Jekyll evaluates Liquid before Markdown. + return html.escape(text, quote=True).replace("{", "{").replace("}", "}") + + +def web_image(raw, destination): + with Image.open(io.BytesIO(raw)) as original: + if original.format not in {"JPEG", "PNG", "WEBP", "HEIF"}: + raise ValueError("Use JPEG, PNG, WebP, or HEIC photos") + image = ImageOps.exif_transpose(original) + image.thumbnail((2560, 2560), Image.Resampling.LANCZOS) + if image.info.get("icc_profile"): + image = ImageCms.profileToProfile( + image, ImageCms.ImageCmsProfile(io.BytesIO(image.info["icc_profile"])), + ImageCms.createProfile("sRGB"), outputMode="RGB", + ) + if image.mode in {"RGBA", "LA"} or "transparency" in image.info: + rgba = image.convert("RGBA") + background = Image.new("RGB", image.size, "white") + background.paste(rgba, mask=rgba.getchannel("A")) + image = background + else: + image = image.convert("RGB") + # Copy pixels only; no EXIF, GPS, XMP, comments, or original profile. + clean = Image.new("RGB", image.size) + clean.paste(image) + clean.save(destination, "JPEG", quality=88, optimize=True) + + +def create_app(data_dir=None, token=None): + app = Flask(__name__) + app.config.update(MAX_CONTENT_LENGTH=100 * 1024 * 1024, + MAX_FORM_MEMORY_SIZE=64 * 1024, MAX_FORM_PARTS=40) + data = Path(data_dir or os.environ.get("PHOTO_DATA", "/data")) + if token is None: + token = Path(os.environ.get("PHOTO_TOKEN_FILE", "/run/secrets/photo_token")).read_text().strip() + if len(token) < 32 or not token.isascii(): + raise ValueError("Photo token must be at least 32 ASCII characters") + expected_auth = ("Bearer " + token).encode() + for directory in ("submissions", "status", "temporary"): + (data / directory).mkdir(parents=True, exist_ok=True) + + @app.before_request + def authenticate(): + if request.path != "/healthz": + auth = request.headers.get("Authorization", "").encode() + if not hmac.compare_digest(auth, expected_auth): + abort(401, "Invalid upload token") + + @app.errorhandler(HTTPException) + def error_response(error): + return jsonify(error=error.description), error.code + + @app.get("/healthz") + def health(): + return {"status": "ok"} + + def response_for(job): + manifest_path = data / "submissions" / job / "manifest.json" + if not re.fullmatch(r"[a-f0-9]{64}", job) or not manifest_path.is_file(): + abort(404) + manifest = json.loads(manifest_path.read_text()) + status_path = data / "status" / (job + ".json") + state = json.loads(status_path.read_text()) if status_path.exists() else {"status": "queued"} + if state["status"] == "pushed": + deployed = Path(os.environ.get("PHOTO_DEPLOYMENT_FILE", "/published/current/photo-publication.json")) + try: + if job in json.loads(deployed.read_text())["photos"]: + state["status"] = "published" + except (OSError, ValueError, KeyError): + pass # A successful push is not yet a successful deployment. + return dict(id=job, count=manifest["count"], **state, + status_url=f"/api/photos/{job}", url=f"/art#photo-{job}") + + @app.get("/api/photos/") + def status(job): + return response_for(job) + + @app.post("/api/photos//retry") + def retry(job): + current = response_for(job) + if current["status"] == "failed": + atomic_json(data / "status" / (job + ".json"), {"status": "queued"}) + return response_for(job), 202 + + @app.post("/api/photos") + def upload(): + photos = request.files.getlist("photos") + if not 1 <= len(photos) <= 20: + abort(400, "Send between 1 and 20 files in the photos form field") + caption = request.form.get("caption", "").strip() + if len(caption) > 4000: + abort(400, "Caption must be at most 4000 characters") + # Ordered byte hashes + caption make retries idempotent, even across restarts. + digest = hashlib.sha256(caption.encode()) + temporary = Path(tempfile.mkdtemp(dir=data / "temporary")) + try: + (temporary / "originals").mkdir() + (temporary / "images").mkdir() + for index, photo in enumerate(photos): + raw = photo.read() + digest.update(hashlib.sha256(raw).digest()) + (temporary / "originals" / str(index)).write_bytes(raw) + try: + web_image(raw, temporary / "images" / f"{index:02}.jpg") + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, + Image.DecompressionBombError, Image.DecompressionBombWarning, + ImageCms.PyCMSError): + abort(400, f"Photo {index + 1} is unsupported, damaged, or exceeds 60 megapixels") + job = digest.hexdigest() + date = datetime.now(timezone.utc).isoformat() + manifest = {"id": job, "date": date, "caption": caption, "count": len(photos)} + atomic_json(temporary / "manifest.json", manifest) + content = ["---", "layout: post", f'title: "photo-{job}"', + f'date: "{date}"', "categories: art", "---", "", + f'
', ""] + for index in range(len(photos)): + content.append(f'

') + if caption: + content.append('' + + public_text(caption).replace("\n", "
") + "") + (temporary / "entry.md").write_text("\n".join(content) + "\n") + # Atomic rename handles simultaneous identical submissions without replacing one. + try: + temporary.rename(data / "submissions" / job) + created = True + except OSError: + if not (data / "submissions" / job / "manifest.json").is_file(): + raise + created = False + return response_for(job), 202 if created else 200 + finally: + shutil.rmtree(temporary, ignore_errors=True) + + return app diff --git a/photo-upload/compose.yaml b/photo-upload/compose.yaml new file mode 100644 index 0000000..9e66ca1 --- /dev/null +++ b/photo-upload/compose.yaml @@ -0,0 +1,42 @@ +services: + upload: + build: + context: . + dockerfile: Containerfile + restart: unless-stopped + ports: ["127.0.0.1:${PHOTO_PORT:-8088}:8000"] + volumes: + - photo-data:/data + - ${PHOTO_TOKEN_FILE:?Set PHOTO_TOKEN_FILE to an absolute path}:/run/secrets/photo_token:ro + - ${PHOTO_PUBLIC_DIR:?Set PHOTO_PUBLIC_DIR to the deployment directory}:/published:ro + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"] + interval: 30s + timeout: 5s + retries: 3 + security_opt: ["no-new-privileges:true"] + cap_drop: ["ALL"] + + publisher: + build: + context: . + dockerfile: Containerfile + command: ["python", "publisher.py"] + restart: unless-stopped + environment: + PHOTO_REPOSITORY: ${PHOTO_REPOSITORY:-git@github.com:mdibaiee/mahdi.blog.git} + PHOTO_BRANCH: ${PHOTO_BRANCH:-master} + PHOTO_GIT_NAME: Photo publisher + PHOTO_GIT_EMAIL: ${PHOTO_GIT_EMAIL:-photos@theread.me} + GIT_SSH_COMMAND: ssh -i /run/secrets/git_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/run/secrets/known_hosts + volumes: + - photo-data:/data + - photo-work:/work + - ${PHOTO_GIT_KEY_FILE:?Set PHOTO_GIT_KEY_FILE to an absolute path}:/run/secrets/git_key:ro + - ${PHOTO_KNOWN_HOSTS_FILE:?Set PHOTO_KNOWN_HOSTS_FILE to an absolute path}:/run/secrets/known_hosts:ro + security_opt: ["no-new-privileges:true"] + cap_drop: ["ALL"] + +volumes: + photo-data: + photo-work: diff --git a/photo-upload/host-nginx.conf.example b/photo-upload/host-nginx.conf.example new file mode 100644 index 0000000..810a2dd --- /dev/null +++ b/photo-upload/host-nginx.conf.example @@ -0,0 +1,10 @@ +# In the existing HTTPS server block, change root after the first Actions deploy: +root /home/mahdi/blog-published/current; + +# Add this alongside the existing /, /raw/, /_xpanel, and TLS configuration. +location /api/photos { + client_max_body_size 100m; + proxy_read_timeout 180s; + proxy_pass http://127.0.0.1:8088; + proxy_set_header Host $host; +} diff --git a/photo-upload/publisher.py b/photo-upload/publisher.py new file mode 100644 index 0000000..323617b --- /dev/null +++ b/photo-upload/publisher.py @@ -0,0 +1,99 @@ +"""Commit queued submissions to GitHub; GitHub Actions builds and deploys.""" +import fcntl +import json +import logging +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time + +from app import atomic_json + +log = logging.getLogger("publisher") + + +class Publisher: + def __init__(self): + self.data = Path(os.environ.get("PHOTO_DATA", "/data")) + self.work = Path(os.environ.get("PHOTO_WORK", "/work")) + self.repo = self.work / "repo" + self.remote = os.environ["PHOTO_REPOSITORY"] + self.branch = os.environ.get("PHOTO_BRANCH", "master") + for path in (self.work, self.data / "submissions", self.data / "status"): + path.mkdir(parents=True, exist_ok=True) + + def git(self, *arguments, directory=None): + result = subprocess.run(["git", *arguments], cwd=directory or self.repo, + text=True, capture_output=True, timeout=300, check=True) + return result.stdout.strip() + + def state(self, job, status, **extra): + atomic_json(self.data / "status" / f"{job}.json", dict(status=status, **extra)) + + def pending(self): + jobs = [] + for path in sorted((self.data / "submissions").iterdir()): + status_path = self.data / "status" / f"{path.name}.json" + status = json.loads(status_path.read_text())["status"] if status_path.exists() else "queued" + if status in {"queued", "publishing"}: + jobs.append(path.name) + return jobs + + def publish(self): + jobs = self.pending() + if not jobs: + return + for job in jobs: + self.state(job, "publishing") + try: + if not (self.repo / ".git").is_dir(): + self.git("clone", "--branch", self.branch, "--single-branch", "--", + self.remote, str(self.repo), directory=self.work) + self.git("fetch", "origin", self.branch) + # Only reset this service-owned disposable checkout, never the user's checkout. + self.git("reset", "--hard", f"origin/{self.branch}") + self.git("config", "user.name", os.environ.get("PHOTO_GIT_NAME", "Photo publisher")) + self.git("config", "user.email", os.environ.get("PHOTO_GIT_EMAIL", "photos@theread.me")) + for job in jobs: + submission = self.data / "submissions" / job + manifest = json.loads((submission / "manifest.json").read_text()) + entry = Path("_art") / f"{manifest['date'][:10]}-photo-{job}.md" + images = Path("img/arts/uploads") / job + (self.repo / entry).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(submission / "entry.md", self.repo / entry) + shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True) + self.git("add", "--", str(entry), str(images)) + if self.git("diff", "--cached", "--name-only"): + self.git("commit", "-m", f"Publish {len(jobs)} photo submission(s)") + self.git("push", "origin", f"HEAD:refs/heads/{self.branch}") + commit = self.git("rev-parse", "HEAD") + for job in jobs: + self.state(job, "pushed", commit=commit) + log.info("Pushed %s submission(s) at %s", len(jobs), commit) + except (OSError, subprocess.SubprocessError) as error: + log.error("Git publication failed: %s", error) + if isinstance(error, subprocess.CalledProcessError): + log.error("%s", error.stderr) + for job in jobs: + self.state(job, "failed", error="Git push failed; check publisher logs and retry.") + raise + + def run(self, once=False): + with (self.work / "publisher.lock").open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + while True: + try: + self.publish() + except (OSError, subprocess.SubprocessError): + if once: + raise + if once: + return + time.sleep(3) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + Publisher().run(once="--once" in sys.argv) diff --git a/photo-upload/requirements.txt b/photo-upload/requirements.txt new file mode 100644 index 0000000..0fdcd75 --- /dev/null +++ b/photo-upload/requirements.txt @@ -0,0 +1,4 @@ +Flask==3.1.3 +gunicorn==26.2.0 +Pillow==12.3.0 +pillow-heif==1.7.0 diff --git a/photo-upload/tests/test_upload.py b/photo-upload/tests/test_upload.py new file mode 100644 index 0000000..b652510 --- /dev/null +++ b/photo-upload/tests/test_upload.py @@ -0,0 +1,164 @@ +import io +import json +from pathlib import Path +import subprocess +import sys +import os + +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app import create_app + +TOKEN = "t" * 48 +AUTH = {"Authorization": "Bearer " + TOKEN} + + +@pytest.fixture +def service(tmp_path): + app = create_app(tmp_path / "data", TOKEN) + app.config["TESTING"] = True + return app, app.test_client(), tmp_path / "data" + + +def photo(color="red", format="JPEG", size=(40, 20), exif=None): + output = io.BytesIO() + Image.new("RGB", size, color).save(output, format, **({"exif": exif} if exif else {})) + return output.getvalue() + + +def upload(client, images=None, caption="", headers=AUTH): + return client.post("/api/photos", headers=headers, data={ + "caption": caption, + "photos": [(io.BytesIO(raw), "../../escape.jpg") for raw in (images or [photo()])], + }) + + +def test_authentication_and_request_limits(service): + app, client, data = service + assert upload(client, headers={}).status_code == 401 + assert upload(client, headers={"Authorization": "Bearer wrong"}).status_code == 401 + assert client.get("/api/photos/" + "a" * 64).status_code == 401 + assert client.get("/healthz").status_code == 200 + assert upload(client, caption="a" * 4001).status_code == 400 + assert client.post("/api/photos", headers=AUTH).status_code == 400 + assert upload(client, [photo()] * 21).status_code == 400 + app.config["MAX_CONTENT_LENGTH"] = 100 + assert upload(client).status_code == 413 + assert not list((data / "submissions").iterdir()) + + +def test_validation_is_all_or_nothing(service): + _, client, data = service + assert upload(client, [photo(), b"not an image"]).status_code == 400 + assert upload(client, [photo(format="GIF")]).status_code == 400 + assert not list((data / "submissions").iterdir()) + assert not list((data / "temporary").iterdir()) + + +def test_album_caption_order_and_retry_deduplication(service): + _, client, data = service + caption = ' {% include secret %} {{ site.email }}\nsecond line' + images = [photo("red"), photo("blue")] + response = upload(client, images, caption) + assert response.status_code == 202 + job = response.json["id"] + folder = data / "submissions" / job + entry = (folder / "entry.md").read_text() + assert "