diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4d59ad6..578d48c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,6 +23,10 @@ jobs: persist-credentials: false - name: Test deployment scripts run: python3 -m unittest discover -s scripts/tests -v + - name: Test publishing service + run: | + python3 -m pip install -r photo-upload/requirements.txt 'pytest>=8,<10' + python3 -m pytest -q photo-upload/tests - uses: ruby/setup-ruby@a0102e0972be65f351c307e2d64b9314a57c8073 # v1 with: ruby-version: '3.1.2' diff --git a/_includes/head.html b/_includes/head.html index bafd179..807e62b 100644 --- a/_includes/head.html +++ b/_includes/head.html @@ -2,6 +2,7 @@ + {% if page.upload_revision %}{% endif %} {% if page.title %}{{ page.title }}{% else %}{{ site.title }}{% endif %} diff --git a/photo-upload/Containerfile b/photo-upload/Containerfile index 80fe5e4..147d03c 100644 --- a/photo-upload/Containerfile +++ b/photo-upload/Containerfile @@ -3,6 +3,6 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends git openssh-client && rm -rf /var/lib/apt/lists/* COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt -COPY app.py publisher.py ./ +COPY app.py publisher.py posts.py ./ ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 GIT_TERMINAL_PROMPT=0 CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--workers", "1", "--threads", "2", "--timeout", "180", "app:create_app()"] diff --git a/photo-upload/README.md b/photo-upload/README.md index a9d8358..de3d6ce 100644 --- a/photo-upload/README.md +++ b/photo-upload/README.md @@ -155,6 +155,112 @@ The token is visible in the shortcut editor; remove it before sharing the shortc Rotate it by replacing the token file and recreating the upload container with `podman compose up -d --force-recreate upload`. +## Snippets shortcut + +The same service and token also publish to `https://theread.me/snippets/`. +Text stays literal (including line breaks and Persian text); standalone HTTP/HTTPS +URL lines become clickable links. Images use the same optimization and metadata +removal as photo uploads. Each submission is prepended below the introduction, +with a separator from the previous snippet. Existing snippets are preserved. + +Create **Publish snippet**: + +1. Enable **Show in Share Sheet**, accepting **Text**, **URLs**, and **Images**. +2. **Repeat with Each** item in **Shortcut Input**. Use **Get Type** on the repeat + item. If it is **Image**, add the item to a variable called **Snippet Images**. + Otherwise use **Get Text from Input** on the item and add its result to + **Snippet Text**. These variables accumulate the shared items during this run. +3. **Combine Text** from **Snippet Text**, separated by **New Lines**. For an + image-only share, leave the text empty. Don't fetch the contents of shared URLs: + send the link itself. +4. **Ask for Input** (Text): “Note (optional)”. Keep its output separate from the + shared text. You can also run the shortcut without shared input and use a text + prompt as **Snippet Text** to write a new snippet. +5. **Get Contents of URL**: `https://theread.me/api/snippets`, method **POST**, + request body **Form**: + - Header `Authorization`: `Bearer YOUR_TOKEN` (same token as photos). + - `text`: **Text**, the combined shared text. + - `caption`: **Text**, the optional note. + - `photos`: **File**, **Snippet Images**, when sharing images. For text-only + requests, omit this field. Use an **If** on whether there are images to choose + between the form with `photos` and the text-only form. + - Delete any blank header rows. Do not manually set `Content-Type`. +6. **Show Result** using the actual response. A response with `error` means the + upload failed; don't show a fixed success message. `queued` means accepted; + publishing happens afterwards. The returned `status_url` can be polled with the + same authorization header; `published` confirms deployment. + +For the simplest initial shortcut, accept just Text and URLs and omit the image +branch and `photos` field; add the image branch when that works on your phone. + +`POST /api/snippets` accepts up to 20,000 characters of `text`, an optional +4,000-character `caption`, and up to 20 images in repeated `photos` parts, within +the existing 100 MiB request limit. At least text or an image is required. +It returns `id`, `count` (image count), `url`, `status_url`, and `status`. +`GET /api/snippets/` checks publication, and +`POST /api/snippets//retry` retries failed Git publication. Identical text, +note, and original images in the same order are deduplicated. + +Keep `` in `snippets.md`: it marks the insertion point. +The publisher fails safely if that marker is removed. GitHub Actions includes +rendered snippet IDs in `photo-publication.json`, alongside photo IDs. + +## Markdown blog-post shortcut + +Create **Publish blog post** and enable **Show in Share Sheet**, accepting **Files**. +Share one UTF-8 `.md` or `.markdown` file, up to 1 MiB. In **Get Contents of URL**: + +- URL: `https://theread.me/api/posts`; method **POST**; request body **Form**. +- Header: `Authorization` = `Bearer YOUR_TOKEN` (no empty header rows). +- Form field `file`: type **File**, value **Shortcut Input**. +- Let Shortcuts set `Content-Type` automatically. + +Show the actual response, and poll `https://theread.me` plus its `status_url` +with the same header. Matching happens in the publisher: `202` only means the +file was accepted, not that a new post or update was published. The status +response supplies the final `url`, `post_id`, and `action` (`created` or `updated`). +`failed` includes an error; `published` confirms the deployed revision. An older +upload can become `superseded` when a newer revision replaces it. + +No front matter is required. The first `# Heading` becomes the title, or the +filename is used if there is no heading. Optional YAML front matter supports +`title`, `subtitle`, `lang`, `description`, `categories`, `tags`, `toc`, `math`, +`date`, and `permalink` (a path such as `/my-essay/`). Markdown formatting is +preserved; Liquid template evaluation is disabled for imported post bodies. +The original file is retained privately in the service volume. + +### Automatic updates by content + +The publisher compares the body against existing Markdown posts, ignoring front +matter and whitespace changes. It updates a unique match with at least 92% word +sequence similarity, at least 200 characters in both texts, and a lead of at +least 8 percentage points over the next plausible match. An exact body match of +at least 40 characters also qualifies. These scores are a conservative heuristic, +not a probability. Renaming the file does not prevent matching; reusing a filename +for unrelated writing does not overwrite its previous post. + +If several posts match closely, the upload fails without changing Git. If no +post meets the threshold, it creates a new post. Large rewrites and very short +posts may therefore need an explicit ID. For a known imported post, add the +returned `post_id` to its front matter to target it directly: + +```yaml +--- +title: My revised essay +post_id: THE_ID_FROM_THE_UPLOAD_STATUS +--- +``` + +For an older Git-authored post, first add a unique `post_id` to that existing +post's front matter and use the same ID in the uploaded file. Updates preserve +the original publication date and public URL, and remain reversible through Git. +Identical file retries are deduplicated and do not roll back a newer revision. + +The upload takes the Markdown file only. Relative links to images or attachments +in an export folder will not upload those files; use existing public image URLs. +Export Markdown from the source app before sharing; proprietary note files, +PDFs, ZIP bundles, and rich-text files are not Markdown uploads. + ## API, failures, and backups All API routes require the bearer token: diff --git a/photo-upload/app.py b/photo-upload/app.py index 5cb4e2a..6f19eb5 100644 --- a/photo-upload/app.py +++ b/photo-upload/app.py @@ -17,6 +17,7 @@ from flask import Flask, abort, jsonify, request from PIL import Image, ImageCms, ImageOps, UnidentifiedImageError from pillow_heif import register_heif_opener from werkzeug.exceptions import HTTPException +from posts import parse_upload register_heif_opener() Image.MAX_IMAGE_PIXELS = 60_000_000 @@ -67,7 +68,8 @@ def web_image(raw, destination): def create_app(data_dir=None, token=None): app = Flask(__name__) app.config.update(MAX_CONTENT_LENGTH=100 * 1024 * 1024, - MAX_FORM_MEMORY_SIZE=64 * 1024, MAX_FORM_PARTS=40) + # Must exceed Werkzeug's 64 KiB multipart read buffer plus headers. + MAX_FORM_MEMORY_SIZE=512 * 1024, MAX_FORM_PARTS=40) data = Path(data_dir or os.environ.get("PHOTO_DATA", "/data")) if token is None: token = Path(os.environ.get("PHOTO_TOKEN_FILE", "/run/secrets/photo_token")).read_text().strip() @@ -92,22 +94,78 @@ def create_app(data_dir=None, token=None): def health(): return {"status": "ok"} - def response_for(job): + def response_for(job, kind="photo"): manifest_path = data / "submissions" / job / "manifest.json" if not re.fullmatch(r"[a-f0-9]{64}", job) or not manifest_path.is_file(): abort(404) manifest = json.loads(manifest_path.read_text()) + if manifest.get("kind", "photo") != kind: + abort(404) status_path = data / "status" / (job + ".json") state = json.loads(status_path.read_text()) if status_path.exists() else {"status": "queued"} if state["status"] == "pushed": deployed = Path(os.environ.get("PHOTO_DEPLOYMENT_FILE", "/published/current/photo-publication.json")) try: - if job in json.loads(deployed.read_text())["photos"]: + if job in json.loads(deployed.read_text()).get(kind + "s", []): state["status"] = "published" except (OSError, ValueError, KeyError): pass # A successful push is not yet a successful deployment. - return dict(id=job, count=manifest["count"], **state, - status_url=f"/api/photos/{job}", url=f"/art#photo-{job}") + page = "art" if kind == "photo" else "snippets/" + result = dict(id=job, count=manifest["count"], + status_url=f"/api/{kind}s/{job}", + url=None if kind == "post" else f"/{page}#{kind}-{job}") + result.update(state) + return result + + @app.get("/api/posts/") + def post_status(job): + return response_for(job, "post") + + @app.post("/api/posts//retry") + def post_retry(job): + current = response_for(job, "post") + if current["status"] == "failed": + atomic_json(data / "status" / (job + ".json"), {"status": "queued"}) + return response_for(job, "post"), 202 + + @app.post("/api/posts") + def upload_post(): + files = request.files.getlist("file") + if len(files) != 1: + abort(400, "Send one Markdown file in the file form field") + raw = files[0].read(1024 * 1024 + 1) + try: + incoming = parse_upload(files[0].filename, raw) + except ValueError as error: + abort(400, str(error)) + job = hashlib.sha256(b"post\0" + incoming["key"].encode() + b"\0" + raw).hexdigest() + temporary = Path(tempfile.mkdtemp(dir=data / "temporary")) + try: + atomic_json(temporary / "post.json", incoming) + (temporary / "original.md").write_bytes(raw) + atomic_json(temporary / "manifest.json", dict(id=job, kind="post", count=1, + date=datetime.now(timezone.utc).isoformat())) + try: + temporary.rename(data / "submissions" / job) + created = True + except OSError: + if not (data / "submissions" / job / "manifest.json").is_file(): + raise + created = False + return response_for(job, "post"), 202 if created else 200 + finally: + shutil.rmtree(temporary, ignore_errors=True) + + @app.get("/api/snippets/") + def snippet_status(job): + return response_for(job, "snippet") + + @app.post("/api/snippets//retry") + def snippet_retry(job): + current = response_for(job, "snippet") + if current["status"] == "failed": + atomic_json(data / "status" / (job + ".json"), {"status": "queued"}) + return response_for(job, "snippet"), 202 @app.get("/api/photos/") def status(job): @@ -121,10 +179,17 @@ def create_app(data_dir=None, token=None): return response_for(job), 202 @app.post("/api/photos") + @app.post("/api/snippets") def upload(): + kind = "snippet" if request.path == "/api/snippets" else "photo" photos = request.files.getlist("photos") - if not 1 <= len(photos) <= 20: + if not (0 if kind == "snippet" else 1) <= len(photos) <= 20: abort(400, "Send between 1 and 20 files in the photos form field") + text = request.form.get("text", "").strip() if kind == "snippet" else "" + if len(text) > 20000: + abort(400, "Snippet text must be at most 20000 characters") + if kind == "snippet" and not text and not photos: + abort(400, "Send text or images for the snippet") caption = request.form.get("caption", "").strip() if len(caption) > 4000: abort(400, "Caption must be at most 4000 characters") @@ -133,6 +198,8 @@ def create_app(data_dir=None, token=None): abort(400, "Highlight name must be at most 200 characters") # Ordered byte hashes + caption make retries idempotent, even across restarts. digest = hashlib.sha256(caption.encode()) + if kind == "snippet": + digest.update(b"\0snippet\0" + text.encode()) temporary = Path(tempfile.mkdtemp(dir=data / "temporary")) try: (temporary / "originals").mkdir() @@ -154,6 +221,8 @@ def create_app(data_dir=None, token=None): date = datetime.now(timezone.utc).isoformat() manifest = {"id": job, "date": date, "caption": caption, "highlight": highlight, "count": len(photos)} + if kind == "snippet": + manifest.update(kind=kind, text=text) atomic_json(temporary / "manifest.json", manifest) content = ["---", "layout: post", f'title: "photo-{job}"', f'date: "{date}"', "categories: art"] @@ -169,6 +238,20 @@ def create_app(data_dir=None, token=None): if caption: content.append('' + public_text(caption).replace("\n", "
") + "") + if kind == "snippet": + content = [f'
', ""] + for paragraph in (text, caption): + if paragraph: + safe = public_text(paragraph) + # Link standalone URL lines; all other shared text stays literal. + lines = [f'{line}' + if re.fullmatch(r"https?://[^\s<>]+", line) else line + for line in safe.splitlines()] + content.append('

' + '
\n'.join(lines) + '

') + for index in range(len(photos)): + content.append(f'

') + content.extend(["", "----", ""]) (temporary / "entry.md").write_text("\n".join(content) + "\n") # Atomic rename handles simultaneous identical submissions without replacing one. try: @@ -178,7 +261,7 @@ def create_app(data_dir=None, token=None): if not (data / "submissions" / job / "manifest.json").is_file(): raise created = False - return response_for(job), 202 if created else 200 + return response_for(job, kind), 202 if created else 200 finally: shutil.rmtree(temporary, ignore_errors=True) diff --git a/photo-upload/host-nginx.conf.example b/photo-upload/host-nginx.conf.example index 810a2dd..ec5eb05 100644 --- a/photo-upload/host-nginx.conf.example +++ b/photo-upload/host-nginx.conf.example @@ -2,8 +2,9 @@ root /home/mahdi/blog-published/current; # Add this alongside the existing /, /raw/, /_xpanel, and TLS configuration. -location /api/photos { +location ~ ^/api/(photos|snippets|posts)(/|$) { client_max_body_size 100m; + client_body_timeout 900s; proxy_read_timeout 180s; proxy_pass http://127.0.0.1:8088; proxy_set_header Host $host; diff --git a/photo-upload/posts.py b/photo-upload/posts.py new file mode 100644 index 0000000..b96df61 --- /dev/null +++ b/photo-upload/posts.py @@ -0,0 +1,161 @@ +"""Markdown import with conservative content matching and explicit ID overrides.""" +from difflib import SequenceMatcher +from datetime import date, datetime +import hashlib +import re +import unicodedata + +import yaml + + +def split_markdown(text): + text = text.replace("\r\n", "\n").lstrip("\ufeff") + if text.startswith("---\n"): + parts = text.split("\n---", 1) + if len(parts) != 2 or (parts[1] and not parts[1].startswith("\n")): + raise ValueError("Markdown front matter must end with a --- line") + try: + metadata = yaml.safe_load(parts[0][4:]) or {} + except yaml.YAMLError: + raise ValueError("Invalid YAML front matter") from None + if not isinstance(metadata, dict): + raise ValueError("Front matter must contain named fields") + return metadata, parts[1].lstrip("\n") + return {}, text + + +def filename_key(filename): + return unicodedata.normalize("NFC", filename).casefold() + + +def parse_upload(filename, raw): + if not filename or "/" in filename or "\\" in filename or not filename.lower().endswith((".md", ".markdown")): + raise ValueError("Share one .md or .markdown file with a plain filename") + if len(filename) > 240 or len(raw) > 1024 * 1024: + raise ValueError("Markdown files must be at most 1 MiB with filenames up to 240 characters") + try: + text = raw.decode("utf-8-sig") + except UnicodeDecodeError: + raise ValueError("Markdown files must use UTF-8") from None + metadata, body = split_markdown(text) + if not body.strip(): + raise ValueError("The Markdown file is empty") + clean = {} + for name in ("title", "subtitle", "lang", "description", "permalink", "post_id"): + if name in metadata: + if not isinstance(metadata[name], str) or len(metadata[name]) > 1000: + raise ValueError(f"{name} must be text of at most 1000 characters") + clean[name] = metadata[name].strip() + for name in ("categories", "tags"): + if name in metadata: + value = metadata[name] + if isinstance(value, str): + value = value.split() + if not isinstance(value, list) or len(value) > 30 or any(not isinstance(item, str) or len(item) > 100 for item in value): + raise ValueError(f"{name} must be a short list of names") + clean[name] = value + for name in ("toc", "math"): + if name in metadata: + if not isinstance(metadata[name], bool): + raise ValueError(f"{name} must be true or false") + clean[name] = metadata[name] + if "date" in metadata: + value = metadata["date"] + if isinstance(value, (date, datetime)): + value = value.isoformat() + try: + datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + raise ValueError("date must be an ISO date or timestamp") from None + clean["date"] = str(value) + explicit_id = clean.pop("post_id", None) + if explicit_id is not None and not re.fullmatch(r"[a-zA-Z0-9_-]{1,80}", explicit_id): + raise ValueError("post_id must be 1–80 letters, digits, underscores, or hyphens") + if "permalink" in clean and not re.fullmatch(r"/[\w/-]+/", clean["permalink"]): + raise ValueError("permalink must be a local path such as /my-post/") + title = clean.get("title") + if not title: + heading = re.search(r"^#\s+(.+)$", body, re.MULTILINE) + clean["title"] = heading.group(1).strip() if heading else re.sub(r"\.(md|markdown)$", "", filename, flags=re.I) + return dict(filename=filename, key=filename_key(filename), explicit_id=explicit_id, + metadata=clean, body=body) + + +def publish_post(repo, submission, manifest): + import json + incoming = json.loads((submission / "post.json").read_text()) + candidates = [] + scored = [] + records = [] + for path in sorted((repo / "_posts").glob("*")): + if path.suffix.lower() not in {".md", ".markdown"}: + continue + metadata, body = split_markdown(path.read_text()) + records.append((path, metadata)) + if incoming["explicit_id"]: + match = metadata.get("upload_id") == incoming["explicit_id"] or metadata.get("post_id") == incoming["explicit_id"] + else: + match = False + score = content_similarity(incoming["body"], body) + if score >= 0.84: + scored.append((score, path, metadata)) + if match: + candidates.append((path, metadata)) + if not incoming["explicit_id"] and scored: + scored.sort(key=lambda item: item[0], reverse=True) + if scored[0][0] >= 0.92: + if len(scored) > 1 and scored[0][0] - scored[1][0] < 0.08: + raise ValueError("Multiple posts have similar content; add the intended post_id to disambiguate") + candidates.append(scored[0][1:]) + if len(candidates) > 1: + raise ValueError("More than one post matches this post ID") + identity = incoming["explicit_id"] or manifest["id"] + if candidates: + entry, metadata = candidates[0] + identity = metadata.get("upload_id", identity) + # Keep the public URL and original publication date during updates. + metadata.update({key: value for key, value in incoming["metadata"].items() if key not in {"date", "permalink"}}) + permalink = metadata.get("permalink") or "/" + re.sub(r"^\d{4}-\d{2}-\d{2}-", "", entry.stem) + "/" + else: + metadata = dict(incoming["metadata"]) + metadata.setdefault("date", manifest["date"]) + slug = re.sub(r"[^\w-]+", "-", incoming["metadata"]["title"].lower()).strip("-")[:80] or "post" + permalink = metadata.get("permalink", f"/{slug}-{identity[:8]}/") + entry = repo / "_posts" / f"{str(metadata['date'])[:10]}-upload-{identity}.md" + if entry.exists(): + raise ValueError("Post identity conflicts with another file") + # Avoid taking over another page's URL, including non-post pages. + for other in list(repo.glob("*.md")) + list(repo.glob("*.html")): + other_metadata, _ = split_markdown(other.read_text()) + records.append((other, other_metadata)) + for other, other_metadata in records: + url = other_metadata.get("permalink") + if not url and other.parent.name == "_posts": + url = re.sub(r"^\d{4}-\d{2}-\d{2}-", "", other.stem) + if url and str(url).strip("/") == permalink.strip("/"): + raise ValueError("Another page already uses this permalink") + permalink = "/" + str(permalink).strip("/") + "/" + metadata.update(layout="post", permalink=permalink, upload_id=identity, + upload_filename=incoming["key"], upload_revision=manifest["id"], + render_with_liquid=False) + entry.parent.mkdir(parents=True, exist_ok=True) + entry.write_text("---\n" + yaml.safe_dump(metadata, allow_unicode=True, sort_keys=False) + + "---\n\n" + incoming["body"]) + return entry.relative_to(repo), permalink, "updated" if candidates else "created", identity + + +def content_similarity(left, right): + """Ignore whitespace; require substantial text before accepting a fuzzy match.""" + left = " ".join(unicodedata.normalize("NFC", left).split()) + right = " ".join(unicodedata.normalize("NFC", right).split()) + if left == right: + return 1.0 if len(left) >= 40 else 0.0 + if min(len(left), len(right)) < 200: + return 0.0 + # Size bound avoids expensive comparisons to obviously different posts. + if min(len(left), len(right)) / max(len(left), len(right)) < 0.84: + return 0.0 + matcher = SequenceMatcher(None, left.split(), right.split()) + if matcher.quick_ratio() < 0.84: + return 0.0 + return matcher.ratio() diff --git a/photo-upload/publisher.py b/photo-upload/publisher.py index 323617b..3446570 100644 --- a/photo-upload/publisher.py +++ b/photo-upload/publisher.py @@ -10,6 +10,7 @@ import sys import time from app import atomic_json +from posts import publish_post log = logging.getLogger("publisher") @@ -48,6 +49,7 @@ class Publisher: for job in jobs: self.state(job, "publishing") try: + outcomes = {} if not (self.repo / ".git").is_dir(): self.git("clone", "--branch", self.branch, "--single-branch", "--", self.remote, str(self.repo), directory=self.work) @@ -56,9 +58,31 @@ class Publisher: self.git("reset", "--hard", f"origin/{self.branch}") self.git("config", "user.name", os.environ.get("PHOTO_GIT_NAME", "Photo publisher")) self.git("config", "user.email", os.environ.get("PHOTO_GIT_EMAIL", "photos@theread.me")) + # Prepending oldest first leaves the newest snippet at the top. + jobs.sort(key=lambda job: (json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"], job)) for job in jobs: submission = self.data / "submissions" / job manifest = json.loads((submission / "manifest.json").read_text()) + if manifest.get("kind") == "post": + entry, url, action, identity = publish_post(self.repo, submission, manifest) + self.git("add", "--", str(entry)) + outcomes[job] = dict(url=url, action=action, post_id=identity) + continue + if manifest.get("kind") == "snippet": + entry = self.repo / "snippets.md" + existing = entry.read_text() + marker = "" + if marker not in existing: + raise OSError("snippets.md is missing its upload insertion marker") + if f'
' not in existing: + entry.write_text(existing.replace(marker, marker + "\n\n" + + (submission / "entry.md").read_text(), 1)) + images = Path("img/snippets/uploads") / job + if manifest["count"]: + shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True) + self.git("add", "--", str(images)) + self.git("add", "--", "snippets.md") + continue entry = Path("_art") / f"{manifest['date'][:10]}-photo-{job}.md" images = Path("img/arts/uploads") / job (self.repo / entry).parent.mkdir(parents=True, exist_ok=True) @@ -66,18 +90,30 @@ class Publisher: shutil.copytree(submission / "images", self.repo / images, dirs_exist_ok=True) self.git("add", "--", str(entry), str(images)) if self.git("diff", "--cached", "--name-only"): - self.git("commit", "-m", f"Publish {len(jobs)} photo submission(s)") + self.git("commit", "-m", f"Publish {len(jobs)} submission(s)") self.git("push", "origin", f"HEAD:refs/heads/{self.branch}") commit = self.git("rev-parse", "HEAD") for job in jobs: - self.state(job, "pushed", commit=commit) + self.state(job, "pushed", commit=commit, **outcomes.get(job, {})) + # A subsequent revision replaces the old one in the rendered site. + for job, outcome in outcomes.items(): + job_date = json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"] + for status_path in (self.data / "status").glob("*.json"): + previous = json.loads(status_path.read_text()) + if status_path.stem == job or previous.get("post_id") != outcome["post_id"]: + continue + old_manifest = self.data / "submissions" / status_path.stem / "manifest.json" + if old_manifest.exists() and json.loads(old_manifest.read_text())["date"] < job_date: + previous.update(status="superseded", replaced_by=job) + atomic_json(status_path, previous) log.info("Pushed %s submission(s) at %s", len(jobs), commit) - except (OSError, subprocess.SubprocessError) as error: + except (OSError, ValueError, subprocess.SubprocessError) as error: log.error("Git publication failed: %s", error) if isinstance(error, subprocess.CalledProcessError): log.error("%s", error.stderr) for job in jobs: - self.state(job, "failed", error="Git push failed; check publisher logs and retry.") + self.state(job, "failed", error=str(error) if isinstance(error, ValueError) else + "Git push failed; check publisher logs and retry.") raise def run(self, once=False): @@ -86,7 +122,7 @@ class Publisher: while True: try: self.publish() - except (OSError, subprocess.SubprocessError): + except (OSError, ValueError, subprocess.SubprocessError): if once: raise if once: diff --git a/photo-upload/requirements.txt b/photo-upload/requirements.txt index 0fdcd75..08b42b2 100644 --- a/photo-upload/requirements.txt +++ b/photo-upload/requirements.txt @@ -2,3 +2,4 @@ Flask==3.1.3 gunicorn==26.2.0 Pillow==12.3.0 pillow-heif==1.7.0 +PyYAML==6.0.3 diff --git a/photo-upload/tests/test_posts.py b/photo-upload/tests/test_posts.py new file mode 100644 index 0000000..ff51600 --- /dev/null +++ b/photo-upload/tests/test_posts.py @@ -0,0 +1,81 @@ +import io +import json + +from test_upload import AUTH, git, publisher, service +from posts import content_similarity + + +BODY = "\n\n".join(f"Paragraph {i}: I walked beside the river and considered how memory changes our understanding of a place. The details matter more than the names we give them." for i in range(12)) + + +def send(client, text, filename="essay.md"): + return client.post("/api/posts", headers=AUTH, data={"file": (io.BytesIO(text.encode()), filename)}) + + +def test_post_validation(service): + _, client, _ = service + assert client.post("/api/posts").status_code == 401 + assert client.post("/api/posts", headers=AUTH).status_code == 400 + assert send(client, BODY, "../escape.md").status_code == 400 + assert send(client, BODY, "essay.pdf").status_code == 400 + assert send(client, "---\ntitle: [bad\n---\nBody").status_code == 400 + assert send(client, "---\npost_id: ../../escape\n---\nBody").status_code == 400 + assert send(client, " ").status_code == 400 + assert send(client, "x" * (1024 * 1024 + 1)).status_code == 400 + + +def test_content_update_preserves_url_date_and_no_duplicate(publisher, service, monkeypatch, tmp_path): + run, remote, client = publisher + first_text = "---\ntitle: My walk\ndate: 2020-01-02\npermalink: /my-walk/\n---\n\n" + BODY + first = send(client, first_text) + assert first.status_code == 202 + assert run().returncode == 0 + status = client.get(first.json["status_url"], headers=AUTH).json + assert status["action"] == "created" and status["url"] == "/my-walk/" + second = send(client, "# My revised walk\n\n" + BODY.replace("beside the river", "along the river", 1), "renamed.md") + result = run() + assert result.returncode == 0, result.stderr + status2 = client.get(second.json["status_url"], headers=AUTH).json + assert status2["action"] == "updated" + assert status2["url"] == status["url"] + assert status2["post_id"] == status["post_id"] + paths = [p for p in git(remote, "ls-tree", "-r", "--name-only", "master").splitlines() if p.startswith("_posts/")] + assert len(paths) == 1 + saved = git(remote, "show", "master:" + paths[0]) + assert "2020-01-02" in saved and "along the river" in saved + assert "render_with_liquid: false" in saved + assert client.get(first.json["status_url"], headers=AUTH).json["status"] == "superseded" + receipt = tmp_path / "deployed.json" + monkeypatch.setenv("PHOTO_DEPLOYMENT_FILE", str(receipt)) + receipt.write_text(json.dumps({"posts": [second.json["id"]]})) + assert client.get(second.json["status_url"], headers=AUTH).json["status"] == "published" + assert send(client, first_text).status_code == 200 + + +def test_unrelated_same_filename_creates_new_post(publisher): + run, remote, client = publisher + send(client, BODY) + assert run().returncode == 0 + response = send(client, "# Another subject\n\n" + "Astronomy explores stars, galaxies and planetary orbits. " * 20) + assert run().returncode == 0 + assert client.get(response.json["status_url"], headers=AUTH).json["action"] == "created" + assert len([p for p in git(remote, "ls-tree", "-r", "--name-only", "master").splitlines() if p.startswith("_posts/")]) == 2 + + +def test_ambiguous_matches_fail_without_pushing(publisher): + run, remote, client = publisher + for identity in ("one", "two"): + send(client, f"---\npost_id: {identity}\ntitle: {identity}\n---\n" + BODY, identity + ".md") + assert run().returncode == 0 + head = git(remote, "rev-parse", "master") + response = send(client, BODY.replace("details", "small details", 1)) + assert run().returncode != 0 + state = client.get(response.json["status_url"], headers=AUTH).json + assert state["status"] == "failed" and "Multiple posts" in state["error"] + assert git(remote, "rev-parse", "master") == head + + +def test_similarity_requires_substantial_unambiguous_content(): + assert content_similarity("short shared title", "short shared title") == 0 + assert content_similarity(BODY, BODY.replace("river", "lake", 1)) >= 0.92 + assert content_similarity(BODY, "Completely unrelated writing.") == 0 diff --git a/photo-upload/tests/test_upload.py b/photo-upload/tests/test_upload.py index 70597e7..69f1c55 100644 --- a/photo-upload/tests/test_upload.py +++ b/photo-upload/tests/test_upload.py @@ -5,6 +5,7 @@ from pathlib import Path import subprocess import sys import os +import random import pytest from PIL import Image @@ -59,6 +60,16 @@ def test_validation_is_all_or_nothing(service): assert not list((data / "temporary").iterdir()) +def test_multi_megabyte_multipart_file(service): + _, client, _ = service + output = io.BytesIO() + Image.frombytes("RGB", (1800, 1800), random.Random(2).randbytes(1800 * 1800 * 3)).save( + output, "JPEG", quality=95) + assert len(output.getvalue()) > 3 * 1024 * 1024 + response = upload(client, [output.getvalue()]) + assert response.status_code == 202, response.json + + def test_album_caption_order_and_retry_deduplication(service): _, client, data = service caption = ' {% include secret %} {{ site.email }}\nsecond line' @@ -149,6 +160,7 @@ def publisher(tmp_path, service): git(source, "config", "user.name", "Test") git(source, "config", "user.email", "test@example.com") (source / "art.html").write_text("Initial site") + (source / "snippets.md").write_text("---\nlayout: post\n---\nIntroduction\n\n\n\nOld snippet\n") git(source, "add", ".") git(source, "commit", "-m", "Initial") git(source, "remote", "add", "origin", str(remote)) @@ -200,3 +212,47 @@ def test_rejected_push_can_retry(publisher): result = run() assert result.returncode == 0, result.stderr assert client.get(response.json["status_url"], headers=AUTH).json["status"] == "pushed" + + +def test_snippet_intake_validation_and_literal_text(service): + _, client, data = service + assert client.post("/api/snippets", data={"text": "hello"}).status_code == 401 + assert client.post("/api/snippets", headers=AUTH).status_code == 400 + assert client.post("/api/snippets", headers=AUTH, data={"text": "x" * 20001}).status_code == 400 + text = 'Book quote\nفارسی {% include secret %}\nhttps://example.com/?a=1&b=2' + response = client.post("/api/snippets", headers=AUTH, data={"text": text}) + assert response.status_code == 202 + job = response.json["id"] + assert response.json["url"] == f"/snippets/#snippet-{job}" + assert client.get(f"/api/photos/{job}", headers=AUTH).status_code == 404 + entry = (data / "submissions" / job / "entry.md").read_text() + assert '