diff --git a/photo-upload/README.md b/photo-upload/README.md index 1ccae1f..872246c 100644 --- a/photo-upload/README.md +++ b/photo-upload/README.md @@ -131,11 +131,16 @@ Create **Publish to blog**: 1. Enable **Show in Share Sheet**, accepting **Images**. 2. **Ask for Input** (Text): “Caption (optional)”. 3. Add another **Ask for Input** (Text): “Highlight name (optional)”. -4. **Make Archive** from **Shortcut Input**, format **ZIP**. Keep this output in +4. **Repeat with Each** item in **Shortcut Input**. Inside the repeat, use + **Get Details of Images** to get **Date Taken**, then **Format Date** using + **ISO 8601**. After **End Repeat**, **Combine Text** from **Repeat Results** + with **New Lines** and save it as **Photo Dates**. This preserves capture + dates even when Shortcuts strips them from the exported JPEG data. +5. **Make Archive** from **Shortcut Input**, format **ZIP**. Keep this output in a variable named **Photo Archive**. This packages every selected image into one upload; using an image list directly in a File form field can send only the first image on some Shortcuts versions. -5. **Get Contents of URL**: `https://theread.me/api/photos`. +6. **Get Contents of URL**: `https://theread.me/api/photos`. - Method: **POST**. - Header: `Authorization` = `Bearer YOUR_TOKEN`. - Request Body: **Form**. @@ -143,8 +148,9 @@ Create **Publish to blog**: - `caption`: type **Text**, value the caption response. - `highlight`: type **Text**, value the highlight-name response. Select each prompt's specific output variable so the caption and highlight stay separate. + - `photo_dates`: type **Text**, value **Photo Dates**. - Do not set Content-Type; Shortcuts supplies the multipart boundary. -6. Show the actual response and check its `count` against the number of selected +7. Show the actual response and check its `count` against the number of selected images. Only show “Uploaded; publishing” when there is no `error`. Optionally poll the returned `status_url` with the same header. `pushed` means “GitHub Actions is building/deploying”; only `published` means the deployed site's receipt contains this album. @@ -157,8 +163,14 @@ Older batch documents are still split at render time for backward compatibility. Metadata sidecars under `__MACOSX/` and `.DS_Store` are ignored. Ordinary API clients can also send repeated `photos` parts; send either `archive` or `photos`, never both. -Select still photos; videos/Live Photo video components aren't supported. Dates -are upload time in UTC; captions are plain text. A caption is optional. +Select still photos; videos/Live Photo video components aren't supported. The +service reads EXIF `DateTimeOriginal` automatically when it is present. Because +iOS Shortcuts can blank that EXIF field while creating the ZIP, `photo_dates` +supplies the same Photos-library metadata explicitly, one ISO 8601 value per +image in archive order. An upload without either source is rejected instead of +being placed under the upload year. Re-sharing the same images, caption, and +highlight with dates replaces the earlier undated upload. Captions are plain +text and optional. An optional highlight name (up to 200 characters) adds one link to **Highlights**, pointing to the first photo in the batch. Leave it blank for an ordinary upload. diff --git a/photo-upload/app.py b/photo-upload/app.py index dda630f..7286b84 100644 --- a/photo-upload/app.py +++ b/photo-upload/app.py @@ -15,7 +15,7 @@ import warnings import zipfile from flask import Flask, abort, jsonify, request -from PIL import Image, ImageCms, ImageOps, UnidentifiedImageError +from PIL import ExifTags, Image, ImageCms, ImageOps, UnidentifiedImageError from pillow_heif import register_heif_opener from werkzeug.exceptions import HTTPException from posts import parse_upload @@ -27,6 +27,71 @@ GALLERY_MAX_BYTES = 300 * 1024 warnings.simplefilter("error", Image.DecompressionBombWarning) +def exif_photo_date(exif): + """Return the original capture time without exposing or retaining other metadata.""" + try: + nested = exif.get_ifd(ExifTags.IFD.Exif) + except (AttributeError, KeyError, TypeError, ValueError): + nested = {} + + def value(tag): + candidate = nested.get(tag, exif.get(tag)) + if isinstance(candidate, bytes): + candidate = candidate.decode("ascii", "ignore") + return str(candidate or "").strip(" \0") + + # Prefer when the shutter fired, then digitization time, then the generic + # image timestamp. Offset and subseconds belong to their matching date tag. + for date_tag, offset_tag, subsecond_tag in ( + (36867, 36881, 37521), (36868, 36882, 37522), (306, 36880, 37520)): + raw_date = value(date_tag) + if not raw_date: + continue + try: + taken = datetime.strptime(raw_date, "%Y:%m:%d %H:%M:%S") + except ValueError: + continue + subseconds = re.sub(r"\D", "", value(subsecond_tag))[:6] + if subseconds: + taken = taken.replace(microsecond=int(subseconds.ljust(6, "0"))) + offset = value(offset_tag) + if re.fullmatch(r"[+-]\d{2}:\d{2}", offset): + sign = 1 if offset[0] == "+" else -1 + hours, minutes = map(int, offset[1:].split(":")) + taken = taken.replace(tzinfo=timezone(sign * timedelta(hours=hours, minutes=minutes))) + return taken + return None + + +def submitted_photo_dates(raw, count): + """Parse newline-separated ISO dates supplied from the Photos library.""" + if not raw.strip(): + return None + lines = [line.strip() for line in raw.splitlines() if line.strip()] + if len(lines) != count: + raise ValueError("photo_dates must contain one ISO date per photo") + dates = [] + for line in lines: + try: + dates.append(datetime.fromisoformat(line.replace("Z", "+00:00"))) + except ValueError as error: + raise ValueError("photo_dates must contain valid ISO dates") from error + return dates + + +def distinct_photo_dates(dates): + """Keep exact capture times, using earlier seconds only to break true ties.""" + result = [] + used = set() + for taken in dates: + candidate = taken + while candidate.isoformat() in used: + candidate -= timedelta(seconds=1) + result.append(candidate) + used.add(candidate.isoformat()) + return result + + def atomic_json(path, value): fd, temporary = tempfile.mkstemp(dir=path.parent) try: @@ -48,6 +113,7 @@ def web_image(raw, destination): with Image.open(io.BytesIO(raw)) as original: if original.format not in {"JPEG", "PNG", "WEBP", "HEIF"}: raise ValueError("Use JPEG, PNG, WebP, or HEIC photos") + taken_at = exif_photo_date(original.getexif()) image = ImageOps.exif_transpose(original) image.thumbnail((GALLERY_MAX_EDGE, GALLERY_MAX_EDGE), Image.Resampling.LANCZOS) if image.info.get("icc_profile"): @@ -74,17 +140,14 @@ def web_image(raw, destination): clean.save(output, "JPEG", quality=quality, optimize=True, progressive=True) if output.tell() <= GALLERY_MAX_BYTES: Path(destination).write_bytes(output.getvalue()) - return + return taken_at clean.thumbnail((max(1, int(clean.width * 0.85)), max(1, int(clean.height * 0.85))), Image.Resampling.LANCZOS) -def photo_entry(job, date, index, count, caption, highlight): - # Give each photo its own collection document. Descending dates preserve the - # selection order because the gallery renders the newest document first. - # Jekyll's collection ordering only preserves whole seconds. Give each item - # a distinct second so filenames cannot disturb the selected photo order. - item_date = datetime.fromisoformat(date) + timedelta(seconds=count - index) +def photo_entry(job, date, index, caption, highlight): + # Each photo carries its own capture date so the gallery is chronological. + item_date = datetime.fromisoformat(date) title = f"photo-{job}" if index == 0 else f"photo-{job}-{index:02}" content = ["---", "layout: post", f'title: "{title}"', f'date: "{item_date.isoformat()}"', "categories: art"] @@ -255,6 +318,11 @@ def create_app(data_dir=None, token=None): highlight = request.form.get("highlight", "").strip() if len(highlight) > 200: abort(400, "Highlight name must be at most 200 characters") + try: + supplied_dates = (submitted_photo_dates(request.form.get("photo_dates", ""), len(photos)) + if kind == "photo" else None) + except ValueError as error: + abort(400, str(error)) # Ordered byte hashes + caption make retries idempotent, even across restarts. digest = hashlib.sha256(caption.encode()) if kind == "snippet": @@ -263,12 +331,15 @@ def create_app(data_dir=None, token=None): try: (temporary / "originals").mkdir() (temporary / "images").mkdir() + capture_dates = [] for index, photo in enumerate(photos): raw = photo if isinstance(photo, bytes) else photo.read() digest.update(hashlib.sha256(raw).digest()) (temporary / "originals" / str(index)).write_bytes(raw) try: - web_image(raw, temporary / "images" / f"{index:02}.jpg") + embedded_date = web_image(raw, temporary / "images" / f"{index:02}.jpg") + if kind == "photo": + capture_dates.append(supplied_dates[index] if supplied_dates else embedded_date) except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning, ImageCms.PyCMSError): @@ -276,18 +347,30 @@ def create_app(data_dir=None, token=None): # Preserve existing IDs when omitted; named highlights participate in deduplication. if highlight: digest.update(b"\0highlight\0" + highlight.encode()) + undated_job = digest.hexdigest() + if supplied_dates: + normalized_dates = "\n".join(taken.isoformat() for taken in supplied_dates) + digest.update(b"\0photo_dates\0" + normalized_dates.encode()) job = digest.hexdigest() date = datetime.now(timezone.utc).isoformat() manifest = {"id": job, "date": date, "caption": caption, "highlight": highlight, "count": len(photos)} + if (supplied_dates and undated_job != job and + (data / "submissions" / undated_job / "manifest.json").is_file()): + manifest["replaces"] = undated_job if kind == "snippet": manifest.update(kind=kind, text=text) atomic_json(temporary / "manifest.json", manifest) if kind == "photo": + if any(taken is None for taken in capture_dates): + abort(400, "Photo capture date is missing; send photo_dates from the Photos library") + capture_dates = distinct_photo_dates(capture_dates) + manifest["capture_dates"] = [taken.isoformat() for taken in capture_dates] + atomic_json(temporary / "manifest.json", manifest) (temporary / "entries").mkdir() for index in range(len(photos)): (temporary / "entries" / f"{index:02}.md").write_text( - photo_entry(job, date, index, len(photos), caption, highlight)) + photo_entry(job, capture_dates[index].isoformat(), index, caption, highlight)) # Retain the old filename for compatibility with publisher versions # that predate one-document-per-photo submissions. content = [(temporary / "entries/00.md").read_text()] diff --git a/photo-upload/publisher.py b/photo-upload/publisher.py index 22a14f0..01a263d 100644 --- a/photo-upload/publisher.py +++ b/photo-upload/publisher.py @@ -4,6 +4,7 @@ import json import logging import os from pathlib import Path +import re import shutil import subprocess import sys @@ -15,6 +16,13 @@ from posts import publish_post log = logging.getLogger("publisher") +def entry_date(path): + match = re.search(r'^date: "(\d{4}-\d{2}-\d{2})', path.read_text(), re.MULTILINE) + if not match: + raise ValueError(f"Photo entry has no valid date: {path.name}") + return match.group(1) + + class Publisher: def __init__(self): self.data = Path(os.environ.get("PHOTO_DATA", "/data")) @@ -50,6 +58,7 @@ class Publisher: self.state(job, "publishing") try: outcomes = {} + replaced_photos = {} if not (self.repo / ".git").is_dir(): self.git("clone", "--branch", self.branch, "--single-branch", "--", self.remote, str(self.repo), directory=self.work) @@ -85,6 +94,14 @@ class Publisher: continue entry = Path("_art") / f"{manifest['date'][:10]}-photo-{job}.md" images = Path("img/arts/uploads") / job + replaces = manifest.get("replaces", "") + if re.fullmatch(r"[a-f0-9]{64}", replaces) and replaces != job: + for previous in (self.repo / "_art").glob(f"*-photo-{replaces}*.md"): + self.git("rm", "--ignore-unmatch", "--", + str(previous.relative_to(self.repo))) + previous_images = Path("img/arts/uploads") / replaces + self.git("rm", "-r", "--ignore-unmatch", "--", str(previous_images)) + replaced_photos[job] = replaces (self.repo / entry).parent.mkdir(parents=True, exist_ok=True) entries = submission / "entries" if entries.is_dir(): @@ -92,7 +109,7 @@ class Publisher: # is republished by a newer service version. self.git("rm", "--ignore-unmatch", "--", str(entry)) for source in sorted(entries.glob("*.md")): - destination = Path("_art") / f"{manifest['date'][:10]}-photo-{job}-{source.stem}.md" + destination = Path("_art") / f"{entry_date(source)}-photo-{job}-{source.stem}.md" shutil.copyfile(source, self.repo / destination) self.git("add", "--", str(destination)) else: @@ -106,6 +123,8 @@ class Publisher: commit = self.git("rev-parse", "HEAD") for job in jobs: self.state(job, "pushed", commit=commit, **outcomes.get(job, {})) + if job in replaced_photos: + self.state(replaced_photos[job], "superseded", replaced_by=job) # A subsequent revision replaces the old one in the rendered site. for job, outcome in outcomes.items(): job_date = json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"] diff --git a/photo-upload/tests/test_upload.py b/photo-upload/tests/test_upload.py index c86a4b4..28e882a 100644 --- a/photo-upload/tests/test_upload.py +++ b/photo-upload/tests/test_upload.py @@ -26,6 +26,10 @@ def service(tmp_path): def photo(color="red", format="JPEG", size=(40, 20), exif=None): + if exif is None: + exif = Image.Exif() + exif[36867] = "2025:01:02 03:04:05" + exif[36881] = "+00:00" output = io.BytesIO() Image.new("RGB", size, color).save(output, format, **({"exif": exif} if exif else {})) return output.getvalue() @@ -64,8 +68,10 @@ def test_validation_is_all_or_nothing(service): def test_multi_megabyte_multipart_file(service): _, client, data = service output = io.BytesIO() + exif = Image.Exif() + exif[36867] = "2025:01:02 03:04:05" Image.frombytes("RGB", (1800, 1800), random.Random(2).randbytes(1800 * 1800 * 3)).save( - output, "JPEG", quality=95) + output, "JPEG", quality=95, exif=exif) assert len(output.getvalue()) > 3 * 1024 * 1024 response = upload(client, [output.getvalue()]) assert response.status_code == 202, response.json @@ -138,6 +144,7 @@ def test_orientation_metadata_and_heic(service): exif = Image.Exif() exif[274] = 6 exif[270] = "private metadata" + exif[36867] = "2025:01:02 03:04:05" for raw in [photo(exif=exif, size=(3000, 1500)), photo(format="HEIF")]: response = upload(client, [raw]) assert response.status_code == 202 @@ -149,6 +156,61 @@ def test_orientation_metadata_and_heic(service): assert image.height > image.width +def test_photo_date_comes_from_original_metadata(service): + _, client, data = service + exif = Image.Exif() + exif[36867] = "2025:04:18 14:23:11" + exif[36881] = "+02:00" + + response = upload(client, [photo(exif=exif)]) + + assert response.status_code == 202 + entry = (data / "submissions" / response.json["id"] / "entries/00.md").read_text() + assert 'date: "2025-04-18T14:23:11+02:00"' in entry + + +def test_photo_dates_can_come_from_photos_library_metadata(service): + _, client, data = service + response = client.post("/api/photos", headers=AUTH, data={ + "photos": [ + (io.BytesIO(photo("red")), "first.jpg"), + (io.BytesIO(photo("blue")), "second.jpg"), + ], + "photo_dates": "2025-04-18T14:23:11+02:00\n2025-04-19T09:10:05+02:00", + }) + assert response.status_code == 202 + entries = sorted((data / "submissions" / response.json["id"] / "entries").glob("*.md")) + assert 'date: "2025-04-18T14:23:11+02:00"' in entries[0].read_text() + assert 'date: "2025-04-19T09:10:05+02:00"' in entries[1].read_text() + + invalid = client.post("/api/photos", headers=AUTH, data={ + "photos": [(io.BytesIO(photo()), "photo.jpg")], + "photo_dates": "not-a-date", + }) + assert invalid.status_code == 400 + assert "valid ISO dates" in invalid.json["error"] + + missing = upload(client, [photo(exif=Image.Exif())]) + assert missing.status_code == 400 + assert "capture date is missing" in missing.json["error"] + + +def test_dated_retry_replaces_the_same_undated_upload(service): + _, client, data = service + raw = photo() + original = upload(client, [raw], "Same caption", highlight="Trip").json + replacement = client.post("/api/photos", headers=AUTH, data={ + "photos": [(io.BytesIO(raw), "photo.jpg")], + "caption": "Same caption", + "highlight": "Trip", + "photo_dates": "2025-04-18T14:23:11+02:00", + }).json + + assert replacement["id"] != original["id"] + manifest = json.loads((data / "submissions" / replacement["id"] / "manifest.json").read_text()) + assert manifest["replaces"] == original["id"] + + def test_optional_highlight_preserves_existing_upload_ids(service): _, client, data = service raw = photo() @@ -239,6 +301,40 @@ def test_git_push_and_crash_recovery(publisher, service, monkeypatch, tmp_path): assert client.get(response.json["status_url"], headers=AUTH).json["status"] == "published" +def test_publisher_names_photo_entry_from_capture_date(publisher, service): + run, remote, client = publisher + exif = Image.Exif() + exif[36867] = "2025:04:18 14:23:11" + response = upload(client, [photo(exif=exif)]) + + result = run() + + assert result.returncode == 0, result.stderr + paths = git(remote, "ls-tree", "-r", "--name-only", "master") + assert f'_art/2025-04-18-photo-{response.json["id"]}-00.md' in paths + + +def test_publisher_replaces_an_undated_version_of_the_same_photos(publisher, service): + run, remote, client = publisher + raw = photo() + original = upload(client, [raw], "Same caption", highlight="Trip").json + assert run().returncode == 0 + replacement = client.post("/api/photos", headers=AUTH, data={ + "photos": [(io.BytesIO(raw), "photo.jpg")], + "caption": "Same caption", + "highlight": "Trip", + "photo_dates": "2025-04-18T14:23:11+02:00", + }).json + + result = run() + + assert result.returncode == 0, result.stderr + paths = git(remote, "ls-tree", "-r", "--name-only", "master") + assert f"photo-{original['id']}" not in paths + assert f"img/arts/uploads/{original['id']}" not in paths + assert f'_art/2025-04-18-photo-{replacement["id"]}-00.md' in paths + + def test_rejected_push_can_retry(publisher): run, remote, client = publisher head = git(remote, "rev-parse", "master")