Date photo uploads from capture metadata

photo-upload/README.md

@@ -131,11 +131,16 @@ Create **Publish to blog**:
 1. Enable **Show in Share Sheet**, accepting **Images**.
 2. **Ask for Input** (Text): “Caption (optional)”.
 3. Add another **Ask for Input** (Text): “Highlight name (optional)”.
-4. **Make Archive** from **Shortcut Input**, format **ZIP**. Keep this output in
+4. **Repeat with Each** item in **Shortcut Input**. Inside the repeat, use
+   **Get Details of Images** to get **Date Taken**, then **Format Date** using
+   **ISO 8601**. After **End Repeat**, **Combine Text** from **Repeat Results**
+   with **New Lines** and save it as **Photo Dates**. This preserves capture
+   dates even when Shortcuts strips them from the exported JPEG data.
+5. **Make Archive** from **Shortcut Input**, format **ZIP**. Keep this output in
    a variable named **Photo Archive**. This packages every selected image into
    one upload; using an image list directly in a File form field can send only
    the first image on some Shortcuts versions.
-5. **Get Contents of URL**: `https://theread.me/api/photos`.
+6. **Get Contents of URL**: `https://theread.me/api/photos`.
    - Method: **POST**.
    - Header: `Authorization` = `Bearer YOUR_TOKEN`.
    - Request Body: **Form**.
@@ -143,8 +148,9 @@ Create **Publish to blog**:
    - `caption`: type **Text**, value the caption response.
    - `highlight`: type **Text**, value the highlight-name response. Select each
      prompt's specific output variable so the caption and highlight stay separate.
+   - `photo_dates`: type **Text**, value **Photo Dates**.
    - Do not set Content-Type; Shortcuts supplies the multipart boundary.
-6. Show the actual response and check its `count` against the number of selected
+7. Show the actual response and check its `count` against the number of selected
    images. Only show “Uploaded; publishing” when there is no `error`. Optionally poll the returned `status_url`
    with the same header. `pushed` means “GitHub Actions is building/deploying”;
    only `published` means the deployed site's receipt contains this album.
@@ -157,8 +163,14 @@ Older batch documents are still split at render time for backward compatibility.
 Metadata sidecars under `__MACOSX/` and `.DS_Store` are ignored. Ordinary API
 clients can also send repeated `photos` parts; send either `archive` or `photos`,
 never both.
-Select still photos; videos/Live Photo video components aren't supported. Dates
-are upload time in UTC; captions are plain text. A caption is optional.
+Select still photos; videos/Live Photo video components aren't supported. The
+service reads EXIF `DateTimeOriginal` automatically when it is present. Because
+iOS Shortcuts can blank that EXIF field while creating the ZIP, `photo_dates`
+supplies the same Photos-library metadata explicitly, one ISO 8601 value per
+image in archive order. An upload without either source is rejected instead of
+being placed under the upload year. Re-sharing the same images, caption, and
+highlight with dates replaces the earlier undated upload. Captions are plain
+text and optional.
 An optional highlight name (up to 200 characters) adds one link to **Highlights**,
 pointing to the first photo in the batch. Leave it blank for an ordinary upload.
 

photo-upload/app.py

@@ -15,7 +15,7 @@ import warnings
 import zipfile
 
 from flask import Flask, abort, jsonify, request
-from PIL import Image, ImageCms, ImageOps, UnidentifiedImageError
+from PIL import ExifTags, Image, ImageCms, ImageOps, UnidentifiedImageError
 from pillow_heif import register_heif_opener
 from werkzeug.exceptions import HTTPException
 from posts import parse_upload
@@ -27,6 +27,71 @@ GALLERY_MAX_BYTES = 300 * 1024
 warnings.simplefilter("error", Image.DecompressionBombWarning)
 
 
+def exif_photo_date(exif):
+    """Return the original capture time without exposing or retaining other metadata."""
+    try:
+        nested = exif.get_ifd(ExifTags.IFD.Exif)
+    except (AttributeError, KeyError, TypeError, ValueError):
+        nested = {}
+
+    def value(tag):
+        candidate = nested.get(tag, exif.get(tag))
+        if isinstance(candidate, bytes):
+            candidate = candidate.decode("ascii", "ignore")
+        return str(candidate or "").strip(" \0")
+
+    # Prefer when the shutter fired, then digitization time, then the generic
+    # image timestamp. Offset and subseconds belong to their matching date tag.
+    for date_tag, offset_tag, subsecond_tag in (
+            (36867, 36881, 37521), (36868, 36882, 37522), (306, 36880, 37520)):
+        raw_date = value(date_tag)
+        if not raw_date:
+            continue
+        try:
+            taken = datetime.strptime(raw_date, "%Y:%m:%d %H:%M:%S")
+        except ValueError:
+            continue
+        subseconds = re.sub(r"\D", "", value(subsecond_tag))[:6]
+        if subseconds:
+            taken = taken.replace(microsecond=int(subseconds.ljust(6, "0")))
+        offset = value(offset_tag)
+        if re.fullmatch(r"[+-]\d{2}:\d{2}", offset):
+            sign = 1 if offset[0] == "+" else -1
+            hours, minutes = map(int, offset[1:].split(":"))
+            taken = taken.replace(tzinfo=timezone(sign * timedelta(hours=hours, minutes=minutes)))
+        return taken
+    return None
+
+
+def submitted_photo_dates(raw, count):
+    """Parse newline-separated ISO dates supplied from the Photos library."""
+    if not raw.strip():
+        return None
+    lines = [line.strip() for line in raw.splitlines() if line.strip()]
+    if len(lines) != count:
+        raise ValueError("photo_dates must contain one ISO date per photo")
+    dates = []
+    for line in lines:
+        try:
+            dates.append(datetime.fromisoformat(line.replace("Z", "+00:00")))
+        except ValueError as error:
+            raise ValueError("photo_dates must contain valid ISO dates") from error
+    return dates
+
+
+def distinct_photo_dates(dates):
+    """Keep exact capture times, using earlier seconds only to break true ties."""
+    result = []
+    used = set()
+    for taken in dates:
+        candidate = taken
+        while candidate.isoformat() in used:
+            candidate -= timedelta(seconds=1)
+        result.append(candidate)
+        used.add(candidate.isoformat())
+    return result
+
+
 def atomic_json(path, value):
     fd, temporary = tempfile.mkstemp(dir=path.parent)
     try:
@@ -48,6 +113,7 @@ def web_image(raw, destination):
     with Image.open(io.BytesIO(raw)) as original:
         if original.format not in {"JPEG", "PNG", "WEBP", "HEIF"}:
             raise ValueError("Use JPEG, PNG, WebP, or HEIC photos")
+        taken_at = exif_photo_date(original.getexif())
         image = ImageOps.exif_transpose(original)
         image.thumbnail((GALLERY_MAX_EDGE, GALLERY_MAX_EDGE), Image.Resampling.LANCZOS)
         if image.info.get("icc_profile"):
@@ -74,17 +140,14 @@ def web_image(raw, destination):
                 clean.save(output, "JPEG", quality=quality, optimize=True, progressive=True)
                 if output.tell() <= GALLERY_MAX_BYTES:
                     Path(destination).write_bytes(output.getvalue())
-                    return
+                    return taken_at
             clean.thumbnail((max(1, int(clean.width * 0.85)), max(1, int(clean.height * 0.85))),
                             Image.Resampling.LANCZOS)
 
 
-def photo_entry(job, date, index, count, caption, highlight):
-    # Give each photo its own collection document. Descending dates preserve the
-    # selection order because the gallery renders the newest document first.
-    # Jekyll's collection ordering only preserves whole seconds. Give each item
-    # a distinct second so filenames cannot disturb the selected photo order.
-    item_date = datetime.fromisoformat(date) + timedelta(seconds=count - index)
+def photo_entry(job, date, index, caption, highlight):
+    # Each photo carries its own capture date so the gallery is chronological.
+    item_date = datetime.fromisoformat(date)
     title = f"photo-{job}" if index == 0 else f"photo-{job}-{index:02}"
     content = ["---", "layout: post", f'title: "{title}"',
                f'date: "{item_date.isoformat()}"', "categories: art"]
@@ -255,6 +318,11 @@ def create_app(data_dir=None, token=None):
         highlight = request.form.get("highlight", "").strip()
         if len(highlight) > 200:
             abort(400, "Highlight name must be at most 200 characters")
+        try:
+            supplied_dates = (submitted_photo_dates(request.form.get("photo_dates", ""), len(photos))
+                              if kind == "photo" else None)
+        except ValueError as error:
+            abort(400, str(error))
         # Ordered byte hashes + caption make retries idempotent, even across restarts.
         digest = hashlib.sha256(caption.encode())
         if kind == "snippet":
@@ -263,12 +331,15 @@ def create_app(data_dir=None, token=None):
         try:
             (temporary / "originals").mkdir()
             (temporary / "images").mkdir()
+            capture_dates = []
             for index, photo in enumerate(photos):
                 raw = photo if isinstance(photo, bytes) else photo.read()
                 digest.update(hashlib.sha256(raw).digest())
                 (temporary / "originals" / str(index)).write_bytes(raw)
                 try:
-                    web_image(raw, temporary / "images" / f"{index:02}.jpg")
+                    embedded_date = web_image(raw, temporary / "images" / f"{index:02}.jpg")
+                    if kind == "photo":
+                        capture_dates.append(supplied_dates[index] if supplied_dates else embedded_date)
                 except (UnidentifiedImageError, OSError, ValueError, SyntaxError,
                         Image.DecompressionBombError, Image.DecompressionBombWarning,
                         ImageCms.PyCMSError):
@@ -276,18 +347,30 @@ def create_app(data_dir=None, token=None):
             # Preserve existing IDs when omitted; named highlights participate in deduplication.
             if highlight:
                 digest.update(b"\0highlight\0" + highlight.encode())
+            undated_job = digest.hexdigest()
+            if supplied_dates:
+                normalized_dates = "\n".join(taken.isoformat() for taken in supplied_dates)
+                digest.update(b"\0photo_dates\0" + normalized_dates.encode())
             job = digest.hexdigest()
             date = datetime.now(timezone.utc).isoformat()
             manifest = {"id": job, "date": date, "caption": caption,
                         "highlight": highlight, "count": len(photos)}
+            if (supplied_dates and undated_job != job and
+                    (data / "submissions" / undated_job / "manifest.json").is_file()):
+                manifest["replaces"] = undated_job
             if kind == "snippet":
                 manifest.update(kind=kind, text=text)
             atomic_json(temporary / "manifest.json", manifest)
             if kind == "photo":
+                if any(taken is None for taken in capture_dates):
+                    abort(400, "Photo capture date is missing; send photo_dates from the Photos library")
+                capture_dates = distinct_photo_dates(capture_dates)
+                manifest["capture_dates"] = [taken.isoformat() for taken in capture_dates]
+                atomic_json(temporary / "manifest.json", manifest)
                 (temporary / "entries").mkdir()
                 for index in range(len(photos)):
                     (temporary / "entries" / f"{index:02}.md").write_text(
-                        photo_entry(job, date, index, len(photos), caption, highlight))
+                        photo_entry(job, capture_dates[index].isoformat(), index, caption, highlight))
                 # Retain the old filename for compatibility with publisher versions
                 # that predate one-document-per-photo submissions.
                 content = [(temporary / "entries/00.md").read_text()]

photo-upload/publisher.py

@@ -4,6 +4,7 @@ import json
 import logging
 import os
 from pathlib import Path
+import re
 import shutil
 import subprocess
 import sys
@@ -15,6 +16,13 @@ from posts import publish_post
 log = logging.getLogger("publisher")
 
 
+def entry_date(path):
+    match = re.search(r'^date: "(\d{4}-\d{2}-\d{2})', path.read_text(), re.MULTILINE)
+    if not match:
+        raise ValueError(f"Photo entry has no valid date: {path.name}")
+    return match.group(1)
+
+
 class Publisher:
     def __init__(self):
         self.data = Path(os.environ.get("PHOTO_DATA", "/data"))
@@ -50,6 +58,7 @@ class Publisher:
             self.state(job, "publishing")
         try:
             outcomes = {}
+            replaced_photos = {}
             if not (self.repo / ".git").is_dir():
                 self.git("clone", "--branch", self.branch, "--single-branch", "--",
                          self.remote, str(self.repo), directory=self.work)
@@ -85,6 +94,14 @@ class Publisher:
                     continue
                 entry = Path("_art") / f"{manifest['date'][:10]}-photo-{job}.md"
                 images = Path("img/arts/uploads") / job
+                replaces = manifest.get("replaces", "")
+                if re.fullmatch(r"[a-f0-9]{64}", replaces) and replaces != job:
+                    for previous in (self.repo / "_art").glob(f"*-photo-{replaces}*.md"):
+                        self.git("rm", "--ignore-unmatch", "--",
+                                 str(previous.relative_to(self.repo)))
+                    previous_images = Path("img/arts/uploads") / replaces
+                    self.git("rm", "-r", "--ignore-unmatch", "--", str(previous_images))
+                    replaced_photos[job] = replaces
                 (self.repo / entry).parent.mkdir(parents=True, exist_ok=True)
                 entries = submission / "entries"
                 if entries.is_dir():
@@ -92,7 +109,7 @@ class Publisher:
                     # is republished by a newer service version.
                     self.git("rm", "--ignore-unmatch", "--", str(entry))
                     for source in sorted(entries.glob("*.md")):
-                        destination = Path("_art") / f"{manifest['date'][:10]}-photo-{job}-{source.stem}.md"
+                        destination = Path("_art") / f"{entry_date(source)}-photo-{job}-{source.stem}.md"
                         shutil.copyfile(source, self.repo / destination)
                         self.git("add", "--", str(destination))
                 else:
@@ -106,6 +123,8 @@ class Publisher:
             commit = self.git("rev-parse", "HEAD")
             for job in jobs:
                 self.state(job, "pushed", commit=commit, **outcomes.get(job, {}))
+                if job in replaced_photos:
+                    self.state(replaced_photos[job], "superseded", replaced_by=job)
             # A subsequent revision replaces the old one in the rendered site.
             for job, outcome in outcomes.items():
                 job_date = json.loads((self.data / "submissions" / job / "manifest.json").read_text())["date"]

photo-upload/tests/test_upload.py

@@ -26,6 +26,10 @@ def service(tmp_path):
 
 
 def photo(color="red", format="JPEG", size=(40, 20), exif=None):
+    if exif is None:
+        exif = Image.Exif()
+        exif[36867] = "2025:01:02 03:04:05"
+        exif[36881] = "+00:00"
     output = io.BytesIO()
     Image.new("RGB", size, color).save(output, format, **({"exif": exif} if exif else {}))
     return output.getvalue()
@@ -64,8 +68,10 @@ def test_validation_is_all_or_nothing(service):
 def test_multi_megabyte_multipart_file(service):
     _, client, data = service
     output = io.BytesIO()
+    exif = Image.Exif()
+    exif[36867] = "2025:01:02 03:04:05"
     Image.frombytes("RGB", (1800, 1800), random.Random(2).randbytes(1800 * 1800 * 3)).save(
-        output, "JPEG", quality=95)
+        output, "JPEG", quality=95, exif=exif)
     assert len(output.getvalue()) > 3 * 1024 * 1024
     response = upload(client, [output.getvalue()])
     assert response.status_code == 202, response.json
@@ -138,6 +144,7 @@ def test_orientation_metadata_and_heic(service):
     exif = Image.Exif()
     exif[274] = 6
     exif[270] = "private metadata"
+    exif[36867] = "2025:01:02 03:04:05"
     for raw in [photo(exif=exif, size=(3000, 1500)), photo(format="HEIF")]:
         response = upload(client, [raw])
         assert response.status_code == 202
@@ -149,6 +156,61 @@ def test_orientation_metadata_and_heic(service):
                 assert image.height > image.width
 
 
+def test_photo_date_comes_from_original_metadata(service):
+    _, client, data = service
+    exif = Image.Exif()
+    exif[36867] = "2025:04:18 14:23:11"
+    exif[36881] = "+02:00"
+
+    response = upload(client, [photo(exif=exif)])
+
+    assert response.status_code == 202
+    entry = (data / "submissions" / response.json["id"] / "entries/00.md").read_text()
+    assert 'date: "2025-04-18T14:23:11+02:00"' in entry
+
+
+def test_photo_dates_can_come_from_photos_library_metadata(service):
+    _, client, data = service
+    response = client.post("/api/photos", headers=AUTH, data={
+        "photos": [
+            (io.BytesIO(photo("red")), "first.jpg"),
+            (io.BytesIO(photo("blue")), "second.jpg"),
+        ],
+        "photo_dates": "2025-04-18T14:23:11+02:00\n2025-04-19T09:10:05+02:00",
+    })
+    assert response.status_code == 202
+    entries = sorted((data / "submissions" / response.json["id"] / "entries").glob("*.md"))
+    assert 'date: "2025-04-18T14:23:11+02:00"' in entries[0].read_text()
+    assert 'date: "2025-04-19T09:10:05+02:00"' in entries[1].read_text()
+
+    invalid = client.post("/api/photos", headers=AUTH, data={
+        "photos": [(io.BytesIO(photo()), "photo.jpg")],
+        "photo_dates": "not-a-date",
+    })
+    assert invalid.status_code == 400
+    assert "valid ISO dates" in invalid.json["error"]
+
+    missing = upload(client, [photo(exif=Image.Exif())])
+    assert missing.status_code == 400
+    assert "capture date is missing" in missing.json["error"]
+
+
+def test_dated_retry_replaces_the_same_undated_upload(service):
+    _, client, data = service
+    raw = photo()
+    original = upload(client, [raw], "Same caption", highlight="Trip").json
+    replacement = client.post("/api/photos", headers=AUTH, data={
+        "photos": [(io.BytesIO(raw), "photo.jpg")],
+        "caption": "Same caption",
+        "highlight": "Trip",
+        "photo_dates": "2025-04-18T14:23:11+02:00",
+    }).json
+
+    assert replacement["id"] != original["id"]
+    manifest = json.loads((data / "submissions" / replacement["id"] / "manifest.json").read_text())
+    assert manifest["replaces"] == original["id"]
+
+
 def test_optional_highlight_preserves_existing_upload_ids(service):
     _, client, data = service
     raw = photo()
@@ -239,6 +301,40 @@ def test_git_push_and_crash_recovery(publisher, service, monkeypatch, tmp_path):
     assert client.get(response.json["status_url"], headers=AUTH).json["status"] == "published"
 
 
+def test_publisher_names_photo_entry_from_capture_date(publisher, service):
+    run, remote, client = publisher
+    exif = Image.Exif()
+    exif[36867] = "2025:04:18 14:23:11"
+    response = upload(client, [photo(exif=exif)])
+
+    result = run()
+
+    assert result.returncode == 0, result.stderr
+    paths = git(remote, "ls-tree", "-r", "--name-only", "master")
+    assert f'_art/2025-04-18-photo-{response.json["id"]}-00.md' in paths
+
+
+def test_publisher_replaces_an_undated_version_of_the_same_photos(publisher, service):
+    run, remote, client = publisher
+    raw = photo()
+    original = upload(client, [raw], "Same caption", highlight="Trip").json
+    assert run().returncode == 0
+    replacement = client.post("/api/photos", headers=AUTH, data={
+        "photos": [(io.BytesIO(raw), "photo.jpg")],
+        "caption": "Same caption",
+        "highlight": "Trip",
+        "photo_dates": "2025-04-18T14:23:11+02:00",
+    }).json
+
+    result = run()
+
+    assert result.returncode == 0, result.stderr
+    paths = git(remote, "ls-tree", "-r", "--name-only", "master")
+    assert f"photo-{original['id']}" not in paths
+    assert f"img/arts/uploads/{original['id']}" not in paths
+    assert f'_art/2025-04-18-photo-{replacement["id"]}-00.md' in paths
+
+
 def test_rejected_push_can_retry(publisher):
     run, remote, client = publisher
     head = git(remote, "rev-parse", "master")