Publish snippets and Markdown posts through authenticated uploads

photo-upload/tests/test_posts.py

@@ -0,0 +1,81 @@
+import io
+import json
+
+from test_upload import AUTH, git, publisher, service
+from posts import content_similarity
+
+
+BODY = "\n\n".join(f"Paragraph {i}: I walked beside the river and considered how memory changes our understanding of a place. The details matter more than the names we give them." for i in range(12))
+
+
+def send(client, text, filename="essay.md"):
+    return client.post("/api/posts", headers=AUTH, data={"file": (io.BytesIO(text.encode()), filename)})
+
+
+def test_post_validation(service):
+    _, client, _ = service
+    assert client.post("/api/posts").status_code == 401
+    assert client.post("/api/posts", headers=AUTH).status_code == 400
+    assert send(client, BODY, "../escape.md").status_code == 400
+    assert send(client, BODY, "essay.pdf").status_code == 400
+    assert send(client, "---\ntitle: [bad\n---\nBody").status_code == 400
+    assert send(client, "---\npost_id: ../../escape\n---\nBody").status_code == 400
+    assert send(client, " ").status_code == 400
+    assert send(client, "x" * (1024 * 1024 + 1)).status_code == 400
+
+
+def test_content_update_preserves_url_date_and_no_duplicate(publisher, service, monkeypatch, tmp_path):
+    run, remote, client = publisher
+    first_text = "---\ntitle: My walk\ndate: 2020-01-02\npermalink: /my-walk/\n---\n\n" + BODY
+    first = send(client, first_text)
+    assert first.status_code == 202
+    assert run().returncode == 0
+    status = client.get(first.json["status_url"], headers=AUTH).json
+    assert status["action"] == "created" and status["url"] == "/my-walk/"
+    second = send(client, "# My revised walk\n\n" + BODY.replace("beside the river", "along the river", 1), "renamed.md")
+    result = run()
+    assert result.returncode == 0, result.stderr
+    status2 = client.get(second.json["status_url"], headers=AUTH).json
+    assert status2["action"] == "updated"
+    assert status2["url"] == status["url"]
+    assert status2["post_id"] == status["post_id"]
+    paths = [p for p in git(remote, "ls-tree", "-r", "--name-only", "master").splitlines() if p.startswith("_posts/")]
+    assert len(paths) == 1
+    saved = git(remote, "show", "master:" + paths[0])
+    assert "2020-01-02" in saved and "along the river" in saved
+    assert "render_with_liquid: false" in saved
+    assert client.get(first.json["status_url"], headers=AUTH).json["status"] == "superseded"
+    receipt = tmp_path / "deployed.json"
+    monkeypatch.setenv("PHOTO_DEPLOYMENT_FILE", str(receipt))
+    receipt.write_text(json.dumps({"posts": [second.json["id"]]}))
+    assert client.get(second.json["status_url"], headers=AUTH).json["status"] == "published"
+    assert send(client, first_text).status_code == 200
+
+
+def test_unrelated_same_filename_creates_new_post(publisher):
+    run, remote, client = publisher
+    send(client, BODY)
+    assert run().returncode == 0
+    response = send(client, "# Another subject\n\n" + "Astronomy explores stars, galaxies and planetary orbits. " * 20)
+    assert run().returncode == 0
+    assert client.get(response.json["status_url"], headers=AUTH).json["action"] == "created"
+    assert len([p for p in git(remote, "ls-tree", "-r", "--name-only", "master").splitlines() if p.startswith("_posts/")]) == 2
+
+
+def test_ambiguous_matches_fail_without_pushing(publisher):
+    run, remote, client = publisher
+    for identity in ("one", "two"):
+        send(client, f"---\npost_id: {identity}\ntitle: {identity}\n---\n" + BODY, identity + ".md")
+    assert run().returncode == 0
+    head = git(remote, "rev-parse", "master")
+    response = send(client, BODY.replace("details", "small details", 1))
+    assert run().returncode != 0
+    state = client.get(response.json["status_url"], headers=AUTH).json
+    assert state["status"] == "failed" and "Multiple posts" in state["error"]
+    assert git(remote, "rev-parse", "master") == head
+
+
+def test_similarity_requires_substantial_unambiguous_content():
+    assert content_similarity("short shared title", "short shared title") == 0
+    assert content_similarity(BODY, BODY.replace("river", "lake", 1)) >= 0.92
+    assert content_similarity(BODY, "Completely unrelated writing.") == 0

photo-upload/tests/test_upload.py

@@ -5,6 +5,7 @@ from pathlib import Path
 import subprocess
 import sys
 import os
+import random
 
 import pytest
 from PIL import Image
@@ -59,6 +60,16 @@ def test_validation_is_all_or_nothing(service):
     assert not list((data / "temporary").iterdir())
 
 
+def test_multi_megabyte_multipart_file(service):
+    _, client, _ = service
+    output = io.BytesIO()
+    Image.frombytes("RGB", (1800, 1800), random.Random(2).randbytes(1800 * 1800 * 3)).save(
+        output, "JPEG", quality=95)
+    assert len(output.getvalue()) > 3 * 1024 * 1024
+    response = upload(client, [output.getvalue()])
+    assert response.status_code == 202, response.json
+
+
 def test_album_caption_order_and_retry_deduplication(service):
     _, client, data = service
     caption = '<script>alert(1)</script> {% include secret %} {{ site.email }}\nsecond line'
@@ -149,6 +160,7 @@ def publisher(tmp_path, service):
     git(source, "config", "user.name", "Test")
     git(source, "config", "user.email", "test@example.com")
     (source / "art.html").write_text("Initial site")
+    (source / "snippets.md").write_text("---\nlayout: post\n---\nIntroduction\n\n<!-- uploaded-snippets -->\n\nOld snippet\n")
     git(source, "add", ".")
     git(source, "commit", "-m", "Initial")
     git(source, "remote", "add", "origin", str(remote))
@@ -200,3 +212,47 @@ def test_rejected_push_can_retry(publisher):
     result = run()
     assert result.returncode == 0, result.stderr
     assert client.get(response.json["status_url"], headers=AUTH).json["status"] == "pushed"
+
+
+def test_snippet_intake_validation_and_literal_text(service):
+    _, client, data = service
+    assert client.post("/api/snippets", data={"text": "hello"}).status_code == 401
+    assert client.post("/api/snippets", headers=AUTH).status_code == 400
+    assert client.post("/api/snippets", headers=AUTH, data={"text": "x" * 20001}).status_code == 400
+    text = 'Book quote\nفارسی <script>bad</script> {% include secret %}\nhttps://example.com/?a=1&b=2'
+    response = client.post("/api/snippets", headers=AUTH, data={"text": text})
+    assert response.status_code == 202
+    job = response.json["id"]
+    assert response.json["url"] == f"/snippets/#snippet-{job}"
+    assert client.get(f"/api/photos/{job}", headers=AUTH).status_code == 404
+    entry = (data / "submissions" / job / "entry.md").read_text()
+    assert '<script>' not in entry and '{%' not in entry
+    assert '<p dir="auto">Book quote<br>\nفارسی' in entry
+    assert '<a href="https://example.com/?a=1&amp;b=2">' in entry
+    assert client.post("/api/snippets", headers=AUTH, data={"text": text}).status_code == 200
+    assert client.post("/api/snippets", headers=AUTH, data={"photos": (io.BytesIO(b"bad"), "bad.jpg")}).status_code == 400
+
+
+def test_snippet_prepend_images_recovery_and_receipt(publisher, service, monkeypatch, tmp_path):
+    run, remote, client = publisher
+    _, _, data = service
+    first = client.post("/api/snippets", headers=AUTH, data={"text": "First snippet"}).json
+    second = client.post("/api/snippets", headers=AUTH, data={
+        "caption": "Image note", "photos": (io.BytesIO(photo()), "photo.jpg"),
+    }).json
+    result = run()
+    assert result.returncode == 0, result.stderr
+    content = git(remote, "show", "master:snippets.md")
+    assert content.startswith("---\nlayout: post\n---\nIntroduction")
+    assert content.index(second["id"]) < content.index(first["id"]) < content.index("Old snippet")
+    assert f'img/snippets/uploads/{second["id"]}/00.jpg' in git(remote, "ls-tree", "-r", "--name-only", "master")
+    head = git(remote, "rev-parse", "master")
+    for job in (first["id"], second["id"]):
+        (data / "status" / f"{job}.json").write_text('{"status":"failed"}')
+        assert client.post(f"/api/snippets/{job}/retry", headers=AUTH).json["status"] == "queued"
+    assert run().returncode == 0
+    assert git(remote, "rev-parse", "master") == head
+    receipt = tmp_path / "deployed.json"
+    monkeypatch.setenv("PHOTO_DEPLOYMENT_FILE", str(receipt))
+    receipt.write_text(json.dumps({"snippets": [first["id"], second["id"]]}))
+    assert client.get(first["status_url"], headers=AUTH).json["status"] == "published"