fix: close library upload flow and harden image storage

tests/contract.json

@@ -1994,5 +1994,53 @@
       "502",
       "503"
     ]
+  },
+  "/api/v1/client-scopes": {
+    "get": [
+      "200",
+      "400",
+      "401",
+      "403",
+      "404",
+      "409",
+      "413",
+      "422",
+      "429",
+      "500",
+      "502",
+      "503"
+    ]
+  },
+  "/api/v1/source-libraries/{source_id}/upload": {
+    "post": [
+      "200",
+      "400",
+      "401",
+      "403",
+      "404",
+      "409",
+      "413",
+      "422",
+      "429",
+      "500",
+      "502",
+      "503"
+    ]
+  },
+  "/api/v1/source-libraries/{source_id}/upload-url": {
+    "post": [
+      "200",
+      "400",
+      "401",
+      "403",
+      "404",
+      "409",
+      "413",
+      "422",
+      "429",
+      "500",
+      "502",
+      "503"
+    ]
   }
 }

tests/schema-contract.json

@@ -951,7 +951,8 @@
           "webdav",
           "google-drive",
           "alipan",
-          "baidu"
+          "baidu",
+          "uploads"
         ],
         "title": "Kind"
       },
@@ -1131,7 +1132,8 @@
           "webdav",
           "google-drive",
           "alipan",
-          "baidu"
+          "baidu",
+          "uploads"
         ],
         "title": "Kind"
       },

tests/test_accounts.py

@@ -242,3 +242,58 @@ def test_only_one_browser_account(app):
             "INSERT INTO users VALUES(?,?,1)",
             ("other", password_hash("another-password")),
         )
+
+
+def test_password_hash_does_not_hold_database_lock(client, app, monkeypatch):
+    import threading
+    from concurrent.futures import ThreadPoolExecutor
+
+    from pigway_frame_flow.core import accounts
+
+    account(app)
+    started, release = threading.Event(), threading.Event()
+    verify = accounts.verify_password
+
+    def delayed(*args):
+        started.set()
+        assert release.wait(5)
+        return verify(*args)
+
+    monkeypatch.setattr(accounts, "verify_password", delayed)
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        request = executor.submit(
+            client.post,
+            "/api/v1/auth/login",
+            json={"username": "test-user", "password": "fixture-password"},
+        )
+        try:
+            assert started.wait(5)
+            query = executor.submit(app.state.service.db.one, "SELECT 1 AS ok")
+            assert query.result(timeout=2)["ok"] == 1
+        finally:
+            release.set()
+        assert request.result(timeout=5).status_code == 200
+
+
+def test_password_changed_during_login_does_not_create_session(
+    client, app, monkeypatch
+):
+    from pigway_frame_flow.core import accounts
+
+    account(app)
+    verify = accounts.verify_password
+
+    def changed(*args):
+        result = verify(*args)
+        app.state.service.db.execute(
+            "UPDATE users SET password_hash=?", (password_hash("replacement-password"),)
+        )
+        return result
+
+    monkeypatch.setattr(accounts, "verify_password", changed)
+    response = client.post(
+        "/api/v1/auth/login",
+        json={"username": "test-user", "password": "fixture-password"},
+    )
+    assert response.status_code == 401
+    assert app.state.service.db.all("SELECT * FROM sessions") == []

tests/test_audit_fixes.py

@@ -0,0 +1,129 @@
+import shutil
+
+import pytest
+
+
+def test_image_write_permission_can_be_granted_without_admin(client):
+    from io import BytesIO
+
+    from PIL import Image
+
+    data = BytesIO()
+    Image.new("RGB", (30, 20), "green").save(data, "PNG")
+    picture = client.post(
+        "/api/v1/source-libraries/manual-uploads/upload",
+        files={"file": ("fixture.png", data.getvalue())},
+    ).json()
+    scopes = client.get("/api/v1/client-scopes").json()
+    assert "images:write" in scopes
+    response = client.post(
+        "/api/v1/clients",
+        json={
+            "client_id": "image-writer",
+            "name": "Writer",
+            "permissions": ["images:write"],
+        },
+    )
+    assert response.status_code == 201
+    client.headers["Authorization"] = "Bearer " + response.json()["token"]
+    # Authorization succeeds and resource lookup fails; administrative routes stay forbidden.
+    assert (
+        client.put("/api/v1/assets/missing/star", json={"starred": True}).status_code
+        == 404
+    )
+    assert (
+        client.put(
+            "/api/v1/source-images/" + picture["id"] + "/star", json={"starred": True}
+        ).status_code
+        == 200
+    )
+    assert (
+        client.put(
+            "/api/v1/source-images/" + picture["id"] + "/exclusion",
+            json={"excluded": True},
+        ).status_code
+        == 409
+    )
+    assert client.get("/api/v1/clients").status_code == 403
+
+
+def test_nested_transaction_rollback_includes_execute(service):
+    with pytest.raises(RuntimeError):
+        with service.db.transaction():
+            service.db.execute("INSERT INTO app_config VALUES('outer','{}')")
+            with service.db.transaction():
+                service.db.execute("INSERT INTO app_config VALUES('inner','{}')")
+            raise RuntimeError("injected failure")
+    assert (
+        service.db.one("SELECT key FROM app_config WHERE key IN ('outer','inner')")
+        is None
+    )
+
+
+def test_low_disk_space_is_reported_without_enabling_cleanup(client, monkeypatch):
+    monkeypatch.setattr(
+        shutil,
+        "disk_usage",
+        lambda _path: shutil._ntuple_diskusage(100 * 1024**3, 99 * 1024**3, 1024**3),
+    )
+    data = client.get("/api/v1/storage").json()
+    assert data["disk"]["low_space"] is True
+    assert data["disk"]["free_bytes"] == 1024**3
+    assert data["policy"]["enabled"] is False
+
+
+def test_unsupported_image_plugin_is_never_invoked(monkeypatch):
+    from PIL import Image
+
+    from pigway_frame_flow.errors import FlowError
+    from pigway_frame_flow.renderers.images import convert
+
+    Image.init()
+    invoked = []
+
+    def vulnerable_plugin(*_args):
+        invoked.append(True)
+        raise RuntimeError("Unsupported plugin must not parse input")
+
+    monkeypatch.setitem(Image.OPEN, "XBM", (vulnerable_plugin, lambda _prefix: True))
+    with pytest.raises(FlowError):
+        convert(b"#define image_width 1\n#define image_height 1\n")
+    assert not invoked
+
+
+def test_promotion_sql_failure_keeps_original_cache(service, monkeypatch):
+    asset = service.assets.put(b"fixture", "cached.jpg", "cache")
+    original = service.db.execute
+
+    def fail(query, args=()):
+        if query.startswith("UPDATE assets SET kind"):
+            raise RuntimeError("injected promotion SQL failure")
+        return original(query, args)
+
+    monkeypatch.setattr(service.db, "execute", fail)
+    with pytest.raises(RuntimeError):
+        service.assets.promote(asset["asset_id"])
+    assert service.assets.path(asset["asset_id"]).parent.name == "cache"
+    assert service.assets.path(asset["asset_id"]).read_bytes() == b"fixture"
+    assert not list((service.settings.data_dir / "images").iterdir())
+    assert not service.db.all("SELECT * FROM asset_file_moves")
+
+
+def test_promotion_restart_finishes_committed_move(service):
+    from pigway_frame_flow.renderers.images import Assets
+
+    asset = service.assets.put(b"fixture", "cached.jpg", "cache")
+    source = service.assets.path(asset["asset_id"])
+    target = service.settings.data_dir / "images" / source.name
+    target.write_bytes(source.read_bytes())
+    service.db.execute(
+        "INSERT INTO asset_file_moves VALUES(?,?,?)",
+        (asset["asset_id"], "cache", "images"),
+    )
+    service.db.execute(
+        "UPDATE assets SET kind='images' WHERE id=?", (asset["asset_id"],)
+    )
+    Assets(service.db, service.settings)
+    assert not source.exists()
+    assert service.assets.path(asset["asset_id"]).read_bytes() == b"fixture"
+    assert not service.db.all("SELECT * FROM asset_file_moves")

tests/test_source_browser.py

@@ -174,3 +174,16 @@ def test_session_changes_invalidate_preview(client, browser, monkeypatch):
     ]
     monkeypatch.setattr(photos.synology, "sid", lambda: "new-session")
     assert client.get(item["preview_url"]).status_code == 404
+
+
+def test_sparse_search_finds_match_after_an_empty_batch(client, browser):
+    _, rows, _ = browser
+    rows[45]["filename"] = "later-match.jpg"
+    first = client.get(
+        "/api/v1/source-libraries/family/browse?filename=later-match&limit=10"
+    ).json()
+    assert first["items"] == [] and first["next_offset"] == 40
+    second = client.get(
+        "/api/v1/source-libraries/family/browse?filename=later-match&limit=10&offset=40"
+    ).json()
+    assert [item["name"] for item in second["items"]] == ["later-match.jpg"]

tests/test_source_libraries.py

@@ -781,3 +781,175 @@ def test_source_deletion_removes_unreferenced_archived_plans(
         service.db.one("SELECT id FROM library_plans WHERE source_id='family'") is None
     )
     assert client.get("/api/v1/generation-plans?source_id=family").status_code == 404
+
+
+def test_manual_upload_survives_reload_and_generates_bound_versions(client, service):
+    data = BytesIO()
+    Image.new("RGB", (700, 500), "blue").save(data, "PNG")
+    path = "/api/v1/source-libraries/manual-uploads/upload"
+    result = client.post(
+        path, files={"file": ("original.png", data.getvalue(), "image/png")}
+    )
+    assert result.status_code == 200, result.text
+    picture = result.json()
+    with Image.open(service.assets.path(picture["id"])) as image:
+        assert image.size == (700, 500)
+    assert (
+        client.post(path, files={"file": ("same.png", data.getvalue())}).json()["id"]
+        == picture["id"]
+    )
+    listed = client.get("/api/v1/source-libraries/manual-uploads/images").json()
+    assert listed["total"] == 1 and listed["items"][0]["id"] == picture["id"]
+    plan = create_plan(client, "manual-uploads")
+    bind(client, "watch", "manual-uploads", plan)
+    generated = client.post(
+        "/api/v1/source-images/" + picture["id"] + "/generate",
+        json={"plan_id": plan["id"]},
+    )
+    assert generated.status_code == 200, generated.text
+    assert generated.json()[0]["plan"]["width"] == 410
+    run = client.post(
+        "/api/v1/source-libraries/manual-uploads/run", json={"deliver": False}
+    )
+    assert run.status_code == 200, run.text
+    assert (
+        client.get("/api/v1/source-libraries/manual-uploads/images").json()["total"]
+        == 1
+    )
+    assert client.get("/api/v1/library-terminals/watch/images").json()["total"] == 1
+
+
+def test_invalid_upload_does_not_create_source(client, service):
+    response = client.post(
+        "/api/v1/source-libraries/manual-uploads/upload",
+        files={"file": ("bad.jpg", b"not an image")},
+    )
+    assert response.status_code == 400
+    assert service.photo_sources.list() == []
+
+
+def test_ingest_sql_failure_rolls_back_asset_and_file(
+    client, service, library_source, monkeypatch
+):
+    original = service.db.execute
+
+    def fail(query, args=()):
+        if "INSERT INTO source_images" in query:
+            raise RuntimeError("injected SQL failure")
+        return original(query, args)
+
+    monkeypatch.setattr(service.db, "execute", fail)
+    with pytest.raises(RuntimeError):
+        service.libraries.import_picture("family", "photo.jpg")
+    assert service.db.all("SELECT * FROM assets") == []
+    assert service.db.all("SELECT * FROM source_images") == []
+    assert list((service.settings.data_dir / "source-images").iterdir()) == []
+    assert service.db.all("SELECT * FROM asset_file_journal") == []
+
+
+def test_generation_sql_failure_keeps_source_without_orphan(
+    client, service, library_source, monkeypatch
+):
+    picture = import_picture(client)
+    plan = create_plan(client)
+    original = service.db.execute
+
+    def fail(query, args=()):
+        if "INSERT INTO library_versions" in query:
+            raise RuntimeError("injected SQL failure")
+        return original(query, args)
+
+    monkeypatch.setattr(service.db, "execute", fail)
+    with pytest.raises(RuntimeError):
+        service.libraries.generate(picture["id"], plan["id"])
+    assert len(service.db.all("SELECT * FROM assets")) == 1
+    assert list((service.settings.data_dir / "renditions").iterdir()) == []
+    assert service.assets.path(picture["id"]).exists()
+
+
+def test_delete_sql_failure_preserves_files_and_relations(
+    client, service, library_source, monkeypatch
+):
+    picture = import_picture(client)
+    plan = create_plan(client)
+    version = service.libraries.generate(picture["id"], plan["id"])
+    original = service.db.execute
+
+    def fail(query, args=()):
+        if query == "DELETE FROM source_images WHERE id=?":
+            raise RuntimeError("injected SQL failure")
+        return original(query, args)
+
+    monkeypatch.setattr(service.db, "execute", fail)
+    with pytest.raises(RuntimeError):
+        service.libraries.delete_picture(picture["id"])
+    assert service.assets.path(version["asset_id"]).exists()
+    assert (
+        service.libraries.picture(picture["id"])["versions"][0]["asset_id"]
+        == version["asset_id"]
+    )
+    assert service.db.all("SELECT * FROM asset_file_journal") == []
+
+
+def test_delete_file_failure_retries_on_restart(
+    client, service, library_source, monkeypatch
+):
+    from pathlib import Path
+
+    from pigway_frame_flow.renderers.images import Assets
+
+    picture = import_picture(client)
+    path = service.assets.path(picture["id"])
+    unlink = Path.unlink
+
+    def fail(file, *args, **kwargs):
+        if file == path:
+            raise OSError("injected unlink failure")
+        return unlink(file, *args, **kwargs)
+
+    with monkeypatch.context() as patch:
+        patch.setattr(Path, "unlink", fail)
+        service.libraries.delete_picture(picture["id"])
+        assert service.db.all("SELECT * FROM source_images") == []
+        assert (
+            service.db.one("SELECT id FROM asset_file_journal")["id"] == picture["id"]
+        )
+    Assets(service.db, service.settings)
+    assert not path.exists()
+    assert service.db.all("SELECT * FROM asset_file_journal") == []
+
+
+def test_asset_disk_write_failure_leaves_no_partial_file(service, monkeypatch):
+    import os
+
+    fsync = os.fsync
+    calls = 0
+
+    def fail(fd):
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise OSError("injected disk write failure")
+        return fsync(fd)
+
+    monkeypatch.setattr(os, "fsync", fail)
+    with pytest.raises(OSError):
+        service.assets.put(b"data", "failed.jpg")
+    assert service.db.all("SELECT * FROM assets") == []
+    assert service.db.all("SELECT * FROM asset_file_journal") == []
+    assert list((service.settings.data_dir / "images").iterdir()) == []
+
+
+def test_asset_startup_removes_interrupted_creation(service):
+    from pigway_frame_flow.renderers.images import Assets
+
+    asset_id = "c" * 32
+    root = service.settings.data_dir / "images"
+    (root / (asset_id + ".jpg")).write_bytes(b"incomplete")
+    (root / (asset_id + ".part")).write_bytes(b"partial")
+    service.db.execute(
+        "INSERT INTO asset_file_journal VALUES(?,?)", (asset_id, "images")
+    )
+    Assets(service.db, service.settings)
+    assert not list(root.iterdir())
+    assert service.db.all("SELECT * FROM asset_file_journal") == []