pigway-frame-flow/pigway_frame_flow/providers/photo_sources.py

"""Read-only image catalogs; provider credentials never enter SQLite."""

import base64
import json
import logging
import os
import posixpath
import random
import stat
import time
from pathlib import Path
from typing import Literal
from urllib.parse import unquote, urljoin, urlsplit
from xml.etree import ElementTree

from pydantic import Field

from pigway_frame_flow.core.photo_preferences import eligible
from pigway_frame_flow.errors import ErrorCode as E
from pigway_frame_flow.errors import FlowError
from pigway_frame_flow.models import StrictModel
from pigway_frame_flow.providers.photo_names import path_caption
from pigway_frame_flow.providers.photo_orientation import dimensions, orientation
from pigway_frame_flow.providers.synology_photos import SynologyPhotos
from pigway_frame_flow.renderers.images import MAX_IMAGE_PIXELS
from pigway_frame_flow.settings import secret
from pigway_frame_flow.storage.database import encode


class PhotoSourceInput(StrictModel):
    name: str = Field(min_length=1, max_length=120)
    kind: Literal[
        "local",
        "synology-photos",
        "webdav",
        "google-drive",
        "alipan",
        "baidu",
        "uploads",
    ]
    location: str = Field(min_length=1, max_length=2048)
    nas_root_path: str | None = Field(default=None, max_length=2048, pattern=r"^/")
    recursive: bool = True
    enabled: bool = True
    catalog_limit: int = Field(default=10000, ge=1, le=100000)
    synology_previews: bool = False
    organized_only: bool = False
    folder_caption: bool = False
    orientation: Literal["auto", "landscape", "portrait"] | None = None
    minimum_short_edge: int = Field(default=0, ge=0, le=50000)
    minimum_long_edge: int = Field(default=0, ge=0, le=50000)
    minimum_sharpness: float = Field(default=0, ge=0, le=10000)
    minimum_brightness: float = Field(default=0, ge=0, le=255)
    maximum_brightness: float = Field(default=255, ge=0, le=255)
    avoid_duplicates: bool = False
    people_space: Literal["shared", "personal"] = "shared"
    people_filter: bool = False
    person_ids: list[int] = Field(default_factory=list, max_length=50)
    person_match: Literal["any", "all"] = "any"
    credential_env: str | None = Field(
        default=None, pattern=r"^FRAME_FLOW_SOURCE_[A-Z0-9_]+$"
    )


class PhotoSourceView(PhotoSourceInput):
    id: str


class PhotoItem(StrictModel):
    starred: bool = False
    excluded: bool = False
    id: str
    name: str
    caption: dict[str, str] | None = None
    source_path: str | None = None
    provider_preview: bool = False
    orientation: Literal["landscape", "portrait", "square"] | None = None
    width: int | None = None
    height: int | None = None


class PhotoCatalog(StrictModel):
    items: list[PhotoItem]
    total: int
    offset: int
    limit: int
    catalog_limit: int


class PhotoSources:
    extensions = {".jpg", ".jpeg", ".png", ".webp"}

    def __init__(self, db, settings, assets):
        self.db, self.settings, self.assets = db, settings, assets
        self.cache = {}
        self.api_items = {}
        from pigway_frame_flow.providers.source_browser import SourceBrowser

        self.browser = SourceBrowser(self)
        self.synology = SynologyPhotos(settings)
        # Register provider secrets before request IDs or errors can be reflected.
        for row in self.list():
            if row["kind"] == "webdav" and row.get("credential_env"):
                try:
                    self.headers(
                        PhotoSourceInput.model_validate(
                            {k: v for k, v in row.items() if k != "id"}
                        )
                    )
                except FlowError:
                    pass

    def list(self):
        return [
            dict(json.loads(row["payload"]), id=row["key"][13:])
            for row in self.db.all(
                "SELECT key,payload FROM app_config WHERE key LIKE 'photo_source:%' ORDER BY key"
            )
        ]

    def get(self, source_id):
        row = self.db.one(
            "SELECT payload FROM app_config WHERE key=?", ("photo_source:" + source_id,)
        )
        if not row:
            raise FlowError(E.NOT_FOUND, "Photo source not found", 404)
        return PhotoSourceInput.model_validate_json(row["payload"])

    @staticmethod
    def require_api_source(source):
        # Block persisted legacy sources as well as new requests, before any disk read.
        if source.kind == "local":
            raise FlowError(
                E.FORBIDDEN,
                "Filesystem photo sources are disabled; use the Synology Photos API",
                403,
            )

    def save(self, source_id, source):
        self.require_api_source(source)
        if source.kind == "uploads" and (
            source_id != "manual-uploads" or source.location != "."
        ):
            raise FlowError(E.INVALID_REQUEST, "Invalid manual upload source")
        if source_id == "manual-uploads" and source.kind != "uploads":
            raise FlowError(E.CONFLICT, "Reserved manual upload source")
        if source.minimum_brightness > source.maximum_brightness:
            raise FlowError(E.INVALID_REQUEST, "Brightness range is reversed")
        if source.kind not in ("local", "synology-photos") and (
            source.organized_only or source.folder_caption or source.synology_previews
        ):
            raise FlowError(
                E.UNSUPPORTED_CAPABILITY,
                "Folder captions and Synology previews require a local photo source",
            )
        if source.orientation and source.kind not in ("local", "synology-photos"):
            raise FlowError(
                E.UNSUPPORTED_CAPABILITY,
                "Orientation filtering requires a local photo source",
            )
        if source.people_filter and source.kind not in ("local", "synology-photos"):
            raise FlowError(
                E.INVALID_REQUEST,
                "Person filtering requires a local Synology photo source",
            )
        if any(i <= 0 for i in source.person_ids) or len(set(source.person_ids)) != len(
            source.person_ids
        ):
            raise FlowError(
                E.INVALID_REQUEST, "Person IDs must be unique positive integers"
            )
        if source.people_filter and not source.person_ids:
            raise FlowError(E.INVALID_REQUEST, "Select at least one person")
        if source.kind == "local":
            self.local_directory(source.location)
            if source.credential_env:
                raise FlowError(
                    E.INVALID_REQUEST, "Local sources do not use credentials"
                )
        elif source.kind == "synology-photos":
            location = source.location.strip("/")
            if location == ".":
                location = ""
            if "\\" in location or any(p in (".", "..") for p in location.split("/")):
                raise FlowError(
                    E.INVALID_REQUEST,
                    "Photos API folder must be a relative folder path or root",
                )
            if source.credential_env or source.synology_previews:
                raise FlowError(
                    E.INVALID_REQUEST,
                    "Photos API sources use the shared Photos session and XL thumbnail download",
                )
            source = source.model_copy(update={"location": location or "."})
        elif source.kind == "webdav":
            source = source.model_copy(
                update={
                    "recursive": False,
                    "location": source.location.rstrip("/") + "/",
                }
            )
            parsed = urlsplit(source.location)
            if (
                parsed.scheme != "https"
                or not parsed.hostname
                or parsed.username
                or parsed.password
                or parsed.query
                or parsed.fragment
                or parsed.port not in (None, 443)
            ):
                raise FlowError(
                    E.INVALID_REQUEST, "WebDAV requires a plain HTTPS directory URL"
                )
            if parsed.hostname.lower() not in self.settings.image_url_hosts:
                raise FlowError(
                    E.FORBIDDEN, "Configure the allowed image host first", 403
                )
        self.db.execute(
            "INSERT OR REPLACE INTO app_config(key,payload) VALUES(?,?)",
            ("photo_source:" + source_id, encode(source.model_dump())),
        )
        self.cache.pop(source_id, None)
        self.api_items.pop(source_id, None)
        return dict(source.model_dump(), id=source_id)

    def delete(self, source_id):
        self.get(source_id)
        for row in self.db.all("SELECT payload FROM jobs"):
            if json.loads(row["payload"])["source"].get("source_id") == source_id:
                raise FlowError(E.CONFLICT, "Remove jobs using this source first", 409)
        for row in self.db.all(
            "SELECT payload FROM operations WHERE status IN ('QUEUED','RENDERING','UPLOADING','SYNCING','UNKNOWN')"
        ):
            payload = json.loads(row["payload"])
            if payload.get("content", payload).get("source_id") == source_id:
                raise FlowError(E.CONFLICT, "This source has pending operations", 409)
        self.db.execute(
            "DELETE FROM app_config WHERE key=?", ("photo_source:" + source_id,)
        )
        self.db.execute("DELETE FROM photo_preferences WHERE source_id=?", (source_id,))
        self.cache.pop(source_id, None)
        self.api_items.pop(source_id, None)

    def local_directory(self, location):
        root = self.settings.photo_root.resolve()
        directory = (root / location).resolve()
        if Path(location).is_absolute() or not directory.is_relative_to(root):
            raise FlowError(
                E.FORBIDDEN,
                "Photo directory must be inside the mounted photo root",
                403,
            )
        if not directory.is_dir():
            raise FlowError(
                E.NOT_FOUND,
                "Photo directory is unavailable; check its read-only mount",
                404,
            )
        try:
            with os.scandir(directory):
                pass
        except PermissionError:
            raise FlowError(
                E.FORBIDDEN,
                "Photo directory is not readable; configure the container's existing NAS read group",
                403,
            ) from None
        return directory

    def headers(self, source):
        if not source.credential_env:
            return {}
        try:
            value = secret(source.credential_env)
        except OSError:
            raise FlowError(
                E.CONFIG_REQUIRED, "WebDAV credential secret is unavailable"
            ) from None
        if value and value not in self.settings.additional_secrets:
            self.settings.additional_secrets.append(value)
        try:
            credentials = json.loads(value)
            username, password = credentials["username"], credentials["password"]
            if (
                not isinstance(username, str)
                or not isinstance(password, str)
                or ":" in username
            ):
                raise ValueError()
        except (ValueError, KeyError, TypeError):
            raise FlowError(
                E.CONFIG_REQUIRED,
                "WebDAV credential secret must contain username and password",
            ) from None
        for sensitive in (value, username, password):
            if sensitive and sensitive not in self.settings.additional_secrets:
                self.settings.additional_secrets.append(sensitive)
        encoded = base64.b64encode((username + ":" + password).encode()).decode()
        if encoded not in self.settings.additional_secrets:
            self.settings.additional_secrets.append(encoded)
        return {"Authorization": "Basic " + encoded}

    def thumbnail(self, path):
        candidate = path.parent / "@eaDir" / path.name / "SYNOPHOTO_THUMB_XL.jpg"
        try:
            if candidate.is_symlink() or not candidate.resolve().is_relative_to(
                path.parent.resolve()
            ):
                return None
            if (
                candidate.is_file()
                and candidate.stat().st_size <= self.settings.max_bytes
            ):
                return candidate
        except OSError:
            pass
        return None

    def catalog(self, source_id, refresh=False, *, target_size=(1200, 1600)):
        source = self.get(source_id)
        self.require_api_source(source)
        if not source.enabled:
            raise FlowError(E.CONFLICT, "Photo source is disabled", 409)
        if source.kind == "uploads":
            return [
                {
                    "id": row["item_id"],
                    "name": row["name"],
                    "caption": json.loads(row["caption"]),
                    "starred": bool(row["starred"]),
                    "excluded": bool(row["excluded"]),
                }
                for row in self.db.all(
                    "SELECT s.item_id,s.caption,a.name,COALESCE(p.starred,0) AS starred,COALESCE(p.excluded,0) AS excluded FROM source_images s JOIN assets a ON a.id=s.id LEFT JOIN photo_preferences p ON p.source_id=s.source_id AND p.item_id=s.item_id WHERE s.source_id=? ORDER BY s.created_at DESC,s.id DESC",
                    (source_id,),
                )
            ]
        session_identity = (
            self.synology.sid() if source.kind == "synology-photos" else None
        )
        wanted = (
            orientation(*target_size)
            if source.orientation == "auto"
            else source.orientation
        )
        cached = self.cache.get(source_id)
        if (
            cached
            and not refresh
            and time.monotonic() - cached[0] < 60
            and cached[2] == wanted
            and cached[3] == session_identity
        ):
            return cached[1]
        allowed = (
            self.synology.paths(
                source.person_ids,
                source.person_match,
                refresh=refresh,
                space=source.people_space,
            )
            if source.people_filter and source.kind == "local"
            else None
        )
        items = []
        sizes = (
            {r["path"]: r for r in self.db.all("SELECT * FROM photo_dimensions")}
            if source.orientation
            else {}
        )
        size_updates = []
        if source.kind == "local":
            directory = self.local_directory(source.location)
            root = self.settings.photo_root.resolve()
            # Do not follow symlink directories or import Synology's thumbnail trees.
            for folder, dirs, files in os.walk(directory, followlinks=False):
                dirs[:] = (
                    sorted(
                        d
                        for d in dirs
                        if d != "#recycle"
                        and not d.startswith(".")
                        and not d.startswith("@")
                        and not Path(folder, d).is_symlink()
                    )
                    if source.recursive
                    else []
                )
                caption = path_caption(
                    Path(folder).relative_to(root).as_posix() + "/__photo__.jpg"
                )
                if source.organized_only and caption is None:
                    continue
                for name in sorted(files):
                    path = Path(folder, name)
                    if (
                        name.startswith(".")
                        or path.suffix.lower() not in self.extensions
                        or path.is_symlink()
                    ):
                        continue
                    try:
                        large = path.stat().st_size > self.settings.max_bytes
                        preview = (
                            large
                            and source.synology_previews
                            and self.thumbnail(path) is not None
                        )
                        if large and not preview:
                            continue
                        relative = path.relative_to(root).as_posix()
                        if allowed is not None and relative not in allowed:
                            continue
                        info = {}
                        if wanted or source.kind == "local":
                            key = str(path)
                            stamp = path.stat()
                            cached_size = sizes.get(key)
                            if (
                                cached_size
                                and cached_size["mtime"] == stamp.st_mtime_ns
                                and cached_size["bytes"] == stamp.st_size
                            ):
                                width, height = (
                                    cached_size["width"],
                                    cached_size["height"],
                                )
                            else:
                                size = dimensions(root, path)
                                if size is None and preview:
                                    preview_path = self.thumbnail(path)
                                    size = (
                                        dimensions(root, preview_path)
                                        if preview_path
                                        else None
                                    )
                                if size is None:
                                    if wanted:
                                        continue
                                    size = (0, 0)
                                width, height = size
                                size_updates.append(
                                    (
                                        key,
                                        stamp.st_mtime_ns,
                                        stamp.st_size,
                                        width,
                                        height,
                                    )
                                )
                            if not self.resolution_matches(source, width, height):
                                continue
                            direction = orientation(width, height)
                            if wanted and direction != wanted:
                                continue
                            info = {
                                "width": width,
                                "height": height,
                                "orientation": direction,
                            }
                        if width * height > MAX_IMAGE_PIXELS:
                            if (
                                source.synology_previews
                                and self.thumbnail(path) is not None
                            ):
                                preview = True
                            else:
                                continue
                        items.append(
                            {
                                **info,
                                "id": relative,
                                "name": name,
                                "provider_preview": preview,
                                "caption": caption if source.folder_caption else None,
                            }
                        )
                    except OSError:
                        continue
                    if len(items) >= source.catalog_limit:
                        break
                if len(items) >= source.catalog_limit:
                    break
        elif source.kind == "synology-photos":
            self.synology.sid()
            rows = self.synology.items(
                source.person_ids if source.people_filter else [],
                source.person_match,
                space=source.people_space,
                limit=100000,
            )
            location = "" if source.location == "." else source.location.strip("/")
            self.api_items[source_id] = {}
            for row in rows:
                folder = row.get("additional", {}).get("folder")
                if isinstance(folder, dict):
                    folder = folder.get("name")
                if not isinstance(folder, str) or not isinstance(
                    row.get("filename"), str
                ):
                    raise FlowError(
                        E.CLOUD_UNAVAILABLE,
                        "Photos API response lacks folder or filename",
                        502,
                    )
                relative = folder.strip("/") + "/" + row["filename"]
                if location and not relative.startswith(location + "/"):
                    continue
                if not source.recursive and folder.strip("/") != location:
                    continue
                caption = path_caption(relative)
                if source.organized_only and caption is None:
                    continue
                size = row.get("additional", {}).get("resolution", {})
                width, height = size.get("width", 0), size.get("height", 0)
                # Photos resolution is already oriented; EXIF must not swap it twice.
                if not self.resolution_matches(source, width, height):
                    continue
                direction = orientation(width, height)
                if wanted and (not width or not height or direction != wanted):
                    continue
                items.append(
                    {
                        "id": str(row["id"]),
                        "name": row["filename"],
                        "provider_preview": True,
                        "source_path": (
                            source.nas_root_path.rstrip("/")
                            + "/"
                            + relative.lstrip("/")
                            if source.nas_root_path
                            else "/" + relative.lstrip("/")
                        ),
                        "caption": caption if source.folder_caption else None,
                        "width": width,
                        "height": height,
                        "orientation": direction,
                    }
                )
                self.api_items[source_id][str(row["id"])] = row
                if len(items) >= source.catalog_limit:
                    break
        elif source.kind in ("google-drive", "alipan", "baidu"):
            from pigway_frame_flow.providers.cloud_drives import CloudDrives

            items = CloudDrives(self).catalog(source_id, source)
        else:
            data = self.assets.fetch_url(
                source.location,
                method="PROPFIND",
                headers={**self.headers(source), "Depth": "1"},
                limit=2 * 1024 * 1024,
                accepted_status=(207,),
            )
            if (
                b"\x00" in data
                or b"<!DOCTYPE" in data.upper()
                or b"<!ENTITY" in data.upper()
            ):
                raise FlowError(E.INVALID_REQUEST, "Unsafe WebDAV XML")
            try:
                document = ElementTree.fromstring(data)
                base = urlsplit(source.location)
                prefix = posixpath.normpath(unquote(base.path)).rstrip("/") + "/"
                seen = set()
                for response in document.findall("{DAV:}response"):
                    href = response.findtext("{DAV:}href")
                    if not href or response.find(".//{DAV:}collection") is not None:
                        continue
                    url = urljoin(source.location, href)
                    parsed = urlsplit(url)
                    if (
                        (parsed.scheme, parsed.netloc) != (base.scheme, base.netloc)
                        or not posixpath.normpath(unquote(parsed.path)).startswith(
                            prefix
                        )
                        or "\\" in unquote(parsed.path)
                        or url in seen
                        or parsed.query
                        or parsed.fragment
                    ):
                        continue
                    caption = path_caption(unquote(parsed.path))
                    if source.organized_only and caption is None:
                        continue
                    if Path(parsed.path).suffix.lower() in self.extensions:
                        seen.add(url)
                        items.append(
                            {
                                "id": url,
                                "name": unquote(Path(parsed.path).name),
                                "caption": caption if source.folder_caption else None,
                            }
                        )
                    if len(items) >= source.catalog_limit:
                        break
            except ElementTree.ParseError:
                raise FlowError(E.INVALID_REQUEST, "Invalid WebDAV catalog") from None
        if size_updates:
            with self.db.transaction() as connection:
                connection.executemany(
                    "INSERT OR REPLACE INTO photo_dimensions(path,mtime,bytes,width,height) VALUES(?,?,?,?,?)",
                    size_updates,
                )
        self.cache[source_id] = (time.monotonic(), items, wanted, session_identity)
        return items

    @staticmethod
    def resolution_matches(source, width, height):
        short, long = sorted((width, height))
        return short >= source.minimum_short_edge and long >= source.minimum_long_edge

    def read(self, source_id, item_id=None, *, target_size=(1200, 1600)):
        source = self.get(source_id)
        browsed = (
            self.browser.cached(source_id, item_id)
            if item_id and source.kind == "synology-photos"
            else None
        )
        items = (
            [browsed[1]]
            if browsed
            else self.catalog(source_id, target_size=target_size)
        )
        if not item_id:
            items = eligible(self.db, source_id, items)
        if not items:
            raise FlowError(
                E.NOT_FOUND, "Photo source contains no eligible images", 404
            )
        item = (
            next((x for x in items if x["id"] == item_id), None)
            if item_id
            else random.SystemRandom().choice(items)
        )
        if item is None:
            raise FlowError(E.NOT_FOUND, "Photo is not in this source", 404)
        if source.kind == "uploads":
            row = self.db.one(
                "SELECT id,caption FROM source_images WHERE source_id=? AND item_id=?",
                (source_id, item["id"]),
            )
            if not row:
                raise FlowError(E.NOT_FOUND, "Uploaded source picture not found", 404)
            return (
                self.assets.path(row["id"]).read_bytes(),
                item["name"],
                json.loads(row["caption"]),
            )
        if source.kind == "local":
            root = self.settings.photo_root.resolve()
            path = root / item["id"]
            if item.get("provider_preview"):
                preview = self.thumbnail(path)
                if preview is None:
                    raise FlowError(E.NOT_FOUND, "Synology preview is unavailable", 404)
                path = preview
            if path.is_symlink() or not path.resolve().is_relative_to(
                self.local_directory(source.location)
            ):
                raise FlowError(E.FORBIDDEN, "Photo path is outside its source", 403)
            try:
                # Open every component without following links; a renamed parent
                # must not turn a previously cataloged image into a secret-file read.
                parent_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
                try:
                    components = path.relative_to(root).parts
                    for component in components[:-1]:
                        child_fd = os.open(
                            component,
                            os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
                            dir_fd=parent_fd,
                        )
                        os.close(parent_fd)
                        parent_fd = child_fd
                    file_fd = os.open(
                        components[-1],
                        os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK,
                        dir_fd=parent_fd,
                    )
                    with os.fdopen(file_fd, "rb") as handle:
                        if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
                            raise FlowError(
                                E.FORBIDDEN, "Photo must be a regular file", 403
                            )
                        data = handle.read(self.settings.max_bytes + 1)
                finally:
                    os.close(parent_fd)
            except OSError:
                raise FlowError(E.NOT_FOUND, "Photo is unavailable", 404) from None
        elif source.kind == "synology-photos":
            data = self.synology.download(
                browsed[2] if browsed else self.api_items[source_id][item["id"]],
                space=source.people_space,
            )
            if source.folder_caption:
                caption = dict(item.get("caption") or {})
                try:
                    caption["people"] = " & ".join(
                        [p["name"] for p in item.get("people", []) if p["name"]]
                        if browsed
                        else self.synology.item_people(
                            item["id"], space=source.people_space
                        )
                    )
                except FlowError:
                    # Optional labels must not prevent displaying a valid photo.
                    logging.getLogger(__name__).warning(
                        "Photos person labels unavailable; keeping photo caption"
                    )
                item = {
                    **item,
                    "caption": {k: v for k, v in caption.items() if v} or None,
                }
        elif source.kind in ("google-drive", "alipan", "baidu"):
            from pigway_frame_flow.providers.cloud_drives import CloudDrives

            data = CloudDrives(self).read(source_id, source, item)
        else:
            data = self.assets.fetch_url(item["id"], headers=self.headers(source))
        if len(data) > self.settings.max_bytes:
            raise FlowError(E.INVALID_REQUEST, "Image exceeds size limit")
        if source.orientation or source.minimum_short_edge or source.minimum_long_edge:
            from io import BytesIO

            from PIL import Image

            with Image.open(BytesIO(data), formats=("JPEG", "PNG", "WEBP")) as image:
                width, height = image.size
                if image.getexif().get(274, 1) in (5, 6, 7, 8):
                    width, height = height, width
                wanted = (
                    orientation(*target_size)
                    if source.orientation == "auto"
                    else source.orientation
                )
                # Validate the fetched rendition before a crop can hide a mismatch.
                if wanted and orientation(width, height) != wanted:
                    raise FlowError(
                        E.INVALID_REQUEST,
                        "Photo direction does not match the source orientation",
                    )
                if not self.resolution_matches(source, width, height):
                    raise FlowError(
                        E.INVALID_REQUEST,
                        "Photo is below the configured minimum resolution",
                    )
        return data, item["name"], item.get("caption")