Add optional photo providers, layout studio and scoped integrations
This commit is contained in:
Pigway
@@ -0,0 +1,227 @@
|
||||
"""Read-only OAuth-token drive readers. Credentials never enter catalog metadata."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
from pigway_frame_flow.errors import ErrorCode as E
|
||||
from pigway_frame_flow.errors import FlowError
|
||||
from pigway_frame_flow.settings import secret
|
||||
|
||||
|
||||
class CloudDrives:
|
||||
def __init__(self, photos):
|
||||
self.photos = photos
|
||||
self.settings = photos.settings
|
||||
|
||||
def credentials(self, source_id, source):
|
||||
raw = secret(source.credential_env) if source.credential_env else ""
|
||||
path = self.settings.config_dir / ("source-" + source_id + ".secret")
|
||||
if not raw and path.exists():
|
||||
raw = path.read_text()
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
if (
|
||||
not isinstance(value.get("access_token"), str)
|
||||
or not value["access_token"]
|
||||
):
|
||||
raise ValueError()
|
||||
for key in ["access_token", "refresh_token", "client_secret"]:
|
||||
if value.get(key):
|
||||
self.settings.additional_secrets.append(value[key])
|
||||
return value
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise FlowError(
|
||||
E.CONFIG_REQUIRED,
|
||||
"Authorize this cloud drive with a valid OAuth access token",
|
||||
409,
|
||||
) from None
|
||||
|
||||
def fetch(self, url, **kwargs):
|
||||
# Only fixed provider API hosts and explicit download domains reach the network.
|
||||
host = urlsplit(url).hostname
|
||||
trusted = {"www.googleapis.com", "openapi.aliyundrive.com", "pan.baidu.com"}
|
||||
if host in trusted and host not in self.settings.image_url_hosts:
|
||||
self.settings.image_url_hosts.append(host)
|
||||
return self.photos.assets.fetch_url(url, **kwargs)
|
||||
|
||||
def api(self, url, token, *, body=None, params=None):
|
||||
headers = {"Authorization": "Bearer " + token}
|
||||
if body is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
try:
|
||||
raw = self.fetch(
|
||||
url + ("?" + urlencode(params) if params else ""),
|
||||
method="POST" if body is not None else "GET",
|
||||
headers=headers,
|
||||
body=json.dumps(body).encode() if body is not None else None,
|
||||
limit=2 * 1024 * 1024,
|
||||
)
|
||||
data = json.loads(raw)
|
||||
if data.get("error") or data.get("code") or data.get("errno", 0):
|
||||
raise ValueError()
|
||||
return data
|
||||
except Exception:
|
||||
raise FlowError(
|
||||
E.CLOUD_UNAVAILABLE,
|
||||
"Cloud drive query failed; check authorization and permissions",
|
||||
502,
|
||||
) from None
|
||||
|
||||
def catalog(self, source_id, source):
|
||||
credentials = self.credentials(source_id, source)
|
||||
token = credentials["access_token"]
|
||||
items: list[dict] = []
|
||||
queue = [source.location]
|
||||
visited = set()
|
||||
page_count = 0
|
||||
while queue and len(items) < source.catalog_limit:
|
||||
folder = queue.pop(0)
|
||||
if folder in visited:
|
||||
continue
|
||||
visited.add(folder)
|
||||
cursor = None
|
||||
offset = 0
|
||||
while len(items) < source.catalog_limit:
|
||||
page_count += 1
|
||||
if page_count > 200:
|
||||
raise FlowError(
|
||||
E.CONFLICT,
|
||||
"Drive query exceeds 200 pages; select a smaller folder",
|
||||
409,
|
||||
)
|
||||
if source.kind == "google-drive":
|
||||
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,200}", folder):
|
||||
raise FlowError(
|
||||
E.INVALID_REQUEST, "Invalid Google Drive folder ID"
|
||||
)
|
||||
page = self.api(
|
||||
"https://www.googleapis.com/drive/v3/files",
|
||||
token,
|
||||
params={
|
||||
"q": f"'{folder}' in parents and trashed = false",
|
||||
"pageSize": 1000,
|
||||
"fields": "nextPageToken,files(id,name,mimeType)",
|
||||
**({"pageToken": cursor} if cursor else {}),
|
||||
},
|
||||
)
|
||||
rows = page.get("files", [])
|
||||
next_page = page.get("nextPageToken")
|
||||
elif source.kind == "alipan":
|
||||
drive = credentials.get("drive_id")
|
||||
if not drive:
|
||||
raise FlowError(
|
||||
E.CONFIG_REQUIRED,
|
||||
"AliPan credentials require drive_id",
|
||||
409,
|
||||
)
|
||||
page = self.api(
|
||||
"https://openapi.aliyundrive.com/adrive/v1.0/openFile/list",
|
||||
token,
|
||||
body={
|
||||
"drive_id": drive,
|
||||
"parent_file_id": folder,
|
||||
"limit": 100,
|
||||
**({"marker": cursor} if cursor else {}),
|
||||
},
|
||||
)
|
||||
rows = page.get("items", [])
|
||||
next_page = page.get("next_marker")
|
||||
else:
|
||||
page = self.api(
|
||||
"https://pan.baidu.com/rest/2.0/xpan/file",
|
||||
token,
|
||||
params={
|
||||
"method": "list",
|
||||
"dir": folder,
|
||||
"start": offset,
|
||||
"limit": 1000,
|
||||
"access_token": token,
|
||||
},
|
||||
)
|
||||
rows = page.get("list", [])
|
||||
next_page = str(offset + 1000) if len(rows) == 1000 else None
|
||||
for row in rows:
|
||||
name = row.get("name", row.get("server_filename", ""))
|
||||
id = str(row.get("id", row.get("file_id", row.get("fs_id", ""))))
|
||||
directory = (
|
||||
row.get("mimeType") == "application/vnd.google-apps.folder"
|
||||
or row.get("type") == "folder"
|
||||
or row.get("isdir") == 1
|
||||
)
|
||||
if directory:
|
||||
if source.recursive:
|
||||
queue.append(row.get("path", id))
|
||||
continue
|
||||
if (
|
||||
not id
|
||||
or Path(name).suffix.lower() not in self.photos.extensions
|
||||
):
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"id": id,
|
||||
"name": name,
|
||||
"caption": None,
|
||||
"provider_preview": False,
|
||||
}
|
||||
)
|
||||
if len(items) >= source.catalog_limit:
|
||||
break
|
||||
if not next_page:
|
||||
break
|
||||
if next_page == cursor:
|
||||
raise FlowError(
|
||||
E.CLOUD_UNAVAILABLE, "Drive pagination did not advance", 502
|
||||
)
|
||||
cursor = next_page
|
||||
offset += 1000
|
||||
return items
|
||||
|
||||
def read(self, source_id, source, item):
|
||||
credentials = self.credentials(source_id, source)
|
||||
token = credentials["access_token"]
|
||||
id = item["id"]
|
||||
if source.kind == "google-drive":
|
||||
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,200}", id):
|
||||
raise FlowError(E.INVALID_REQUEST, "Invalid file ID")
|
||||
return self.fetch(
|
||||
"https://www.googleapis.com/drive/v3/files/" + id + "?alt=media",
|
||||
headers={"Authorization": "Bearer " + token},
|
||||
)
|
||||
suffixes: tuple[str, ...]
|
||||
if source.kind == "alipan":
|
||||
result = self.api(
|
||||
"https://openapi.aliyundrive.com/adrive/v1.0/openFile/getDownloadUrl",
|
||||
token,
|
||||
body={"drive_id": credentials["drive_id"], "file_id": id},
|
||||
)
|
||||
url = result.get("url", "")
|
||||
suffixes = ("aliyuncs.com", "alipan.com", "aliyundrive.com")
|
||||
else:
|
||||
result = self.api(
|
||||
"https://pan.baidu.com/rest/2.0/xpan/multimedia",
|
||||
token,
|
||||
params={
|
||||
"method": "filemetas",
|
||||
"fsids": json.dumps([int(id)]),
|
||||
"dlink": 1,
|
||||
"access_token": token,
|
||||
},
|
||||
)
|
||||
url = result.get("list", [{}])[0].get("dlink", "")
|
||||
suffixes = ("baidupcs.com", "baidu.com")
|
||||
host = urlsplit(url).hostname or ""
|
||||
if not any(
|
||||
host == suffix or host.endswith("." + suffix) for suffix in suffixes
|
||||
):
|
||||
raise FlowError(E.FORBIDDEN, "Unexpected drive download host", 403)
|
||||
if host not in self.settings.image_url_hosts:
|
||||
self.settings.image_url_hosts.append(host)
|
||||
if source.kind == "baidu":
|
||||
url += ("&" if "?" in url else "?") + urlencode({"access_token": token})
|
||||
return self.fetch(
|
||||
url,
|
||||
headers={"User-Agent": "pan.baidu.com"} if source.kind == "baidu" else {},
|
||||
)
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Optional read-only calendar and state providers, no vendor dependency in templates."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
import requests
|
||||
from pydantic import Field
|
||||
|
||||
from pigway_frame_flow.errors import ErrorCode as E
|
||||
from pigway_frame_flow.errors import FlowError
|
||||
from pigway_frame_flow.models import StrictModel
|
||||
from pigway_frame_flow.storage.database import encode
|
||||
|
||||
|
||||
class ExternalProviderConfig(StrictModel):
|
||||
kind: str = Field(pattern=r"^(google-calendar|ics|home-assistant|http-json)$")
|
||||
name: str = Field(min_length=1, max_length=120)
|
||||
endpoint: str = Field(max_length=2048)
|
||||
enabled: bool = True
|
||||
allow_insecure_lan: bool = False
|
||||
entities: list[str] = Field(default_factory=list, max_length=100)
|
||||
|
||||
|
||||
class ExternalProviders:
|
||||
def __init__(self, service):
|
||||
self.service = service
|
||||
self.cache = {}
|
||||
|
||||
def list(self):
|
||||
return [
|
||||
dict(json.loads(row["payload"]), id=row["key"][9:])
|
||||
for row in self.service.db.all(
|
||||
"SELECT key,payload FROM app_config WHERE key LIKE 'provider:%'"
|
||||
)
|
||||
]
|
||||
|
||||
def config(self, id):
|
||||
row = self.service.db.one(
|
||||
"SELECT payload FROM app_config WHERE key=?", ("provider:" + id,)
|
||||
)
|
||||
if not row:
|
||||
raise FlowError(E.NOT_FOUND, "Provider not found", 404)
|
||||
return ExternalProviderConfig.model_validate_json(row["payload"])
|
||||
|
||||
def save(self, id, config):
|
||||
if not re.fullmatch(r"[a-z0-9-]{1,60}", id):
|
||||
raise FlowError(E.INVALID_REQUEST, "Invalid provider ID")
|
||||
if config.kind != "google-calendar":
|
||||
parsed = urlsplit(config.endpoint)
|
||||
if (
|
||||
parsed.scheme not in ("http", "https")
|
||||
or not parsed.hostname
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.fragment
|
||||
):
|
||||
raise FlowError(
|
||||
E.INVALID_REQUEST,
|
||||
"Provider needs an HTTP(S) endpoint without credentials",
|
||||
)
|
||||
if not config.allow_insecure_lan and (
|
||||
parsed.scheme != "https"
|
||||
or parsed.hostname not in self.service.settings.image_url_hosts
|
||||
):
|
||||
raise FlowError(
|
||||
E.FORBIDDEN,
|
||||
"Provider host must be allowed; LAN access requires explicit opt-in",
|
||||
403,
|
||||
)
|
||||
self.service.db.execute(
|
||||
"INSERT OR REPLACE INTO app_config VALUES(?,?)",
|
||||
("provider:" + id, encode(config.model_dump())),
|
||||
)
|
||||
self.cache.pop(id, None)
|
||||
return dict(config.model_dump(), id=id)
|
||||
|
||||
def get(self, id):
|
||||
config = self.config(id)
|
||||
if not config.enabled:
|
||||
raise FlowError(E.CONFLICT, "Provider disabled", 409)
|
||||
if id in self.cache and time.monotonic() < self.cache[id][0]:
|
||||
return self.cache[id][1]
|
||||
secretfile = self.service.settings.config_dir / ("provider-" + id + ".secret")
|
||||
token = secretfile.read_text().strip() if secretfile.exists() else ""
|
||||
if token:
|
||||
self.service.settings.additional_secrets.append(token)
|
||||
headers = {"Authorization": "Bearer " + token} if token else {}
|
||||
if config.kind == "google-calendar":
|
||||
if not token:
|
||||
raise FlowError(
|
||||
E.CONFIG_REQUIRED,
|
||||
"Google Calendar requires an authorized OAuth token",
|
||||
409,
|
||||
)
|
||||
start = datetime.now(UTC)
|
||||
from urllib.parse import urlencode
|
||||
|
||||
url = (
|
||||
"https://www.googleapis.com/calendar/v3/calendars/"
|
||||
+ quote(config.endpoint, safe="")
|
||||
+ "/events?"
|
||||
+ urlencode(
|
||||
{
|
||||
"timeMin": start.isoformat(),
|
||||
"timeMax": (start + timedelta(days=7)).isoformat(),
|
||||
"singleEvents": "true",
|
||||
"orderBy": "startTime",
|
||||
"maxResults": 100,
|
||||
}
|
||||
)
|
||||
)
|
||||
if "www.googleapis.com" not in self.service.settings.image_url_hosts:
|
||||
self.service.settings.image_url_hosts.append("www.googleapis.com")
|
||||
else:
|
||||
url = config.endpoint.rstrip("/") + (
|
||||
"/api/states" if config.kind == "home-assistant" else ""
|
||||
)
|
||||
try:
|
||||
if config.allow_insecure_lan:
|
||||
# Endpoint is an explicitly administrator-approved, fixed LAN destination.
|
||||
with requests.get(
|
||||
url, headers=headers, timeout=10, allow_redirects=False, stream=True
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
parts = []
|
||||
total = 0
|
||||
for part in response.iter_content(65536):
|
||||
total += len(part)
|
||||
if total > 2 * 1024 * 1024:
|
||||
raise ValueError()
|
||||
parts.append(part)
|
||||
raw = b"".join(parts)
|
||||
else:
|
||||
raw = self.service.assets.fetch_url(
|
||||
url, headers=headers, limit=2 * 1024 * 1024
|
||||
)
|
||||
result: dict
|
||||
if config.kind == "ics":
|
||||
events = []
|
||||
text = (
|
||||
raw.decode("utf-8-sig").replace("\r\n ", "").replace("\r\n", "\n")
|
||||
)
|
||||
for block in text.split("BEGIN:VEVENT")[1:]:
|
||||
block = block.split("END:VEVENT")[0]
|
||||
values = {
|
||||
line.split(":", 1)[0].split(";")[0]: line.split(":", 1)[1]
|
||||
for line in block.splitlines()
|
||||
if ":" in line
|
||||
}
|
||||
if "RRULE" in values:
|
||||
raise FlowError(
|
||||
E.UNSUPPORTED_CAPABILITY,
|
||||
"Recurring ICS events require a calendar provider with expanded instances",
|
||||
422,
|
||||
)
|
||||
if values.get("SUMMARY"):
|
||||
events.append(values["SUMMARY"].replace("\\n", " "))
|
||||
result = {"events": events[:100]}
|
||||
elif config.kind == "google-calendar":
|
||||
result = {
|
||||
"events": [
|
||||
{
|
||||
"title": x.get("summary", "Untitled"),
|
||||
"start": x.get("start", {}).get(
|
||||
"dateTime", x.get("start", {}).get("date")
|
||||
),
|
||||
}
|
||||
for x in json.loads(raw).get("items", [])
|
||||
]
|
||||
}
|
||||
elif config.kind == "home-assistant":
|
||||
result = {
|
||||
"events": [
|
||||
x["entity_id"] + ": " + str(x["state"])
|
||||
for x in json.loads(raw)
|
||||
if not config.entities or x["entity_id"] in config.entities
|
||||
]
|
||||
}
|
||||
else:
|
||||
result = json.loads(raw)
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError()
|
||||
result = self.service.settings.sanitize(result)
|
||||
self.cache[id] = (time.monotonic() + 300, result)
|
||||
return result
|
||||
except FlowError:
|
||||
raise
|
||||
except Exception:
|
||||
raise FlowError(
|
||||
E.CLOUD_UNAVAILABLE,
|
||||
"External provider unavailable; check connection and credentials",
|
||||
502,
|
||||
) from None
|
||||
@@ -27,7 +27,7 @@ from pigway_frame_flow.storage.database import encode
|
||||
|
||||
class PhotoSourceInput(StrictModel):
|
||||
name: str = Field(min_length=1, max_length=120)
|
||||
kind: Literal["local", "webdav"]
|
||||
kind: Literal["local", "webdav", "google-drive", "alipan", "baidu"]
|
||||
location: str = Field(min_length=1, max_length=2048)
|
||||
recursive: bool = True
|
||||
enabled: bool = True
|
||||
@@ -36,6 +36,11 @@ class PhotoSourceInput(StrictModel):
|
||||
organized_only: bool = False
|
||||
folder_caption: bool = False
|
||||
orientation: Literal["auto", "landscape", "portrait"] | None = None
|
||||
minimum_sharpness: float = Field(default=0, ge=0, le=10000)
|
||||
minimum_brightness: float = Field(default=0, ge=0, le=255)
|
||||
maximum_brightness: float = Field(default=255, ge=0, le=255)
|
||||
avoid_duplicates: bool = False
|
||||
people_space: Literal["shared", "personal"] = "shared"
|
||||
people_filter: bool = False
|
||||
person_ids: list[int] = Field(default_factory=list, max_length=50)
|
||||
person_match: Literal["any", "all"] = "any"
|
||||
@@ -102,6 +107,15 @@ class PhotoSources:
|
||||
return PhotoSourceInput.model_validate_json(row["payload"])
|
||||
|
||||
def save(self, source_id, source):
|
||||
if source.minimum_brightness > source.maximum_brightness:
|
||||
raise FlowError(E.INVALID_REQUEST, "Brightness range is reversed")
|
||||
if source.kind != "local" and (
|
||||
source.organized_only or source.folder_caption or source.synology_previews
|
||||
):
|
||||
raise FlowError(
|
||||
E.UNSUPPORTED_CAPABILITY,
|
||||
"Folder captions and Synology previews require a local photo source",
|
||||
)
|
||||
if source.orientation and source.kind != "local":
|
||||
raise FlowError(
|
||||
E.UNSUPPORTED_CAPABILITY,
|
||||
@@ -126,7 +140,7 @@ class PhotoSources:
|
||||
raise FlowError(
|
||||
E.INVALID_REQUEST, "Local sources do not use credentials"
|
||||
)
|
||||
else:
|
||||
elif source.kind == "webdav":
|
||||
source = source.model_copy(
|
||||
update={
|
||||
"recursive": False,
|
||||
@@ -266,7 +280,12 @@ class PhotoSources:
|
||||
):
|
||||
return cached[1]
|
||||
allowed = (
|
||||
self.synology.paths(source.person_ids, source.person_match, refresh=refresh)
|
||||
self.synology.paths(
|
||||
source.person_ids,
|
||||
source.person_match,
|
||||
refresh=refresh,
|
||||
space=source.people_space,
|
||||
)
|
||||
if source.people_filter
|
||||
else None
|
||||
)
|
||||
@@ -387,6 +406,10 @@ class PhotoSources:
|
||||
break
|
||||
if len(items) >= source.catalog_limit:
|
||||
break
|
||||
elif source.kind in ("google-drive", "alipan", "baidu"):
|
||||
from pigway_frame_flow.providers.cloud_drives import CloudDrives
|
||||
|
||||
items = CloudDrives(self).catalog(source_id, source)
|
||||
else:
|
||||
data = self.assets.fetch_url(
|
||||
source.location,
|
||||
@@ -503,6 +526,10 @@ class PhotoSources:
|
||||
os.close(parent_fd)
|
||||
except OSError:
|
||||
raise FlowError(E.NOT_FOUND, "Photo is unavailable", 404) from None
|
||||
elif source.kind in ("google-drive", "alipan", "baidu"):
|
||||
from pigway_frame_flow.providers.cloud_drives import CloudDrives
|
||||
|
||||
data = CloudDrives(self).read(source_id, source, item)
|
||||
else:
|
||||
data = self.assets.fetch_url(item["id"], headers=self.headers(source))
|
||||
if len(data) > self.settings.max_bytes:
|
||||
|
||||
@@ -83,6 +83,9 @@ class SynologyPhotos:
|
||||
("SYNO.API.Auth", "login"),
|
||||
("SYNO.API.Auth", "logout"),
|
||||
("SYNO.FotoTeam.Browse.Person", "list"),
|
||||
("SYNO.Foto.Browse.Person", "list"),
|
||||
("SYNO.Foto.Browse.Item", "list"),
|
||||
("SYNO.Foto.Browse.Folder", "get"),
|
||||
("SYNO.FotoTeam.Browse.Item", "list"),
|
||||
("SYNO.FotoTeam.Browse.Folder", "get"),
|
||||
}:
|
||||
@@ -193,9 +196,10 @@ class SynologyPhotos:
|
||||
return rows
|
||||
raise FlowError(E.CONFLICT, "Photos catalog exceeds query limit", 409)
|
||||
|
||||
def people(self):
|
||||
def people(self, space="shared"):
|
||||
prefix = "SYNO.FotoTeam" if space == "shared" else "SYNO.Foto"
|
||||
with self.lock:
|
||||
rows = self.pages("SYNO.FotoTeam.Browse.Person", 3)
|
||||
rows = self.pages(prefix + ".Browse.Person", 3)
|
||||
return [
|
||||
{
|
||||
"id": r["id"],
|
||||
@@ -206,13 +210,17 @@ class SynologyPhotos:
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def paths(self, person_ids, mode, refresh=False):
|
||||
key = (tuple(sorted(person_ids)), mode)
|
||||
def paths(self, person_ids, mode, refresh=False, *, space="shared"):
|
||||
prefix = "SYNO.FotoTeam" if space == "shared" else "SYNO.Foto"
|
||||
key = (tuple(sorted(person_ids)), mode, space)
|
||||
with self.lock:
|
||||
cached = self.cache.get(key)
|
||||
if cached and not refresh and time.monotonic() - cached[0] < 900:
|
||||
return cached[1]
|
||||
known = {p["id"] for p in self.people()}
|
||||
known = {
|
||||
p["id"]
|
||||
for p in (self.people() if space == "shared" else self.people(space))
|
||||
}
|
||||
ids = person_ids or sorted(known)
|
||||
if not set(ids) <= known:
|
||||
raise FlowError(E.INVALID_REQUEST, "Selected person no longer exists")
|
||||
@@ -221,7 +229,7 @@ class SynologyPhotos:
|
||||
for person in ids:
|
||||
paths = set()
|
||||
for row in self.pages(
|
||||
"SYNO.FotoTeam.Browse.Item",
|
||||
prefix + ".Browse.Item",
|
||||
7,
|
||||
person_id=person,
|
||||
additional=json.dumps(["folder"]),
|
||||
@@ -241,7 +249,7 @@ class SynologyPhotos:
|
||||
)
|
||||
if folder_id not in folders:
|
||||
folders[folder_id] = self.request(
|
||||
"SYNO.FotoTeam.Browse.Folder",
|
||||
prefix + ".Browse.Folder",
|
||||
"get",
|
||||
2,
|
||||
id=folder_id,
|
||||
|
||||
Reference in New Issue
Block a user