228 lines
9.3 KiB
Python
228 lines
9.3 KiB
Python
"""Read-only OAuth-token drive readers. Credentials never enter catalog metadata."""
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from urllib.parse import urlencode, urlsplit
|
|
|
|
from pigway_frame_flow.errors import ErrorCode as E
|
|
from pigway_frame_flow.errors import FlowError
|
|
from pigway_frame_flow.settings import secret
|
|
|
|
|
|
class CloudDrives:
|
|
def __init__(self, photos):
|
|
self.photos = photos
|
|
self.settings = photos.settings
|
|
|
|
def credentials(self, source_id, source):
|
|
raw = secret(source.credential_env) if source.credential_env else ""
|
|
path = self.settings.config_dir / ("source-" + source_id + ".secret")
|
|
if not raw and path.exists():
|
|
raw = path.read_text()
|
|
try:
|
|
value = json.loads(raw)
|
|
if (
|
|
not isinstance(value.get("access_token"), str)
|
|
or not value["access_token"]
|
|
):
|
|
raise ValueError()
|
|
for key in ["access_token", "refresh_token", "client_secret"]:
|
|
if value.get(key):
|
|
self.settings.additional_secrets.append(value[key])
|
|
return value
|
|
except (ValueError, TypeError, AttributeError):
|
|
raise FlowError(
|
|
E.CONFIG_REQUIRED,
|
|
"Authorize this cloud drive with a valid OAuth access token",
|
|
409,
|
|
) from None
|
|
|
|
def fetch(self, url, **kwargs):
|
|
# Only fixed provider API hosts and explicit download domains reach the network.
|
|
host = urlsplit(url).hostname
|
|
trusted = {"www.googleapis.com", "openapi.aliyundrive.com", "pan.baidu.com"}
|
|
if host in trusted and host not in self.settings.image_url_hosts:
|
|
self.settings.image_url_hosts.append(host)
|
|
return self.photos.assets.fetch_url(url, **kwargs)
|
|
|
|
def api(self, url, token, *, body=None, params=None):
|
|
headers = {"Authorization": "Bearer " + token}
|
|
if body is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
try:
|
|
raw = self.fetch(
|
|
url + ("?" + urlencode(params) if params else ""),
|
|
method="POST" if body is not None else "GET",
|
|
headers=headers,
|
|
body=json.dumps(body).encode() if body is not None else None,
|
|
limit=2 * 1024 * 1024,
|
|
)
|
|
data = json.loads(raw)
|
|
if data.get("error") or data.get("code") or data.get("errno", 0):
|
|
raise ValueError()
|
|
return data
|
|
except Exception:
|
|
raise FlowError(
|
|
E.CLOUD_UNAVAILABLE,
|
|
"Cloud drive query failed; check authorization and permissions",
|
|
502,
|
|
) from None
|
|
|
|
def catalog(self, source_id, source):
|
|
credentials = self.credentials(source_id, source)
|
|
token = credentials["access_token"]
|
|
items: list[dict] = []
|
|
queue = [source.location]
|
|
visited = set()
|
|
page_count = 0
|
|
while queue and len(items) < source.catalog_limit:
|
|
folder = queue.pop(0)
|
|
if folder in visited:
|
|
continue
|
|
visited.add(folder)
|
|
cursor = None
|
|
offset = 0
|
|
while len(items) < source.catalog_limit:
|
|
page_count += 1
|
|
if page_count > 200:
|
|
raise FlowError(
|
|
E.CONFLICT,
|
|
"Drive query exceeds 200 pages; select a smaller folder",
|
|
409,
|
|
)
|
|
if source.kind == "google-drive":
|
|
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,200}", folder):
|
|
raise FlowError(
|
|
E.INVALID_REQUEST, "Invalid Google Drive folder ID"
|
|
)
|
|
page = self.api(
|
|
"https://www.googleapis.com/drive/v3/files",
|
|
token,
|
|
params={
|
|
"q": f"'{folder}' in parents and trashed = false",
|
|
"pageSize": 1000,
|
|
"fields": "nextPageToken,files(id,name,mimeType)",
|
|
**({"pageToken": cursor} if cursor else {}),
|
|
},
|
|
)
|
|
rows = page.get("files", [])
|
|
next_page = page.get("nextPageToken")
|
|
elif source.kind == "alipan":
|
|
drive = credentials.get("drive_id")
|
|
if not drive:
|
|
raise FlowError(
|
|
E.CONFIG_REQUIRED,
|
|
"AliPan credentials require drive_id",
|
|
409,
|
|
)
|
|
page = self.api(
|
|
"https://openapi.aliyundrive.com/adrive/v1.0/openFile/list",
|
|
token,
|
|
body={
|
|
"drive_id": drive,
|
|
"parent_file_id": folder,
|
|
"limit": 100,
|
|
**({"marker": cursor} if cursor else {}),
|
|
},
|
|
)
|
|
rows = page.get("items", [])
|
|
next_page = page.get("next_marker")
|
|
else:
|
|
page = self.api(
|
|
"https://pan.baidu.com/rest/2.0/xpan/file",
|
|
token,
|
|
params={
|
|
"method": "list",
|
|
"dir": folder,
|
|
"start": offset,
|
|
"limit": 1000,
|
|
"access_token": token,
|
|
},
|
|
)
|
|
rows = page.get("list", [])
|
|
next_page = str(offset + 1000) if len(rows) == 1000 else None
|
|
for row in rows:
|
|
name = row.get("name", row.get("server_filename", ""))
|
|
id = str(row.get("id", row.get("file_id", row.get("fs_id", ""))))
|
|
directory = (
|
|
row.get("mimeType") == "application/vnd.google-apps.folder"
|
|
or row.get("type") == "folder"
|
|
or row.get("isdir") == 1
|
|
)
|
|
if directory:
|
|
if source.recursive:
|
|
queue.append(row.get("path", id))
|
|
continue
|
|
if (
|
|
not id
|
|
or Path(name).suffix.lower() not in self.photos.extensions
|
|
):
|
|
continue
|
|
items.append(
|
|
{
|
|
"id": id,
|
|
"name": name,
|
|
"caption": None,
|
|
"provider_preview": False,
|
|
}
|
|
)
|
|
if len(items) >= source.catalog_limit:
|
|
break
|
|
if not next_page:
|
|
break
|
|
if next_page == cursor:
|
|
raise FlowError(
|
|
E.CLOUD_UNAVAILABLE, "Drive pagination did not advance", 502
|
|
)
|
|
cursor = next_page
|
|
offset += 1000
|
|
return items
|
|
|
|
def read(self, source_id, source, item):
|
|
credentials = self.credentials(source_id, source)
|
|
token = credentials["access_token"]
|
|
id = item["id"]
|
|
if source.kind == "google-drive":
|
|
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,200}", id):
|
|
raise FlowError(E.INVALID_REQUEST, "Invalid file ID")
|
|
return self.fetch(
|
|
"https://www.googleapis.com/drive/v3/files/" + id + "?alt=media",
|
|
headers={"Authorization": "Bearer " + token},
|
|
)
|
|
suffixes: tuple[str, ...]
|
|
if source.kind == "alipan":
|
|
result = self.api(
|
|
"https://openapi.aliyundrive.com/adrive/v1.0/openFile/getDownloadUrl",
|
|
token,
|
|
body={"drive_id": credentials["drive_id"], "file_id": id},
|
|
)
|
|
url = result.get("url", "")
|
|
suffixes = ("aliyuncs.com", "alipan.com", "aliyundrive.com")
|
|
else:
|
|
result = self.api(
|
|
"https://pan.baidu.com/rest/2.0/xpan/multimedia",
|
|
token,
|
|
params={
|
|
"method": "filemetas",
|
|
"fsids": json.dumps([int(id)]),
|
|
"dlink": 1,
|
|
"access_token": token,
|
|
},
|
|
)
|
|
url = result.get("list", [{}])[0].get("dlink", "")
|
|
suffixes = ("baidupcs.com", "baidu.com")
|
|
host = urlsplit(url).hostname or ""
|
|
if not any(
|
|
host == suffix or host.endswith("." + suffix) for suffix in suffixes
|
|
):
|
|
raise FlowError(E.FORBIDDEN, "Unexpected drive download host", 403)
|
|
if host not in self.settings.image_url_hosts:
|
|
self.settings.image_url_hosts.append(host)
|
|
if source.kind == "baidu":
|
|
url += ("&" if "?" in url else "?") + urlencode({"access_token": token})
|
|
return self.fetch(
|
|
url,
|
|
headers={"User-Agent": "pan.baidu.com"} if source.kind == "baidu" else {},
|
|
)
|