diff --git a/docs/AUTOMATIC_COLLAGE.md b/docs/AUTOMATIC_COLLAGE.md new file mode 100644 index 0000000..41d2fd0 --- /dev/null +++ b/docs/AUTOMATIC_COLLAGE.md @@ -0,0 +1,55 @@ +# Automatic collage + +The navigation starts with Dashboard, followed by Gallery, Photo Sources, Upload, +Templates, Layout Studio, Automation, Playlists, History, Connections and Settings. + +## Web workflow / 界面操作 + +在「拼图」选择照片来源与数量（1–12），填写可选标题或排版要求，点击 +「随机选图并生成预览」。默认使用启用的数据源；没有数据源时使用本地上传照片。 +AI 只接收图片尺寸、内部资产 ID 和排版文字，不发送照片。 +AI 未配置或请求失败会明确报错，不会偷偷切换引擎；取消 AI 勾选可用自动网格。 +预览不会上传或切换画框，确认效果后再点「显示到画框」。 + +Choose a source and count, then create a preview. Only dimensions, asset IDs and +layout instructions are sent to AI. AI configuration/errors are explicit; uncheck +AI for deterministic grid layout. Preview never changes the frame. Canvas +orientation follows the target device. Manual layout is under Advanced. + +「创建自动化任务」保存来源和数量，而不是固定图片列表。任务初始暂停， +请在自动化中编辑执行时间后启用。每次运行重新随机选图，但不同运行之间可能重复； +单次不重复选同一照片。照片不足时失败，不重复图片凑数。 +NAS 来源继续使用人物、方向与质量筛选；一次最多检查数量的五倍候选图片。 +输入图缓存与生成图在 NAS；原始 NAS 照片不会被修改或删除。 +定时拼图沿用云库每任务保留最近一张成功图的策略，下一张确认成功前不删除上一张。 + +Create automation stores the source/count, initially paused. Edit its schedule +and enable it. Each execution selects new candidates; repeats across executions +are possible. Too few usable photos fails before upload. Existing unknown-write, +restoration and cloud-retention safeguards apply. + +## Public API + +`POST /api/v1/render` accepts this additive content type, returning HTTP 202 and +an operation ID. `POST /api/v1/display` uses the same content inside its existing +request envelope. Scheduler `source` accepts the same object; job previews are +read-only. + +```json +{ + "type": "auto_collage", + "source_id": "nas-photos", + "count": 4, + "title": "Our moments", + "orientation": "auto", + "style": "grid", + "use_ai": true, + "prompt": "Feature one photo with smaller supporting photos", + "profile": "color" +} +``` + +`source_id: null` chooses local uploaded assets (generated and cache images are +excluded). AI configuration is shared with existing Layout Studio. No new API key +or vendor dependency is introduced. The OpenAPI contract test records the new +content schema while keeping all existing schema fields unchanged. diff --git a/docs/UI_LAYOUT_AUDIT.md b/docs/UI_LAYOUT_AUDIT.md index b33921c..63201de 100644 --- a/docs/UI_LAYOUT_AUDIT.md +++ b/docs/UI_LAYOUT_AUDIT.md @@ -1,7 +1,7 @@ # UI layout audit The Mock-only `scripts/ui-layout-audit.cjs` checks all 11 navigation pages, -local/cloud gallery layouts and six modal states at 1440, 768 and 390 CSS pixels, +local/cloud gallery layouts and six modal states plus expanded manual collage at 1440, 768 and 390 CSS pixels, in Chinese/English and light/dark themes. It checks page horizontal overflow and at least 8 pixels between visible input/select/textarea/button rectangles, and 44-pixel control heights on mobile. @@ -32,5 +32,5 @@ node scripts/ui-layout-audit.cjs Set `NODE_PATH` to the installed Playwright runtime if needed. Optional `FRAME_FLOW_SCREENSHOT_DIR` saves fixture-only screenshots for visual review. -The full audit passed 217 checks with no console errors. This complements +The full audit passed 229 checks with no console errors. This complements `ui-smoke.cjs` and `ui-integrations.cjs`; it does not replace functional tests. diff --git a/pigway_frame_flow/api/app.py b/pigway_frame_flow/api/app.py index 9c16ff9..a5e91b0 100644 --- a/pigway_frame_flow/api/app.py +++ b/pigway_frame_flow/api/app.py @@ -29,6 +29,7 @@ from pigway_frame_flow.models import ( Accepted, Asset, AssetBatch, + AutoCollageContent, CurrentDisplay, DeleteRequest, Device, @@ -485,7 +486,7 @@ def create_app(settings=None, start_worker=True): dependencies=[Depends(require("render:write"))], ) def render( - payload: TemplateContent, + payload: TemplateContent | AutoCollageContent, request: Request, idempotency_key: str | None = Header(default=None), ): @@ -1175,9 +1176,11 @@ def create_app(settings=None, start_worker=True): idempotency_key: str | None = Header(default=None), ): source = get_job(job_id)["source"] - if source["type"] != "template": + if source["type"] not in ("template", "auto_collage"): raise FlowError( - E.UNSUPPORTED_CAPABILITY, "Job preview requires a template source", 422 + E.UNSUPPORTED_CAPABILITY, + "Job preview requires template or auto-collage content", + 422, ) return submit( request, "render", source, get_job(job_id)["device_id"], idempotency_key diff --git a/pigway_frame_flow/display/cloud_library.py b/pigway_frame_flow/display/cloud_library.py index 588de82..ce502a0 100644 --- a/pigway_frame_flow/display/cloud_library.py +++ b/pigway_frame_flow/display/cloud_library.py @@ -23,7 +23,7 @@ class CloudLibrary: job_id = ( job["job_id"] if job - and content.get("type") == "photo_source" + and content.get("type") in ("photo_source", "auto_collage") and content.get("item_id") is None else None ) diff --git a/pigway_frame_flow/display/service.py b/pigway_frame_flow/display/service.py index 51608f2..b0be75a 100644 --- a/pigway_frame_flow/display/service.py +++ b/pigway_frame_flow/display/service.py @@ -11,6 +11,7 @@ from pigway_frame_flow.drivers.base import DriverRegistry from pigway_frame_flow.errors import ErrorCode as E from pigway_frame_flow.errors import FlowError from pigway_frame_flow.models import ( + AutoCollageContent, DisplayRequest, FutureContent, ImageContent, @@ -81,6 +82,11 @@ class DisplayService: ) from None for region in plan.regions: self.assets.path(region.asset_id) + if ( + isinstance(request.content, AutoCollageContent) + and request.content.source_id + ): + self.photo_sources.get(request.content.source_id) if isinstance(request.content, PhotoSourceContent): self.photo_sources.get(request.content.source_id) if writing and not self.settings.mock and not self.settings.write_enabled: @@ -191,6 +197,12 @@ class DisplayService: return self.db.one("SELECT * FROM device_state WHERE device_id=?", (id,)) def content_asset(self, content, device_id=None, operation_id=None): + if isinstance(content, AutoCollageContent): + from pigway_frame_flow.extensions.auto_collage import prepare + + return self.content_asset( + prepare(self, content, device_id), device_id, operation_id + ) if isinstance(content, ProviderContent): values = {**content.data, **self.providers.get(content.provider_id)} return self.content_asset( diff --git a/pigway_frame_flow/extensions/auto_collage.py b/pigway_frame_flow/extensions/auto_collage.py new file mode 100644 index 0000000..183d113 --- /dev/null +++ b/pigway_frame_flow/extensions/auto_collage.py @@ -0,0 +1,80 @@ +"""Random selection and layout shared by preview, display and scheduled jobs.""" + +import random +from io import BytesIO + +from PIL import Image, ImageOps + +from pigway_frame_flow.errors import ErrorCode as E +from pigway_frame_flow.errors import FlowError +from pigway_frame_flow.extensions.layout import LayoutInput, LayoutStudio +from pigway_frame_flow.models import TemplateContent + + +def prepare(service, content, device_id=None): + rng = random.SystemRandom() + ids = [] + hashes = set() + if content.source_id: + from pigway_frame_flow.extensions.quality import passes, photo_quality + + source = service.photo_sources.get(content.source_id) + items = service.photo_sources.catalog(content.source_id) + candidates = rng.sample(items, min(len(items), content.count * 5)) + for item in candidates: + try: + data, name, _ = service.photo_sources.read( + content.source_id, item["id"] + ) + quality = photo_quality(data) + if not passes(quality, source) or quality["sha256"] in hashes: + continue + if source.avoid_duplicates and quality["fingerprint"] in hashes: + continue + hashes.update([quality["sha256"], quality["fingerprint"]]) + with Image.open(BytesIO(data)) as image: + photo = ImageOps.exif_transpose(image).convert("RGB") + photo.thumbnail((2400, 2400)) + output = BytesIO() + photo.save(output, "JPEG", quality=92) + asset = service.assets.put(output.getvalue(), name, "cache") + if asset["asset_id"] not in ids: + ids.append(asset["asset_id"]) + except (OSError, ValueError): + continue + if len(ids) == content.count: + break + else: + rows = service.db.all( + "SELECT id AS asset_id FROM assets WHERE kind='images' ORDER BY created_at DESC LIMIT 10000" + ) + ids = rng.sample( + [row["asset_id"] for row in rows], min(content.count, len(rows)) + ) + if len(ids) != content.count: + raise FlowError( + E.INVALID_REQUEST, "Not enough distinct photos for requested count", 422 + ) + values = content.model_dump(exclude={"type", "source_id", "count", "profile"}) + if content.orientation == "auto": + from pigway_frame_flow.renderers.frame_geometry import frame_canvas + + display = ( + service.registry.get(device_id or service.settings.device_id) + .device() + .display + ) + width, height = frame_canvas(display) + values["orientation"] = "landscape" if width > height else "portrait" + plan = LayoutStudio(service.renderer).plan( + LayoutInput( + asset_ids=ids, + **values, + ) + ) + return TemplateContent( + type="template", + template="poster" if content.style == "poster" else "collage", + data={"layout": plan.model_dump()}, + profile=content.profile, + ) diff --git a/pigway_frame_flow/models.py b/pigway_frame_flow/models.py index 1816459..c3a5308 100644 --- a/pigway_frame_flow/models.py +++ b/pigway_frame_flow/models.py @@ -68,6 +68,19 @@ class PhotoSourceContent(StrictModel): profile: Profile = Profile.color +class AutoCollageContent(StrictModel): + type: Literal["auto_collage"] + source_id: str | None = Field(default=None, pattern=r"^[a-z0-9-]{1,60}$") + count: int = Field(default=4, ge=1, le=12) + title: str = Field(default="", max_length=100) + message: str = Field(default="", max_length=500) + orientation: Literal["auto", "portrait", "landscape"] = "auto" + style: Literal["grid", "hero", "poster"] = "grid" + use_ai: bool = True + prompt: str = Field(default="", max_length=1500) + profile: Profile = Profile.color + + class ProviderContent(StrictModel): type: Literal["provider"] provider_id: str = Field(pattern=r"^[a-z0-9-]{1,60}$") @@ -86,6 +99,7 @@ Content = Annotated[ | URLContent | TemplateContent | PhotoSourceContent + | AutoCollageContent | ProviderContent | FutureContent, Field(discriminator="type"), diff --git a/pigway_frame_flow/renderers/images.py b/pigway_frame_flow/renderers/images.py index b76cea2..80b038b 100644 --- a/pigway_frame_flow/renderers/images.py +++ b/pigway_frame_flow/renderers/images.py @@ -97,7 +97,7 @@ class Assets: "task" if job else "template" - if content.get("type") == "template" + if content.get("type") in ("template", "auto_collage") else "manual" if operation["client_id"].startswith("user:") else "external" diff --git a/pigway_frame_flow/web/app.js b/pigway_frame_flow/web/app.js index dec928a..4b94222 100644 --- a/pigway_frame_flow/web/app.js +++ b/pigway_frame_flow/web/app.js @@ -3,7 +3,7 @@ const $=s=>document.querySelector(s); const state={lang:localStorage.getItem('ff-language')||'zh',theme:localStorage.getItem('ff-theme')||'system',page:location.hash.slice(1)||'dashboard',token:'',csrf:'',device:'living-room',settings:null,devices:[],templates:[],gallery:[],assets:[],blobs:[],tab:'cloud',preview:null,templatePreview:null,selectedTemplate:null,epoch:0}; function forgetLogin(){localStorage.removeItem('ff-login');state.token='';state.csrf='';state.settings=null;state.devices=[];state.templates=[];} async function loadConnection(){state.settings=await api('/api/v1/settings');try{state.devices=await api('/api/v1/devices');}catch(error){state.devices=[];toast(error.message);}state.templates=await api('/api/v1/templates');state.device=state.devices[0]?.id||state.device;} -const icons={studio:'▦',connections:'↔',dashboard:'◫',gallery:'▧','photo-sources':'▣',upload:'↥',templates:'▦',automation:'◷',playlists:'▤',history:'↺',settings:'⚙'}; +const icons={dashboard:'◫',gallery:'▧','photo-sources':'▣',upload:'↥',templates:'▦',studio:'▦',automation:'◷',playlists:'▤',history:'↺',connections:'↔',settings:'⚙'}; const t=k=>window.MESSAGES[state.lang][k]||k; const esc=v=>String(v??'—').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); const fmt=v=>v?new Date(v).toLocaleString(state.lang==='zh'?'zh-CN':'en-GB',{timeZone:'Asia/Shanghai'}):'—'; diff --git a/pigway_frame_flow/web/extensions.js b/pigway_frame_flow/web/extensions.js index 023ac27..8f2f099 100644 --- a/pigway_frame_flow/web/extensions.js +++ b/pigway_frame_flow/web/extensions.js @@ -3,6 +3,7 @@ const tr=(zh,en)=>state.lang==='zh'?zh:en; async function layoutStudioPage(){ const assets=await api('/api/v1/assets?source=uploaded&limit=200');let selected=new Set(),plan=null; $('#main').innerHTML=`
${tr('一次选择 1–12 张图片，生成多图拼版或海报。AI 只接收文字与尺寸，不接收照片。预览不会上传到画框。','Choose 1–12 photos for a collage or poster. AI receives text and dimensions, never photos. Preview does not upload to the frame.')}

${tr('选择照片','Choose photos')}

${tr('可以一次上传多张，保留原始比例。或勾选已有本地照片。','Upload several photos at once, preserving aspect ratio, or select local photos.')}

${assets.map(a=>``).join('')}

${tr('规划后可拖动照片区域或修改坐标尺寸。区域不能重叠，超出范围的布局会被拒绝。','After planning, drag photo regions or edit coordinates and sizes. Overlapping or out-of-bounds layouts are rejected.')}

${t('preview')}

${tr('先规划布局，再生成最终图片','Plan the layout, then render the final image')}
`; + await autoCollagePanel(); let rendered=null; const invalidate=()=>{rendered=null;$('#studio-display').disabled=true;$('#studio-download').disabled=true;}; $('#studio-assets').onchange=()=>{selected=new Set([...document.querySelectorAll('#studio-assets input:checked')].map(x=>x.value));plan=null;$('#studio-render').disabled=true;invalidate();}; @@ -56,3 +57,15 @@ async function aiImagePanel(){ async function mqttPanel(){ const status=await api('/api/v1/integrations/mqtt');const c=status.config;const card=document.createElement('section');card.className='card';card.innerHTML=`

MQTT · ${t(status.connected?'connected':'disconnected')}

${tr('默认关闭。须配置环境 Secret FRAME_FLOW_MQTT_TOKEN（有 display:write 权限的客户端 Token），以及可选 FRAME_FLOW_MQTT_PASSWORD。消息必须携带 request_id；retained 消息被拒绝。结果发送到命令主题的 /result。','Disabled by default. Configure FRAME_FLOW_MQTT_TOKEN with display:write scope and optional FRAME_FLOW_MQTT_PASSWORD as environment secrets. Commands require request_id; retained commands are rejected. Results are published to the command topic plus /result.')}

`;$('#main').append(card);$('#mqtt-config').onsubmit=busy(async e=>{e.preventDefault();await api('/api/v1/integrations/mqtt',{method:'PUT',body:{enabled:$('#mqtt-enabled').checked,host:$('#mqtt-host').value,port:Number($('#mqtt-port').value),topic:$('#mqtt-topic').value,username:$('#mqtt-username').value,tls:$('#mqtt-tls').checked,allow_insecure_lan:$('#mqtt-lan').checked}});toast(t('done'));}); } + +async function autoCollagePanel(){ + const sources=await api('/api/v1/photo-sources');const card=document.createElement('section');card.className='card auto-collage'; + card.innerHTML=`

${tr('自动拼图','Automatic collage')}

${tr('选择照片来源和数量，系统每次随机选取不同照片，并自动排版生成一张图片。无需手动拼接。预览不会上传或切换画框。','Choose a photo source and count. Each run randomly selects distinct photos and automatically creates one composition. Preview never uploads to or changes the frame.')}

${tr('AI 需先在连接中心配置，只接收尺寸与文字，不发送照片。未配置时可取消勾选，使用自动网格。','Configure AI in Connections first. Only dimensions and text are sent, never photos. Uncheck AI to use an automatic grid.')}

`; + $('#main').prepend(card);if(sources.some(x=>x.enabled))$('#collage-source').value=sources.find(x=>x.enabled).id;let asset=null; + $('#auto-collage-form').oninput=()=>{asset=null;$('#collage-display').hidden=true;}; + const content=()=>({type:'auto_collage',source_id:$('#collage-source').value||null,count:Number($('#collage-count').value),title:$('#collage-title').value,prompt:$('#collage-prompt').value,use_ai:$('#collage-ai').checked}); + $('#auto-collage-form').onsubmit=busy(async e=>{e.preventDefault();asset=null;$('#collage-display').hidden=true;$('#collage-preview').textContent=t('previewLoading');try{const id=await accepted('/api/v1/render',content());const op=await waitOperation(id);asset=op.result;$('#collage-preview').innerHTML='Collage';await blobImage($('#collage-image'),asset.url);$('#collage-display').hidden=false;$('#collage-display').disabled=!liveWriteAllowed();}catch(error){$('#collage-preview').textContent=error.message;}}); + bind('collage-display',()=>displayDialog({type:'image',asset_id:asset.asset_id})); + bind('collage-job',async()=>{if(!$('#auto-collage-form').reportValidity())return;await api('/api/v1/jobs',{method:'POST',body:{device_id:state.device,name:$('#collage-title').value||tr('自动随机拼图','Automatic random collage'),enabled:false,schedule:{type:'daily',time:'08:00'},source:content(),display_duration:0,restore_previous:false,priority:50}});await navigate('automation');toast(tr('任务已创建并暂停；编辑时间后启用。每次执行重新随机选图。','Created paused. Edit the schedule, then enable. Each run picks new photos.'));}); + const manual=document.createElement('details');manual.className='card manual-collage';manual.innerHTML=`${tr('高级：手动选图与调整布局','Advanced: manual photos and layout')}`;const columns=$('#main>.columns');columns.before(manual);manual.append(columns); +} diff --git a/pigway_frame_flow/web/features.js b/pigway_frame_flow/web/features.js index 2f31230..1966843 100644 --- a/pigway_frame_flow/web/features.js +++ b/pigway_frame_flow/web/features.js @@ -46,16 +46,17 @@ async function weatherSettings(){ async function enhanceHistory(){ const table=document.querySelector('#main table');if(!table)return;table.classList.add('history-table');const heading=document.createElement('th');heading.textContent=t('operation');table.querySelector('thead tr').append(heading);const rows=[...table.querySelectorAll('tbody tr')];const ops=await api('/api/v1/operations');rows.forEach((row,i)=>{const op=ops[i];if(!op)return;const button=document.createElement('button');button.textContent=t('details');const actions=document.createElement('td');actions.className='history-actions';actions.append(button);row.append(actions);button.onclick=busy(async()=>{const current=await api('/api/v1/operations/'+op.operation_id);showModal(`

${t('details')}

${t(current.status)} · ${esc(current.action)}

${esc(current.error?.message||'')}

${esc(JSON.stringify({operation_id:current.operation_id,request_id:current.request_id,created_at:current.created_at,updated_at:current.updated_at,result:current.result},null,2))}
${current.result?.url?'Preview':''}
${buttons('modal-cancel','cancel')}
`);if(current.result?.url)await blobImage($('#operation-preview'),current.result.url);});});} async function enhanceAutomation(){ - const jobs=await api('/api/v1/jobs');const rows=[...document.querySelectorAll('tbody tr')];rows.forEach((row,i)=>{const job=jobs[i];if(!job)return;const actions=row.querySelector('.row-actions');const run=document.createElement('button');run.textContent=t('runNow');run.disabled=!liveWriteAllowed();run.title=run.disabled?t('readOnlyWarning'):'';actions.append(run);run.onclick=()=>confirmDialog(t('runConfirm'),async()=>{await accepted('/api/v1/jobs/'+job.id+'/run',{});await navigate('history');});if(job.source.type==='template'){const preview=document.createElement('button');preview.textContent=t('jobPreview');actions.append(preview);preview.onclick=busy(async()=>{const id=await accepted('/api/v1/jobs/'+job.id+'/preview',{});const op=await waitOperation(id);showModal(`

${t('jobPreview')}

Preview
${buttons('modal-cancel','cancel')}
`);await blobImage($('#job-preview-image'),op.result.url);});}}); + const jobs=await api('/api/v1/jobs');const rows=[...document.querySelectorAll('tbody tr')];rows.forEach((row,i)=>{const job=jobs[i];if(!job)return;const actions=row.querySelector('.row-actions');const run=document.createElement('button');run.textContent=t('runNow');run.disabled=!liveWriteAllowed();run.title=run.disabled?t('readOnlyWarning'):'';actions.append(run);run.onclick=()=>confirmDialog(t('runConfirm'),async()=>{await accepted('/api/v1/jobs/'+job.id+'/run',{});await navigate('history');});if(['template','auto_collage'].includes(job.source.type)){const preview=document.createElement('button');preview.textContent=t('jobPreview');actions.append(preview);preview.onclick=busy(async()=>{const id=await accepted('/api/v1/jobs/'+job.id+'/preview',{});const op=await waitOperation(id);showModal(`

${t('jobPreview')}

Preview
${buttons('modal-cancel','cancel')}
`);await blobImage($('#job-preview-image'),op.result.url);});}}); } async function jobSourceEditor(job){ const raw=$('#job-source');const advanced=document.createElement('details');advanced.innerHTML=`${t('advanced')}`;raw.parentElement.before(advanced);advanced.append(raw.parentElement); const box=document.createElement('div');advanced.before(box);let source=structuredClone(job.source);let images=[],assets=[],photoSources=[];try{[images,assets,photoSources]=await Promise.all([api('/api/v1/devices/'+state.device+'/images'),api('/api/v1/assets'),api('/api/v1/photo-sources')]);}catch(e){toast(e.message);}if(!raw.isConnected)return; - box.innerHTML=`
`; + box.innerHTML=`
`; $('#job-content-type').value=source.type;const old=$('#job-template')?.parentElement;if(old)old.remove(); function update(){raw.value=JSON.stringify(source,null,2);} function fields(){const type=$('#job-content-type').value;const panel=$('#job-content-fields');if(type==='restore'){source={type};panel.innerHTML='';$('#job-restore').checked=false;$('#job-duration').value=0;update();return;} if(type==='template'){const selected=state.templates.find(tpl=>tpl.id===source.template)||state.templates[0];source={type,template:selected.id,data:source.type==='template'?source.data||{}:{},profile:source.profile||'color'};panel.innerHTML=`
`;$('#job-source-profile').value=source.profile;$('#job-source-profile').onchange=()=>{source.profile=$('#job-source-profile').value;update();};const defaults={...selected.defaults,...source.data};const keys=[...new Set([...Object.keys(defaults).filter(k=>typeof defaults[k]!=='object'||Array.isArray(defaults[k])),...(['calendar','daily-dashboard'].includes(selected.id)?['events']:[])])];$('#job-data-fields').innerHTML=keys.map(k=>`
`).join('');$('#job-template').onchange=()=>{source={type:'template',template:$('#job-template').value,data:{}};fields();};panel.querySelectorAll('[data-job-field]').forEach(el=>el.oninput=()=>{source.data[el.dataset.jobField]=el.dataset.jobField==='events'?el.value.split('\n').filter(x=>x.trim()):el.value;update();});update();} + else if(type==='auto_collage'){source={count:4,use_ai:true,...source,type};panel.innerHTML=`

${state.lang==='zh'?'每次执行重新随机选图；取消 AI 使用自动网格。':'Each run picks new photos; uncheck AI for an automatic grid.'}

`;$('#job-collage-source').value=source.source_id||'';for(const id of ['source','count','title','prompt','ai'])$('#job-collage-'+id).oninput=()=>{source={...source,source_id:$('#job-collage-source').value||null,count:Number($('#job-collage-count').value),title:$('#job-collage-title').value,prompt:$('#job-collage-prompt').value,use_ai:$('#job-collage-ai').checked};update();};update();} else if(type==='image'){const selected=source.image_id?'cloud:'+source.image_id:source.asset_id?'local:'+source.asset_id:'';panel.innerHTML=`
`;$('#job-image').value=selected;$('#job-image').onchange=()=>{const [kind,id]=$('#job-image').value.split(':');source=kind==='cloud'?{type:'image',image_id:Number(id)}:{type:'image',asset_id:id};update();};} else if(type==='photo_source'){panel.innerHTML=`

${t('sourceJobHint')}

`;$('#job-photo-source').value=source.source_id||'';$('#job-photo-source').onchange=()=>{source={type:'photo_source',source_id:$('#job-photo-source').value,fit:'contain',profile:'color'};update();};} else{panel.innerHTML=`
`;$('#job-url').oninput=()=>{source={type:'image_url',url:$('#job-url').value};update();};}} diff --git a/pigway_frame_flow/web/style.css b/pigway_frame_flow/web/style.css index 5aa8643..963537e 100644 --- a/pigway_frame_flow/web/style.css +++ b/pigway_frame_flow/web/style.css @@ -100,3 +100,5 @@ form>.field,form>.check-row,form>.hint,form>.notice{margin:0} .photos-connection .badge.warning{color:var(--bad);background:transparent;border:1px solid var(--bad)} form>.modal-actions{margin-top:8px} @media(max-width:720px){button,input:not([type=checkbox]):not([type=radio]),select{min-height:44px}.image-card button,.row-actions button,.history-actions button{font-size:14px;padding:9px 12px}.check-row{min-height:44px;align-items:center}} + +.auto-collage{margin-bottom:24px}.auto-collage h2{margin-bottom:12px}.auto-collage>p{margin-bottom:20px}.auto-collage #collage-preview:not(:empty){margin-top:20px}.auto-collage>.actions{margin-top:16px}.manual-collage>summary{cursor:pointer;font-weight:600}.manual-collage[open]>summary{margin-bottom:24px} diff --git a/scripts/ui-auto-collage.cjs b/scripts/ui-auto-collage.cjs new file mode 100644 index 0000000..8c85969 --- /dev/null +++ b/scripts/ui-auto-collage.cjs @@ -0,0 +1,14 @@ +/* Isolated Mock-only automatic collage workflow. */ +const assert=require('assert');const fs=require('fs');const{chromium}=require('playwright'); +(async()=>{ + const base=process.env.FRAME_FLOW_URL;assert(base);const token=fs.readFileSync(process.env.FRAME_FLOW_ENV_FILE,'utf8').split('\n').find(x=>x.startsWith('FRAME_FLOW_TOKEN=')).slice(17);const headers={Authorization:'Bearer '+token};assert((await(await fetch(base+'/api/v1/settings',{headers})).json()).mock===true); + const form=new FormData();for(let i=0;i<4;i++)form.append('files',new Blob([Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a1OIAAAAASUVORK5CYII=','base64')],{type:'image/png'}),'fixture-'+i+'.png');assert((await fetch(base+'/api/v1/layout/images',{method:'POST',headers,body:form})).status===201); + const browser=await chromium.launch({executablePath:process.env.CHROME_EXECUTABLE||'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'});try{ + const page=await browser.newPage();const errors=[];page.on('pageerror',e=>errors.push(e.message));await page.goto(base);await page.locator('#username').fill(process.env.FRAME_FLOW_TEST_USERNAME);await page.locator('#password').fill(process.env.FRAME_FLOW_TEST_PASSWORD);await page.locator('#login-form button').click();await page.locator('.hero').waitFor();assert(await page.locator('#nav button').first().getAttribute('data-page')==='dashboard'); + await page.evaluate(()=>navigate('studio'));assert(await page.locator('#studio-plan').isHidden());await page.locator('#collage-ai').uncheck();await page.locator('#collage-count').fill('3');await page.locator('#collage-title').fill('Collage fixture');await page.locator('#auto-collage-form button[type=submit]').click();await page.locator('#collage-image').waitFor({timeout:30000});assert(await page.locator('#collage-display').isVisible()); + const uploads=await(await fetch(base+'/api/v1/devices/living-room/images',{headers})).json();assert(uploads.length===1,'Preview must not upload'); + await page.locator('#collage-job').click();await page.locator('[data-job-edit]').last().click();await page.locator('#job-collage-count').waitFor();assert(await page.locator('#job-collage-count').inputValue()==='3');assert(await page.locator('#job-collage-ai').isChecked()===false);await page.locator('#modal-cancel').click(); + const jobs=await(await fetch(base+'/api/v1/jobs',{headers})).json();const job=jobs.find(j=>j.name==='Collage fixture');assert(job&&job.source.type==='auto_collage'&&job.source.count===3&&!job.enabled); + assert.deepStrictEqual(errors,[]);console.log('Automatic collage: 8 workflow checks passed; no frame writes or console errors'); + }finally{await browser.close();} +})().catch(e=>{console.error(e.message);process.exit(1)}); diff --git a/scripts/ui-integrations.cjs b/scripts/ui-integrations.cjs index 2d93d64..20d5c06 100644 --- a/scripts/ui-integrations.cjs +++ b/scripts/ui-integrations.cjs @@ -12,13 +12,13 @@ const assert=require('assert');const fs=require('fs');const {chromium}=require(' for(const width of [1440,768,390])for(const lang of ['zh','en'])for(const theme of ['light','dark']){ await page.setViewportSize({width,height:900});await page.evaluate(({lang,theme})=>{state.lang=lang;document.documentElement.dataset.theme=theme;shell();},{lang,theme}); for(const view of ['studio','connections']){ - await page.evaluate(view=>navigate(view),view);await page.locator(view==='studio'?'#studio-plan':'#mqtt-config').waitFor(); + await page.evaluate(view=>navigate(view),view);if(view==='studio')await page.locator('.manual-collage summary').click();await page.locator(view==='studio'?'#studio-plan':'#mqtt-config').waitFor(); assert(await page.evaluate(()=>document.documentElement.scrollWidth<=window.innerWidth),view+' overflow '+width);checks++; const gaps=await page.locator('.actions').evaluateAll(rows=>rows.flatMap(row=>{const nodes=[...row.querySelectorAll('button')];return nodes.slice(1).map((node,i)=>{const a=nodes[i].getBoundingClientRect(),b=node.getBoundingClientRect();return Math.abs(a.top-b.top)<4?b.left-a.right:b.top-a.bottom;});})); assert(gaps.every(g=>g>=8),view+' buttons touch '+width);checks++; } } - await page.evaluate(()=>navigate('studio'));await page.locator('#studio-files').setInputFiles({name:'fixture.png',mimeType:'image/png',buffer:Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a1OIAAAAASUVORK5CYII=','base64')}); + await page.evaluate(()=>navigate('studio'));await page.locator('.manual-collage summary').click();await page.locator('#studio-files').setInputFiles({name:'fixture.png',mimeType:'image/png',buffer:Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a1OIAAAAASUVORK5CYII=','base64')}); await page.locator('#studio-assets input:checked').waitFor();await page.locator('#studio-plan').click();await page.locator('#studio-layout img').waitFor();await page.locator('#studio-render').click();await page.locator('#studio-result').waitFor({timeout:30000});assert(await page.locator('#studio-download').isEnabled());checks++; await page.locator('#studio-title').fill('Changed');assert(await page.locator('#studio-display').isDisabled());checks++; assert.deepStrictEqual(errors,[]);checks++; diff --git a/scripts/ui-layout-audit.cjs b/scripts/ui-layout-audit.cjs index b0a8c86..e6801eb 100644 --- a/scripts/ui-layout-audit.cjs +++ b/scripts/ui-layout-audit.cjs @@ -20,7 +20,7 @@ const assert=require('assert');const fs=require('fs');const {chromium}=require(' for(const width of [1440,768,390])for(const lang of ['zh','en'])for(const theme of ['light','dark']){ await page.setViewportSize({width,height:900});await page.evaluate(({lang,theme})=>{state.lang=lang;document.documentElement.dataset.theme=theme;shell();},{lang,theme}); for(const view of ['dashboard','gallery','photo-sources','upload','templates','automation','playlists','history','settings','studio','connections']){await page.evaluate(view=>navigate(view),view);await inspect(`${view}/${width}/${lang}/${theme}`);} - await page.evaluate(()=>navigate('photo-sources'));await page.locator('#photos-login').click();await inspect(`Photos login/${width}/${lang}/${theme}`);await page.locator('#modal-cancel').click(); + await page.evaluate(()=>navigate('studio'));await page.locator('.manual-collage summary').click();await inspect(`Manual collage/${width}/${lang}/${theme}`);await page.evaluate(()=>navigate('photo-sources'));await page.locator('#photos-login').click();await inspect(`Photos login/${width}/${lang}/${theme}`);await page.locator('#modal-cancel').click(); await page.locator('#add-photo-source').click();assert(await page.locator('#source-credential').isHidden());assert(await page.locator('#source-oauth').isHidden());await inspect(`NAS source/${width}/${lang}/${theme}`);await page.locator('#source-kind').selectOption('google-drive');assert(await page.locator('#source-oauth').isVisible());assert(await page.locator('#source-local-quality').isHidden());await inspect(`Drive source/${width}/${lang}/${theme}`);await page.locator('#modal-cancel').click(); await page.evaluate(()=>navigate('automation'));await page.locator('#new-job').click();await inspect(`Job/${width}/${lang}/${theme}`);await page.locator('#modal-cancel').click(); await page.evaluate(()=>navigate('playlists'));await page.locator('#new-playlist').click();await inspect(`Playlist/${width}/${lang}/${theme}`);await page.locator('#modal-cancel').click(); diff --git a/tests/schema-contract.json b/tests/schema-contract.json index 50fa320..fbcf436 100644 --- a/tests/schema-contract.json +++ b/tests/schema-contract.json @@ -373,6 +373,9 @@ { "$ref": "#/components/schemas/PhotoSourceContent" }, + { + "$ref": "#/components/schemas/AutoCollageContent" + }, { "$ref": "#/components/schemas/ProviderContent" }, @@ -384,6 +387,7 @@ "discriminator": { "propertyName": "type", "mapping": { + "auto_collage": "#/components/schemas/AutoCollageContent", "html": "#/components/schemas/FutureContent", "image": "#/components/schemas/ImageContent", "image_url": "#/components/schemas/URLContent", @@ -1337,5 +1341,86 @@ "name" ], "title": "PhotosPerson" + }, + "AutoCollageContent": { + "properties": { + "type": { + "type": "string", + "const": "auto_collage", + "title": "Type" + }, + "source_id": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-z0-9-]{1,60}$" + }, + { + "type": "null" + } + ], + "title": "Source Id" + }, + "count": { + "type": "integer", + "maximum": 12.0, + "minimum": 1.0, + "title": "Count", + "default": 4 + }, + "title": { + "type": "string", + "maxLength": 100, + "title": "Title", + "default": "" + }, + "message": { + "type": "string", + "maxLength": 500, + "title": "Message", + "default": "" + }, + "orientation": { + "type": "string", + "enum": [ + "auto", + "portrait", + "landscape" + ], + "title": "Orientation", + "default": "auto" + }, + "style": { + "type": "string", + "enum": [ + "grid", + "hero", + "poster" + ], + "title": "Style", + "default": "grid" + }, + "use_ai": { + "type": "boolean", + "title": "Use Ai", + "default": true + }, + "prompt": { + "type": "string", + "maxLength": 1500, + "title": "Prompt", + "default": "" + }, + "profile": { + "$ref": "#/components/schemas/Profile", + "default": "color" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ], + "title": "AutoCollageContent" } } diff --git a/tests/test_auto_collage.py b/tests/test_auto_collage.py new file mode 100644 index 0000000..3ac6c18 --- /dev/null +++ b/tests/test_auto_collage.py @@ -0,0 +1,160 @@ +from io import BytesIO + +import pytest +from PIL import Image +from pydantic import ValidationError + +from pigway_frame_flow.models import AutoCollageContent + + +def photos(service, count=4): + for i in range(count): + data = BytesIO() + Image.new("RGB", (320 + i, 200), (i * 50, 60, 100)).save(data, "JPEG") + service.assets.put(data.getvalue(), f"photo-{i}.jpg") + + +def test_auto_preview_selects_exact_distinct_photos_without_frame_write( + client, service +): + photos(service) + result = client.post( + "/api/v1/render", json={"type": "auto_collage", "count": 3, "use_ai": False} + ) + assert result.status_code == 202 + service.process_one() + operation = service.operation(result.json()["operation_id"]) + assert operation["status"] == "COMPLETED" + asset = operation["result"]["asset_id"] + import json + + source = json.loads( + service.db.one("SELECT payload FROM asset_sources WHERE asset_id=?", (asset,))[ + "payload" + ] + ) + regions = source["data"]["layout"]["regions"] + assert len({x["asset_id"] for x in regions}) == 3 + assert service.registry.get("living-room").client.uploads == 0 + + +def test_insufficient_photos_and_unconfigured_ai_fail_before_upload(service): + from pigway_frame_flow.errors import FlowError + + photos(service, 1) + for content in [ + AutoCollageContent(type="auto_collage", count=2, use_ai=False), + AutoCollageContent(type="auto_collage", count=1), + ]: + with pytest.raises(FlowError): + service.content_asset(content) + assert service.registry.get("living-room").client.uploads == 0 + + +@pytest.mark.parametrize("count", [0, 13]) +def test_count_bounds(count): + with pytest.raises(ValidationError): + AutoCollageContent(type="auto_collage", count=count) + + +def test_auto_collage_job_contract_and_preview(client, service): + photos(service, 2) + job = client.post( + "/api/v1/jobs", + json={ + "name": "Collage", + "device_id": "living-room", + "enabled": False, + "schedule": {"type": "daily", "time": "08:00"}, + "source": {"type": "auto_collage", "count": 2, "use_ai": False}, + }, + ) + assert job.status_code == 201 + result = client.post("/api/v1/jobs/" + job.json()["id"] + "/preview") + assert result.status_code == 202 + service.process_one() + assert service.operation(result.json()["operation_id"])["status"] == "COMPLETED" + assert service.registry.get("living-room").client.uploads == 0 + + +def test_source_selection_respects_quality_and_keeps_inputs_out_of_manual_library( + service, monkeypatch +): + from types import SimpleNamespace + + source = SimpleNamespace( + minimum_sharpness=0, + minimum_brightness=50, + maximum_brightness=255, + avoid_duplicates=False, + ) + monkeypatch.setattr(service.photo_sources, "get", lambda _: source) + monkeypatch.setattr( + service.photo_sources, "catalog", lambda _: [{"id": str(i)} for i in range(3)] + ) + + def read(_, item): + data = BytesIO() + Image.new("RGB", (320, 240), (int(item) * 80,) * 3).save(data, "JPEG") + return data.getvalue(), item + ".jpg", None + + monkeypatch.setattr(service.photo_sources, "read", read) + content = AutoCollageContent( + type="auto_collage", source_id="nas", count=2, use_ai=False + ) + for _ in range(2): + asset = service.content_asset(content) + assert asset["source"] == "generated" + assert ( + service.db.one("SELECT COUNT(*) AS n FROM assets WHERE kind='images'")["n"] == 0 + ) + assert ( + service.db.one("SELECT COUNT(*) AS n FROM assets WHERE kind='cache'")["n"] == 4 + ) + + +def test_ai_receives_only_selected_asset_metadata(service, monkeypatch): + from pigway_frame_flow.extensions.layout import LayoutStudio, local_plan + + photos(service, 3) + calls = [] + + def plan(_, request): + calls.append(request) + assert request.use_ai is True + assert len(set(request.asset_ids)) == 3 + return local_plan(request) + + monkeypatch.setattr(LayoutStudio, "plan", plan) + service.content_asset( + AutoCollageContent(type="auto_collage", count=3, title="Family") + ) + assert len(calls) == 1 + assert calls[0].title == "Family" + assert service.registry.get("living-room").client.uploads == 0 + + +def test_scheduled_collage_executes_once_and_tracks_cloud_task_head(app, service): + from datetime import UTC, datetime, timedelta + + from pigway_frame_flow.models import JobInput + + photos(service, 3) + scheduler = app.state.scheduler + job = scheduler.save_job( + JobInput( + name="Scheduled collage", + schedule={"type": "daily", "time": "08:00"}, + source={"type": "auto_collage", "count": 3, "use_ai": False}, + ) + ) + due = (datetime.now(UTC) - timedelta(seconds=2)).isoformat() + service.db.execute("UPDATE jobs SET next_run=? WHERE id=?", (due, job["id"])) + scheduler.tick() + scheduler.tick() + assert len(service.db.all("SELECT * FROM job_runs")) == 1 + service.process_one() + service.sync() + assert service.registry.get("living-room").client.uploads == 1 + row = service.db.one("SELECT job_id FROM managed_cloud_uploads") + assert row["job_id"] == job["id"]