import { imageUrlToBase64 } from '../../../ClientUtils'; import { Doc } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { DocCast, ImageCast } from '../../../fields/Types'; import { Upload } from '../../../server/SharedMediaTypes'; import { gptDescribeImage } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { Networking } from '../../Network'; import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; import { AspectRatioLimits, FireflyDimensionsMap, FireflyImageDimensions, FireflyStylePresets } from './FireflyConstants'; export class DrawingFillHandler { static drawingToImage = async (drawing: Doc, strength: number, user_prompt: string) => { const docData = drawing[DocData]; const tags: string[] = ((docData?.tags as unknown as string[]) ?? []).map(tag => tag.slice(1)) ?? []; const styles = tags.filter(tag => FireflyStylePresets.has(tag)); const styleDocs = Doc.Links(drawing) .map(link => Doc.getOppositeAnchor(link, drawing)) .map(anchor => anchor && DocCast(anchor.embedContainer)); const styleRef = styleDocs.filter(doc => doc !== undefined && doc.type === 'image').lastElement(); let styleUrl: string | undefined; if (styleRef) { const styleImg = await DocumentView.GetDocImage(styleRef); if (styleImg) { const { href } = ImageCast(styleImg).url; const hrefParts = href.split('.'); styleUrl = `${hrefParts.slice(0, -1).join('.')}_o.${hrefParts.lastElement()}`; } } DocumentView.GetDocImage(drawing)?.then(imageField => { if (imageField) { const aspectRatio = (drawing.width as number) / (drawing.height as number); let dims: { width: number; height: number }; if (aspectRatio > AspectRatioLimits[FireflyImageDimensions.Widescreen]) { dims = FireflyDimensionsMap[FireflyImageDimensions.Widescreen]; } else if (aspectRatio > AspectRatioLimits[FireflyImageDimensions.Landscape]) { dims = FireflyDimensionsMap[FireflyImageDimensions.Landscape]; } else if (aspectRatio < AspectRatioLimits[FireflyImageDimensions.Portrait]) { dims = FireflyDimensionsMap[FireflyImageDimensions.Portrait]; } else { dims = FireflyDimensionsMap[FireflyImageDimensions.Square]; } const { href } = ImageCast(imageField).url; const hrefParts = href.split('.'); const structureUrl = `${hrefParts.slice(0, -1).join('.')}_o.${hrefParts.lastElement()}`; imageUrlToBase64(structureUrl) .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) .then(prompt => { Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: `${user_prompt}, ${prompt}`, width: dims.width, height: dims.height, structure: structureUrl, strength: strength, presets: styles, styleRef: styleUrl}) .then((info: Upload.ImageInformation) => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai: 'firefly', ai_firefly_prompt: user_prompt || prompt, _width: 500, data_nativeWidth: info.nativeWidth, data_nativeHeight: info.nativeHeight }), OpenWhere.addRight) ).catch(e => alert("create image failed: " + e.toString())); // prettier-ignore }); } return false; }); }; }