From e2b24a5b05519418d6f3d48249878dd4f3eaa1db Mon Sep 17 00:00:00 2001 From: huchenlei Date: Fri, 21 Feb 2025 09:59:14 -0500 Subject: [PATCH 1/4] Move graphToPrompt to executionUtil --- src/scripts/app.ts | 174 ++--------------------------------- src/utils/executionUtil.ts | 180 +++++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 169 deletions(-) create mode 100644 src/utils/executionUtil.ts diff --git a/src/scripts/app.ts b/src/scripts/app.ts index b8c8a81f0..25220aa4a 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -38,6 +38,7 @@ import { } from '@/types/comfyWorkflow' import { ExtensionManager } from '@/types/extensionTypes' import { ColorAdjustOptions, adjustColor } from '@/utils/colorUtil' +import { graphToPrompt } from '@/utils/executionUtil' import { isImageNode } from '@/utils/litegraphUtil' import { deserialiseAndCreate } from '@/utils/vintageClipboard' @@ -1226,176 +1227,11 @@ export class ComfyApp { return graph.serialize({ sortNodes }) } - /** - * Converts the current graph workflow for sending to the API. - * Note: Node widgets are updated before serialization to prepare queueing. - * @returns The workflow and node links - */ async graphToPrompt(graph = this.graph, clean = true) { - for (const outerNode of graph.computeExecutionOrder(false)) { - if (outerNode.widgets) { - for (const widget of outerNode.widgets) { - // Allow widgets to run callbacks before a prompt has been queued - // e.g. random seed before every gen - widget.beforeQueued?.() - } - } - - const innerNodes = outerNode.getInnerNodes - ? outerNode.getInnerNodes() - : [outerNode] - for (const node of innerNodes) { - if (node.isVirtualNode) { - // Don't serialize frontend only nodes but let them make changes - if (node.applyToGraph) { - node.applyToGraph() - } - } - } - } - - const workflow = this.serializeGraph(graph) - - // Remove localized_name from the workflow - for (const node of workflow.nodes) { - for (const slot of node.inputs) { - delete slot.localized_name - } - for (const slot of node.outputs) { - delete slot.localized_name - } - } - - const output = {} - // Process nodes in order of execution - for (const outerNode of graph.computeExecutionOrder(false)) { - const skipNode = - outerNode.mode === LGraphEventMode.NEVER || - outerNode.mode === LGraphEventMode.BYPASS - const innerNodes = - !skipNode && outerNode.getInnerNodes - ? outerNode.getInnerNodes() - : [outerNode] - for (const node of innerNodes) { - if (node.isVirtualNode) { - continue - } - - if ( - node.mode === LGraphEventMode.NEVER || - node.mode === LGraphEventMode.BYPASS - ) { - // Don't serialize muted nodes - continue - } - - const inputs = {} - const widgets = node.widgets - - // Store all widget values - if (widgets) { - for (let i = 0; i < widgets.length; i++) { - const widget = widgets[i] - if (!widget.options || widget.options.serialize !== false) { - inputs[widget.name] = widget.serializeValue - ? await widget.serializeValue(node, i) - : widget.value - } - } - } - - // Store all node links - for (let i = 0; i < node.inputs.length; i++) { - let parent = node.getInputNode(i) - if (parent) { - let link = node.getInputLink(i) - while ( - parent.mode === LGraphEventMode.BYPASS || - parent.isVirtualNode - ) { - let found = false - if (parent.isVirtualNode) { - link = parent.getInputLink(link.origin_slot) - if (link) { - parent = parent.getInputNode(link.target_slot) - if (parent) { - found = true - } - } - } else if (link && parent.mode === LGraphEventMode.BYPASS) { - let all_inputs = [link.origin_slot] - if (parent.inputs) { - // @ts-expect-error convert list of strings to list of numbers - all_inputs = all_inputs.concat(Object.keys(parent.inputs)) - for (let parent_input in all_inputs) { - // @ts-expect-error assign string to number - parent_input = all_inputs[parent_input] - if ( - parent.inputs[parent_input]?.type === node.inputs[i].type - ) { - // @ts-expect-error convert string to number - link = parent.getInputLink(parent_input) - if (link) { - // @ts-expect-error convert string to number - parent = parent.getInputNode(parent_input) - } - found = true - break - } - } - } - } - - if (!found) { - break - } - } - - if (link) { - if (parent?.updateLink) { - link = parent.updateLink(link) - } - if (link) { - inputs[node.inputs[i].name] = [ - String(link.origin_id), - // @ts-expect-error link.origin_slot is already number. - parseInt(link.origin_slot) - ] - } - } - } - } - - const node_data = { - inputs, - class_type: node.comfyClass - } - - // Ignored by the backend. - node_data['_meta'] = { - title: node.title - } - - output[String(node.id)] = node_data - } - } - - // Remove inputs connected to removed nodes - if (clean) { - for (const o in output) { - for (const i in output[o].inputs) { - if ( - Array.isArray(output[o].inputs[i]) && - output[o].inputs[i].length === 2 && - !output[output[o].inputs[i][0]] - ) { - delete output[o].inputs[i] - } - } - } - } - - return { workflow, output } + return graphToPrompt(graph, { + clean, + sortNodes: useSettingStore().get('Comfy.Workflow.SortNodeIdOnSave') + }) } #formatPromptError(error) { diff --git a/src/utils/executionUtil.ts b/src/utils/executionUtil.ts new file mode 100644 index 000000000..4ad96aebd --- /dev/null +++ b/src/utils/executionUtil.ts @@ -0,0 +1,180 @@ +// @ts-strict-ignore +import type { LGraph } from '@comfyorg/litegraph' +import { LGraphEventMode } from '@comfyorg/litegraph' + +/** + * Converts the current graph workflow for sending to the API. + * Note: Node widgets are updated before serialization to prepare queueing. + * @returns The workflow and node links + */ +export const graphToPrompt = async ( + graph: LGraph, + options: { clean?: boolean; sortNodes?: boolean } = {} +) => { + const { clean = true, sortNodes = false } = options + + for (const outerNode of graph.computeExecutionOrder(false)) { + if (outerNode.widgets) { + for (const widget of outerNode.widgets) { + // Allow widgets to run callbacks before a prompt has been queued + // e.g. random seed before every gen + widget.beforeQueued?.() + } + } + + const innerNodes = outerNode.getInnerNodes + ? outerNode.getInnerNodes() + : [outerNode] + for (const node of innerNodes) { + if (node.isVirtualNode) { + // Don't serialize frontend only nodes but let them make changes + if (node.applyToGraph) { + node.applyToGraph() + } + } + } + } + + const workflow = graph.serialize({ sortNodes }) + + // Remove localized_name from the workflow + for (const node of workflow.nodes) { + for (const slot of node.inputs) { + delete slot.localized_name + } + for (const slot of node.outputs) { + delete slot.localized_name + } + } + + const output = {} + // Process nodes in order of execution + for (const outerNode of graph.computeExecutionOrder(false)) { + const skipNode = + outerNode.mode === LGraphEventMode.NEVER || + outerNode.mode === LGraphEventMode.BYPASS + const innerNodes = + !skipNode && outerNode.getInnerNodes + ? outerNode.getInnerNodes() + : [outerNode] + for (const node of innerNodes) { + if (node.isVirtualNode) { + continue + } + + if ( + node.mode === LGraphEventMode.NEVER || + node.mode === LGraphEventMode.BYPASS + ) { + // Don't serialize muted nodes + continue + } + + const inputs = {} + const widgets = node.widgets + + // Store all widget values + if (widgets) { + for (let i = 0; i < widgets.length; i++) { + const widget = widgets[i] + if (!widget.options || widget.options.serialize !== false) { + inputs[widget.name] = widget.serializeValue + ? await widget.serializeValue(node, i) + : widget.value + } + } + } + + // Store all node links + for (let i = 0; i < node.inputs.length; i++) { + let parent = node.getInputNode(i) + if (parent) { + let link = node.getInputLink(i) + while ( + parent.mode === LGraphEventMode.BYPASS || + parent.isVirtualNode + ) { + let found = false + if (parent.isVirtualNode) { + link = parent.getInputLink(link.origin_slot) + if (link) { + parent = parent.getInputNode(link.target_slot) + if (parent) { + found = true + } + } + } else if (link && parent.mode === LGraphEventMode.BYPASS) { + let all_inputs = [link.origin_slot] + if (parent.inputs) { + // @ts-expect-error convert list of strings to list of numbers + all_inputs = all_inputs.concat(Object.keys(parent.inputs)) + for (let parent_input in all_inputs) { + // @ts-expect-error assign string to number + parent_input = all_inputs[parent_input] + if ( + parent.inputs[parent_input]?.type === node.inputs[i].type + ) { + // @ts-expect-error convert string to number + link = parent.getInputLink(parent_input) + if (link) { + // @ts-expect-error convert string to number + parent = parent.getInputNode(parent_input) + } + found = true + break + } + } + } + } + + if (!found) { + break + } + } + + if (link) { + if (parent?.updateLink) { + link = parent.updateLink(link) + } + if (link) { + inputs[node.inputs[i].name] = [ + String(link.origin_id), + // @ts-expect-error link.origin_slot is already number. + parseInt(link.origin_slot) + ] + } + } + } + } + + const node_data = { + inputs, + class_type: node.comfyClass + } + + // Ignored by the backend. + node_data['_meta'] = { + title: node.title + } + + output[String(node.id)] = node_data + } + } + + // Remove inputs connected to removed nodes + if (clean) { + for (const o in output) { + for (const i in output[o].inputs) { + if ( + Array.isArray(output[o].inputs[i]) && + output[o].inputs[i].length === 2 && + !output[output[o].inputs[i][0]] + ) { + delete output[o].inputs[i] + } + } + } + } + + return { workflow, output } +} From 4916513d2d09ccdcfe5cf63b96f3be1deebf5d1f Mon Sep 17 00:00:00 2001 From: huchenlei Date: Fri, 21 Feb 2025 10:18:07 -0500 Subject: [PATCH 2/4] Zod schema --- src/scripts/app.ts | 1 - src/types/comfyWorkflow.ts | 24 +++++++++++++++++++++++- src/utils/executionUtil.ts | 7 +++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 25220aa4a..c99823fcd 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -1280,7 +1280,6 @@ export class ComfyApp { const p = await this.graphToPrompt() try { - // @ts-expect-error Discrepancies between zod and litegraph - in progress const res = await api.queuePrompt(number, p) this.lastNodeErrors = res.node_errors if (this.lastNodeErrors.length > 0) { diff --git a/src/types/comfyWorkflow.ts b/src/types/comfyWorkflow.ts index 7598438a1..2cf78ddbe 100644 --- a/src/types/comfyWorkflow.ts +++ b/src/types/comfyWorkflow.ts @@ -5,6 +5,7 @@ import { fromZodError } from 'zod-validation-error' // innerNode.id = `${this.node.id}:${i}` // Remove it after GroupNode is redesigned. export const zNodeId = z.union([z.number().int(), z.string()]) +export const zNodeInputName = z.string() export type NodeId = z.infer export const zSlotIndex = z.union([ z.number().int(), @@ -96,7 +97,7 @@ const zNodeOutput = z const zNodeInput = z .object({ - name: z.string(), + name: zNodeInputName, type: zDataType, link: z.number().nullable().optional(), slot_index: zSlotIndex.optional() @@ -251,3 +252,24 @@ export async function validateComfyWorkflow( onError(`Invalid workflow against zod schema:\n${error}`) return null } + +/** + * API format workflow for direct API usage. + */ +const zNodeInputValue = z.union([ + // For widget values (can be any type) + z.any(), + // For node links [nodeId, slotIndex] + z.tuple([zNodeId, zSlotIndex]) +]) + +const zNodeData = z.object({ + inputs: z.record(zNodeInputName, zNodeInputValue), + class_type: z.string(), + _meta: z.object({ + title: z.string() + }) +}) + +export const zComfyPrompt = z.record(zNodeId, zNodeData) +export type ComfyPrompt = z.infer diff --git a/src/utils/executionUtil.ts b/src/utils/executionUtil.ts index 4ad96aebd..ecee7e02f 100644 --- a/src/utils/executionUtil.ts +++ b/src/utils/executionUtil.ts @@ -2,6 +2,8 @@ import type { LGraph } from '@comfyorg/litegraph' import { LGraphEventMode } from '@comfyorg/litegraph' +import type { ComfyPrompt, ComfyWorkflowJSON } from '@/types/comfyWorkflow' + /** * Converts the current graph workflow for sending to the API. * Note: Node widgets are updated before serialization to prepare queueing. @@ -10,7 +12,7 @@ import { LGraphEventMode } from '@comfyorg/litegraph' export const graphToPrompt = async ( graph: LGraph, options: { clean?: boolean; sortNodes?: boolean } = {} -) => { +): Promise<{ workflow: ComfyWorkflowJSON; output: ComfyPrompt }> => { const { clean = true, sortNodes = false } = options for (const outerNode of graph.computeExecutionOrder(false)) { @@ -35,7 +37,8 @@ export const graphToPrompt = async ( } } - const workflow = graph.serialize({ sortNodes }) + // @ts-expect-error Convert ISerializedGraph to ComfyWorkflowJSON + const workflow = graph.serialize({ sortNodes }) as ComfyWorkflowJSON // Remove localized_name from the workflow for (const node of workflow.nodes) { From 2a552031e3e087eda69f7173510bed0fff941a59 Mon Sep 17 00:00:00 2001 From: huchenlei Date: Fri, 21 Feb 2025 10:22:02 -0500 Subject: [PATCH 3/4] nit --- src/types/comfyWorkflow.ts | 4 ++-- src/utils/executionUtil.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/types/comfyWorkflow.ts b/src/types/comfyWorkflow.ts index 2cf78ddbe..492ae6891 100644 --- a/src/types/comfyWorkflow.ts +++ b/src/types/comfyWorkflow.ts @@ -271,5 +271,5 @@ const zNodeData = z.object({ }) }) -export const zComfyPrompt = z.record(zNodeId, zNodeData) -export type ComfyPrompt = z.infer +export const zComfyApiWorkflow = z.record(zNodeId, zNodeData) +export type ComfyApiWorkflow = z.infer diff --git a/src/utils/executionUtil.ts b/src/utils/executionUtil.ts index ecee7e02f..d4f2495e9 100644 --- a/src/utils/executionUtil.ts +++ b/src/utils/executionUtil.ts @@ -2,7 +2,7 @@ import type { LGraph } from '@comfyorg/litegraph' import { LGraphEventMode } from '@comfyorg/litegraph' -import type { ComfyPrompt, ComfyWorkflowJSON } from '@/types/comfyWorkflow' +import type { ComfyApiWorkflow, ComfyWorkflowJSON } from '@/types/comfyWorkflow' /** * Converts the current graph workflow for sending to the API. @@ -12,7 +12,7 @@ import type { ComfyPrompt, ComfyWorkflowJSON } from '@/types/comfyWorkflow' export const graphToPrompt = async ( graph: LGraph, options: { clean?: boolean; sortNodes?: boolean } = {} -): Promise<{ workflow: ComfyWorkflowJSON; output: ComfyPrompt }> => { +): Promise<{ workflow: ComfyWorkflowJSON; output: ComfyApiWorkflow }> => { const { clean = true, sortNodes = false } = options for (const outerNode of graph.computeExecutionOrder(false)) { From 4db0d6c3510d7358ee9e72f4c07b32a18d7228f6 Mon Sep 17 00:00:00 2001 From: huchenlei Date: Fri, 21 Feb 2025 10:31:43 -0500 Subject: [PATCH 4/4] Enable ts-strict --- src/utils/executionUtil.ts | 39 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/utils/executionUtil.ts b/src/utils/executionUtil.ts index d4f2495e9..4a990bff6 100644 --- a/src/utils/executionUtil.ts +++ b/src/utils/executionUtil.ts @@ -1,4 +1,3 @@ -// @ts-strict-ignore import type { LGraph } from '@comfyorg/litegraph' import { LGraphEventMode } from '@comfyorg/litegraph' @@ -37,20 +36,19 @@ export const graphToPrompt = async ( } } - // @ts-expect-error Convert ISerializedGraph to ComfyWorkflowJSON - const workflow = graph.serialize({ sortNodes }) as ComfyWorkflowJSON + const workflow = graph.serialize({ sortNodes }) // Remove localized_name from the workflow for (const node of workflow.nodes) { - for (const slot of node.inputs) { + for (const slot of node.inputs ?? []) { delete slot.localized_name } - for (const slot of node.outputs) { + for (const slot of node.outputs ?? []) { delete slot.localized_name } } - const output = {} + const output: ComfyApiWorkflow = {} // Process nodes in order of execution for (const outerNode of graph.computeExecutionOrder(false)) { const skipNode = @@ -73,14 +71,17 @@ export const graphToPrompt = async ( continue } - const inputs = {} + const inputs: ComfyApiWorkflow[string]['inputs'] = {} const widgets = node.widgets // Store all widget values if (widgets) { for (let i = 0; i < widgets.length; i++) { const widget = widgets[i] - if (!widget.options || widget.options.serialize !== false) { + if ( + widget.name && + (!widget.options || widget.options.serialize !== false) + ) { inputs[widget.name] = widget.serializeValue ? await widget.serializeValue(node, i) : widget.value @@ -99,7 +100,7 @@ export const graphToPrompt = async ( ) { let found = false if (parent.isVirtualNode) { - link = parent.getInputLink(link.origin_slot) + link = link ? parent.getInputLink(link.origin_slot) : null if (link) { parent = parent.getInputNode(link.target_slot) if (parent) { @@ -150,17 +151,16 @@ export const graphToPrompt = async ( } } - const node_data = { + output[String(node.id)] = { inputs, - class_type: node.comfyClass - } - - // Ignored by the backend. - node_data['_meta'] = { - title: node.title + // TODO(huchenlei): Filter out all nodes that cannot be mapped to a + // comfyClass. + class_type: node.comfyClass!, + // Ignored by the backend. + _meta: { + title: node.title + } } - - output[String(node.id)] = node_data } } @@ -179,5 +179,6 @@ export const graphToPrompt = async ( } } - return { workflow, output } + // @ts-expect-error Convert ISerializedGraph to ComfyWorkflowJSON + return { workflow: workflow as ComfyWorkflowJSON, output } }