diff --git a/multimodal/omni-tars/core/src/AgentComposer.ts b/multimodal/omni-tars/core/src/AgentComposer.ts index 699027eabe..c47c7e2c71 100644 --- a/multimodal/omni-tars/core/src/AgentComposer.ts +++ b/multimodal/omni-tars/core/src/AgentComposer.ts @@ -32,7 +32,9 @@ export class AgentComposer { */ async initialize(): Promise { for (const plugin of this.plugins) { + const start = Date.now(); await plugin.initialize?.(); + this.logger.info(`initialize agent plugin ${plugin.name} cost: `, Date.now() - start); } } diff --git a/multimodal/omni-tars/mcp-agent/src/McpAgentPlugin.ts b/multimodal/omni-tars/mcp-agent/src/McpAgentPlugin.ts index dcffdc69b3..03c43a61b9 100644 --- a/multimodal/omni-tars/mcp-agent/src/McpAgentPlugin.ts +++ b/multimodal/omni-tars/mcp-agent/src/McpAgentPlugin.ts @@ -30,7 +30,8 @@ export class McpAgentPlugin extends AgentPlugin { } async initialize(): Promise { - await this.mcpManager.init(); + //FIXME:Temporarily remove await to speed up the agent initialization process; the logic of mcpManager.getClient() needs to be added later + this.mcpManager.init(); // Initialize tools this.tools = [ diff --git a/multimodal/omni-tars/omni-agent/src/index.ts b/multimodal/omni-tars/omni-agent/src/index.ts index 26f7637bc5..d97909f151 100644 --- a/multimodal/omni-tars/omni-agent/src/index.ts +++ b/multimodal/omni-tars/omni-agent/src/index.ts @@ -4,7 +4,7 @@ */ import { ComposableAgent } from '@omni-tars/core'; -import { AgentWebUIImplementation } from '@tarko/interface'; +import { AgentWebUIImplementation } from '@tarko/interface'; import { getComposableOption, OmniTarsOption } from './options'; const sandboxBaseUrl = process.env.AIO_SANDBOX_URL ?? '.'; diff --git a/multimodal/omni-tars/omni-agent/src/options.ts b/multimodal/omni-tars/omni-agent/src/options.ts index 891b2b4e39..63e425248a 100644 --- a/multimodal/omni-tars/omni-agent/src/options.ts +++ b/multimodal/omni-tars/omni-agent/src/options.ts @@ -3,66 +3,87 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { CodeAgentExtraOption, codePluginBuilder, CodeToolCallEngineProvider } from "@omni-tars/code-agent"; -import { AgentMode, ComposableAgentOptions, createComposableToolCallEngineFactory } from "@omni-tars/core"; -import { GuiAgentPlugin, GuiToolCallEngineProvider, OperatorManager } from "@omni-tars/gui-agent"; -import { mcpPluginBuilder, MCPTarsExtraOption, McpToolCallEngineProvider } from "@omni-tars/mcp-agent"; -import { AgentAppConfig } from "@tarko/interface"; - +import { + CodeAgentExtraOption, + codePluginBuilder, + CodeToolCallEngineProvider, +} from '@omni-tars/code-agent'; +import { + AgentMode, + ComposableAgentOptions, + createComposableToolCallEngineFactory, +} from '@omni-tars/core'; +import { GuiAgentPlugin, GuiToolCallEngineProvider, OperatorManager } from '@omni-tars/gui-agent'; +import { + mcpPluginBuilder, + MCPTarsExtraOption, + McpToolCallEngineProvider, +} from '@omni-tars/mcp-agent'; +import { AgentAppConfig } from '@tarko/interface'; const mcpToolCallEngine = new McpToolCallEngineProvider(); const omniToolCallEngine = createComposableToolCallEngineFactory({ - engines: [new GuiToolCallEngineProvider('omni'), mcpToolCallEngine, new CodeToolCallEngineProvider()], + engines: [ + new GuiToolCallEngineProvider('omni'), + mcpToolCallEngine, + new CodeToolCallEngineProvider(), + ], defaultEngine: mcpToolCallEngine, }); -const guiToolCallEngine = createComposableToolCallEngineFactory({ engines: [new GuiToolCallEngineProvider('gui')] }); +const guiToolCallEngine = createComposableToolCallEngineFactory({ + engines: [new GuiToolCallEngineProvider('gui')], +}); + +export type OmniTarsOption = AgentAppConfig & + MCPTarsExtraOption & + CodeAgentExtraOption & { + agentMode: AgentMode; + browserMode: 'dom' | 'visual-grounding' | 'hybrid'; + }; +export function getComposableOption(options: OmniTarsOption) { + const { + tavilyApiKey, + googleApiKey, + googleMcpUrl, + sandboxUrl, + ignoreSandboxCheck, + linkReaderAK, + linkReaderMcpUrl, + agentMode = 'omni', + browserMode = 'hybrid', + ...restOptions + } = options; -export type OmniTarsOption = AgentAppConfig & MCPTarsExtraOption & CodeAgentExtraOption & { - agentMode: AgentMode -}; + const baseOptions: Partial = { + ...restOptions, + maxTokens: 32768, + enableStreamingToolCallEvents: true, + }; + const guiPlugin = new GuiAgentPlugin({ + operatorManager: OperatorManager.createHybird(options.sandboxUrl), + }); -export function getComposableOption(options: OmniTarsOption) { - const { + if (agentMode === 'gui') { + baseOptions.toolCallEngine = guiToolCallEngine; + baseOptions.plugins = [guiPlugin]; + } else if (agentMode === 'omni') { + baseOptions.toolCallEngine = omniToolCallEngine; + baseOptions.plugins = [ + mcpPluginBuilder({ tavilyApiKey, googleApiKey, googleMcpUrl, - sandboxUrl, - ignoreSandboxCheck, linkReaderAK, linkReaderMcpUrl, - agentMode = 'omni', - ...restOptions - } = options; - - const baseOptions: Partial = { - ...restOptions, - maxTokens: 32768, - enableStreamingToolCallEvents: true, - }; + }), + codePluginBuilder({ sandboxUrl, ignoreSandboxCheck }), + guiPlugin, + ]; + } - if(agentMode === 'gui') { - baseOptions.toolCallEngine = guiToolCallEngine; - baseOptions.plugins = [ - new GuiAgentPlugin({ operatorManager: OperatorManager.createHybird(options.sandboxUrl) }), - ]; - } else if(agentMode === 'omni') { - baseOptions.toolCallEngine = omniToolCallEngine; - baseOptions.plugins = [ - mcpPluginBuilder({ - tavilyApiKey, - googleApiKey, - googleMcpUrl, - linkReaderAK, - linkReaderMcpUrl, - }), - codePluginBuilder({ sandboxUrl, ignoreSandboxCheck }), - new GuiAgentPlugin({ operatorManager: OperatorManager.createHybird(options.sandboxUrl) }), - ]; - } - - return baseOptions as ComposableAgentOptions; -} \ No newline at end of file + return baseOptions as ComposableAgentOptions; +} diff --git a/multimodal/tarko/agent-server-next/examples/bootstrap.ts b/multimodal/tarko/agent-server-next/examples/bootstrap.ts index aca1916591..6025b3b6c4 100644 --- a/multimodal/tarko/agent-server-next/examples/bootstrap.ts +++ b/multimodal/tarko/agent-server-next/examples/bootstrap.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ import { getContext } from 'hono/context-storage'; -import { AuthHook, CorsHook, AgentServer, ContextStorageHook } from '../src/index'; +import { AuthHook, CorsHook, AgentServer, ContextStorageHook } from '../src/index'; import { resolve } from 'path'; import { ContextVariables } from '../src/types'; @@ -42,7 +42,22 @@ const server = new AgentServer({ type: 'string', title: 'Agent Mode', enum: ['omni', 'gui'], + enumLabels: ['Omni', 'GUI'], default: 'omni', + placement: 'chat-bottom', + }, + + browserMode: { + type: 'string', + title: 'Browser Control', + enum: ['hybrid'], + enumLabels: ['混合模式'], + default: 'hybrid', + placement: 'chat-bottom', + visible: { + dependsOn: 'agentMode', + when: 'gui', + }, }, }, }, @@ -60,16 +75,16 @@ const server = new AgentServer({ }, models: [ { - id: "ep-20250909173748-wcfb2", - provider: "volcengine", - displayName: "T6-SFT", + id: 'ep-20250909173748-wcfb2', + provider: 'volcengine', + displayName: 'T6-SFT', baseURL: process.env.OMNI_TARS_BASE_URL, apiKey: process.env.OMNI_TARS_API_KEY, }, { - id: "ep-20250905175225-hlrvd", - provider: "volcengine", - displayName: "T5-RL", + id: 'ep-20250905175225-hlrvd', + provider: 'volcengine', + displayName: 'T5-RL', baseURL: process.env.OMNI_TARS_BASE_URL, apiKey: process.env.OMNI_TARS_API_KEY, }, @@ -101,15 +116,15 @@ const logger = { name: 'custom', info: (...splat) => { const requestId = getContext<{ Variables: ContextVariables }>().var.requestId; - console.log(`[CUSTOM LOGGER] ${requestId}`, ...splat) + console.log(`[CUSTOM LOGGER] ${requestId}`, ...splat); }, warn: console.warn, error: console.error, debug: console.debug, - setLevel: () => { } -} + setLevel: () => {}, +}; -server.setLogger(logger) +server.setLogger(logger); server.registerHook(AuthHook); server.registerHook(CorsHook); server.registerHook(ContextStorageHook); diff --git a/multimodal/tarko/agent-server-next/src/middlewares/session-restore.ts b/multimodal/tarko/agent-server-next/src/middlewares/session-restore.ts index 3ea0cccefa..b2d80f5a25 100644 --- a/multimodal/tarko/agent-server-next/src/middlewares/session-restore.ts +++ b/multimodal/tarko/agent-server-next/src/middlewares/session-restore.ts @@ -32,7 +32,7 @@ export async function sessionRestoreMiddleware( // If not exist, restored the AgentSession instance based on the database data. if (!session) { - logger.info('session instance not exist, prepare to restore') + logger.info('session instance not exist, prepare to restore'); const start = Date.now(); @@ -47,6 +47,8 @@ export async function sessionRestoreMiddleware( restored.storageUnsubscribe && (server.storageUnsubscribes[sessionId] = restored.storageUnsubscribe); } + } else { + logger.info('get session from sessionPool, sessionId:', session.id); } if (!session) { diff --git a/multimodal/tarko/agent-server-next/src/services/session/AgentSession.ts b/multimodal/tarko/agent-server-next/src/services/session/AgentSession.ts index 66bae9f4db..4a5a5fd21b 100644 --- a/multimodal/tarko/agent-server-next/src/services/session/AgentSession.ts +++ b/multimodal/tarko/agent-server-next/src/services/session/AgentSession.ts @@ -127,7 +127,7 @@ export class AgentSession { throw new Error('Cannot found available resolved agent'); } - // Get stored events for this session before creating the agent + // Get stored events for this session before creating the agent const storedEvents = this.server.daoFactory ? await this.server.daoFactory.getSessionEvents(this.id) : []; @@ -187,15 +187,26 @@ export class AgentSession { // Log AGIO initialization console.debug('AGIO collector initialized', { provider: agentOptions.agio.provider }); } - - this.logger.info('create new agent with config: ', JSON.stringify({ - agent: agentOptions.agent, - share: agentOptions.share, - workspace: agentOptions.workspace, - thinking: agentOptions.thinking, - name: agentOptions.name, - runtimeSettings: transformedOptions - }, null, 2)); + + this.logger.info( + 'create new agent with config: ', + JSON.stringify( + { + agent: agentOptions.agent, + share: agentOptions.share, + workspace: agentOptions.workspace, + thinking: agentOptions.thinking, + name: agentOptions.name, + model: { + id: agentOptions.model?.id, + provider: agentOptions.model?.provider, + }, + runtimeSettings: transformedOptions, + }, + null, + 2, + ), + ); return wrappedAgent; } @@ -552,6 +563,4 @@ export class AgentSession { } } - - export default AgentSession;