refactor(gui-agent): refactor browser operators architecture and fix deployment issues (#1682)

Author: ZhaoHeh

multimodal/agent-tars/core/examples/browser-gui-agent-rt.ts

@@ -0,0 +1,71 @@
+/*
+ * Quick real-environment runner for BrowserGUIAgent
+ * - Launches a local Chrome (via Puppeteer)
+ * - Navigates to a page
+ * - Uses BrowserGUIAgent.screenshot() and saves a WEBP file
+ */
+import { AgentTARS } from '../src';
+
+async function main() {
+  const localAgent = new AgentTARS({
+    model: {
+      provider: 'volcengine',
+      id: 'ep-20250510145437-5sxhs',
+      apiKey: process.env.ARK_API_KEY,
+      displayName: 'doubao-1.5-thinking-vision-pro',
+    },
+    toolCallEngine: 'structured_outputs',
+  });
+  await localAgent.initialize();
+  const tools = localAgent.getTools();
+  console.log('\n📋 Available Tools:');
+  console.log('─'.repeat(80));
+  tools.forEach((tool, index) => {
+    const num = (index + 1).toString().padStart(2, ' ');
+    const name = tool.name.padEnd(30, ' ');
+    const desc = (tool.description || 'No description').substring(0, 45).replace(/\n/g, ' ');
+    console.log(`${num}. ${name} │ ${desc}`);
+  });
+  console.log('─'.repeat(80));
+  console.log(`Total: ${tools.length} tools\n`);
+
+  // Test tasks to run
+  const tasks = [
+    'Open https://seed-tars.com',
+    'Use gui to go to https://www.producthunt.com/, search the top products for "AI", from the results, identify the top-listed product (the top 3 result). Collect the following information from that product\'s card: 1. Product name 2. Short description 3. Number of upvotes summarize it and report to me.',
+    'Use gui action, go to https://sample-files.com/documents/pdf/, find the 65KB pdf file, preview it, scroll the file from top to bottom.',
+  ];
+
+  // Execute tasks iteratively
+  for (let i = 0; i < tasks.length; i++) {
+    const task = tasks[i];
+    console.log(`\n🚀 Executing Task ${i + 1}:`);
+    console.log(`📝 ${task}`);
+    console.log('─'.repeat(80));
+
+    try {
+      const response = await localAgent.run(task);
+      console.log(`✅ Task ${i + 1} Response:`, response);
+    } catch (error) {
+      console.error(`❌ Task ${i + 1} Failed:`, error);
+    }
+
+    console.log('─'.repeat(80));
+  }
+  console.log('🎉 All tasks completed. Exiting...');
+
+  // Clean up resources and exit
+  console.log('\n🧹 Cleaning up resources...');
+  try {
+    await localAgent.cleanup();
+    console.log('✅ Cleanup completed');
+  } catch (error) {
+    console.error('❌ Cleanup failed:', error);
+  }
+  process.exit(0);
+}
+
+main().catch((err) => {
+  console.error('Runner failed:', err);
+  process.exit(1);
+});

multimodal/agent-tars/core/package.json

@@ -29,6 +29,8 @@
   },
   "devDependencies": {
     "@gui-agent/operator-browser": "workspace:*",
+    "@gui-agent/action-parser": "workspace:*",
+    "@gui-agent/shared": "workspace:*",
     "@agent-infra/mcp-server-browser": "1.1.10",
     "@agent-infra/mcp-server-commands": "1.1.10",
     "@agent-infra/mcp-server-filesystem": "1.1.10",

multimodal/agent-tars/core/src/environments/local/browser/browser-gui-agent.ts

@@ -9,8 +9,12 @@ import { BrowserOperator } from '@gui-agent/operator-browser';
 import { ConsoleLogger, AgentEventStream, Tool, z } from '@tarko/mcp-agent';
 import { ImageCompressor, formatBytes } from '@tarko/shared-media-utils';
 import { ActionInputs, PredictionParsed } from '@agent-tars/interface';
+import { ActionParserHelper } from '@gui-agent/action-parser';
+import { Coordinates, NormalizeCoordinates } from '@gui-agent/shared/types';
+import { normalizeActionCoords } from '@gui-agent/shared/utils';
 import {
   convertToGUIResponse,
+  convertToAgentUIAction,
   createGUIErrorResponse,
   GUIExecuteResult,
 } from '@tarko/shared-utils';
@@ -37,6 +41,22 @@ export interface GUIAgentOptions {
   eventStream?: AgentEventStream.Processor;
 }
 
+const actionParserHelper = new ActionParserHelper();
+
+const defaultNormalizeCoords: NormalizeCoordinates = (rawCoords: Coordinates) => {
+  if (!rawCoords.raw) {
+    return { normalized: rawCoords };
+  }
+  const normalizedCoords = {
+    ...rawCoords,
+    normalized: {
+      x: rawCoords.raw.x / 1000,
+      y: rawCoords.raw.y / 1000,
+    },
+  };
+  return { normalized: normalizedCoords };
+};
+
 /**
  * Browser GUI Agent for visual browser automation
  */
@@ -109,30 +129,39 @@ wait()                                         - Wait 5 seconds and take a scree
       }),
       function: async ({ thought, step, action }) => {
         try {
-          const parsed = this.parseAction(action);
-          parsed.thought = thought;
+          const parsedAction = actionParserHelper.parseActionCallString(action);
+          if (!parsedAction) {
+            return createGUIErrorResponse(action, 'Invalid action format');
+          }
+          const normalizedCoordsAction = normalizeActionCoords(
+            parsedAction,
+            defaultNormalizeCoords,
+          );
 
           this.logger.debug({
             thought,
             step,
             action,
-            parsedAction: JSON.stringify(parsed, null, 2),
+            normalizedCoordsAction: JSON.stringify(normalizedCoordsAction, null, 2),
             screenDimensions: {
               width: this.screenWidth,
               height: this.screenHeight,
             },
           });
 
-          const operatorResult: GUIExecuteResult = await this.browserOperator.execute({
-            parsedPrediction: parsed,
-            screenWidth: this.screenWidth || 1920,
-            screenHeight: this.screenHeight || 1080,
+          const operatorResult = await this.browserOperator.doExecute({
+            actions: [normalizedCoordsAction],
           });
+          this.logger.debug('Browser action completed', operatorResult);
 
           await sleep(500);
 
-          const guiResponse = convertToGUIResponse(action, parsed, operatorResult);
-          return guiResponse;
+          return {
+            success: true,
+            action: action,
+            normalizedAction: convertToAgentUIAction(normalizedCoordsAction),
+            observation: undefined, // Reserved for future implementation
+          };
         } catch (error) {
           this.logger.error(
             `Browser action failed: ${error instanceof Error ? error.message : String(error)}`,
@@ -164,7 +193,7 @@ wait()                                         - Wait 5 seconds and take a scree
     // Record screenshot start time
     const startTime = performance.now();
 
-    const output = await this.browserOperator.screenshot();
+    const output = await this.browserOperator.doScreenshot();
 
     // Calculate screenshot time
     const endTime = performance.now();

multimodal/gui-agent/agent-sdk/.gitignore

@@ -1 +1,2 @@
 snapshots
+snapshot

multimodal/gui-agent/agent-sdk/examples/configs/browser-ve-15vp.config.ts

@@ -6,14 +6,15 @@ import 'dotenv/config';
 import path from 'path';
 
 import { defineConfig } from '@tarko/agent-cli';
-import { browserOperator } from './operators';
+import { browserOperator, remoteBrowserOperator } from './operators';
 import { doubao_1_5_vp } from './models';
-import { systemPromptTemplate1 } from './promptTemps';
+import { systemPromptTemplate2 } from './promptTemps';
 
 export default defineConfig({
-  operator: browserOperator,
+  // operator: browserOperator,
+  operator: remoteBrowserOperator,
   model: doubao_1_5_vp,
-  systemPrompt: systemPromptTemplate1,
+  systemPrompt: systemPromptTemplate2,
   snapshot: {
     enable: true,
     storageDirectory: path.join(__dirname, '../snapshots/browser-ve-15vp'),

multimodal/gui-agent/agent-sdk/examples/configs/operators.ts

@@ -4,35 +4,27 @@
  */
 import { AdbOperator } from '@gui-agent/operator-adb';
 import { NutJSOperator } from '@gui-agent/operator-nutjs';
-import { Operator, ScreenContext } from '@gui-agent/shared/base';
 import {
-  SupportedActionType,
-  ScreenshotOutput,
-  ExecuteParams,
-  ExecuteOutput,
-} from 'gui-agent/shared/src/types';
+  LocalBrowserOperator,
+  RemoteBrowserOperator,
+  SearchEngine,
+} from '@gui-agent/operator-browser';
 
 const computerOperator = new NutJSOperator();
 const androidOperator = new AdbOperator();
+const browserOperator = new LocalBrowserOperator({
+  searchEngine: SearchEngine.GOOGLE,
+  showActionInfo: false,
+  showWaterFlow: false,
+  highlightClickableElements: false,
+});
 
-class MockedBrowserOperator extends Operator {
-  protected initialize(): Promise<void> {
-    throw new Error('Method not implemented.');
-  }
-  protected supportedActions(): Array<SupportedActionType> {
-    throw new Error('Method not implemented.');
-  }
-  protected screenContext(): ScreenContext {
-    throw new Error('Method not implemented.');
-  }
-  protected screenshot(): Promise<ScreenshotOutput> {
-    throw new Error('Method not implemented.');
-  }
-  protected execute(params: ExecuteParams): Promise<ExecuteOutput> {
-    throw new Error('Method not implemented.');
-  }
-}
+const remoteBrowserOperator = new RemoteBrowserOperator({
+  wsEndpoint: 'ws://localhost:9222/devtools/browser/<id>',
+  searchEngine: SearchEngine.GOOGLE,
+  showActionInfo: true,
+  showWaterFlow: true,
+  highlightClickableElements: true,
+});
 
-const browserOperator = new MockedBrowserOperator();
-
-export { computerOperator, androidOperator, browserOperator };
+export { computerOperator, androidOperator, browserOperator, remoteBrowserOperator };

multimodal/gui-agent/agent-sdk/examples/configs/promptTemps.ts

@@ -49,3 +49,48 @@ export const systemPromptTemplate1: SystemPromptTemplate = {
       .join('\n');
   },
 };
+
+export const systemPromptTemplate2: SystemPromptTemplate = {
+  template: SYSTEM_PROMPT_1,
+  actionsToString: (actions) => {
+    return actions
+      .map((action) => {
+        switch (action) {
+          case 'click':
+            return `click(point='<point>x1 y1</point>')`;
+          case 'right_click':
+            return `right_single(point='<point>x1 y1</point>')`;
+          case 'double_click':
+            return `left_double(point='<point>x1 y1</point>')`;
+          case 'navigate':
+            return `navigate(url='xxx') # Navigate to the given url.`;
+          case 'navigate_back':
+            return `navigate_back() # Navigate back to the previous page.`;
+          case 'drag':
+            return `drag(start_point='<point>x1 y1</point>', end_point='<point>x2 y2</point>') # Swipe/Drag to show more information or select elements. The direction of the page movement is opposite to the finger's movement`;
+          case 'hotkey':
+            return `hotkey(key='ctrl c') # Split keys with a space and use lowercase. Also, do not use more than 3 keys in one hotkey action.`;
+          case 'type':
+            return `type(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content. `;
+          case 'scroll':
+            return `scroll(point='<point>x1 y1</point>', direction='down or up or right or left') # Show more information on the \`direction\` side.`;
+          case 'long_press':
+            return `long_press(point='<point>x1 y1</point>')`;
+          case 'press_back':
+            return `press_back() # Press the back button. 如果你想切换应用不需要press_back，直接open_app。`;
+          case 'press_home':
+            return `press_home() # Press the home button. 如果你想切换应用不需要press_home，直接open_app。`;
+          case 'open_app':
+            return `open_app(app_name='xxx') # Open the app with the given name. You can only use the apps in the app_list.`;
+          case 'wait':
+            return `wait() #Sleep for 5s and take a screenshot to check for any changes.`;
+          case 'finished':
+            return `finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.`;
+          default:
+            return null;
+        }
+      })
+      .filter((actionString) => actionString !== null)
+      .join('\n');
+  },
+};

multimodal/gui-agent/operator-browser/package.json

@@ -37,7 +37,7 @@
     "@agent-infra/browser": "0.1.1",
     "@agent-infra/logger": "0.0.2-beta.2",
     "@agent-infra/puppeteer-enhance": "0.1.6",
-    "@ui-tars/sdk": "1.2.3"
+    "@gui-agent/shared": "workspace:*"
   },
   "devDependencies": {
     "@rslib/core": "0.10.0",

multimodal/gui-agent/operator-browser/src/LocalBrowserOperator.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { ConsoleLogger, LogLevel } from '@agent-infra/logger';
+import { LocalBrowser, BrowserFinder, BrowserType } from '@agent-infra/browser';
+
+import { LocalBrowserOperatorOptions, SearchEngine, searchEngineUrlMap } from './types';
+import { BrowserOperator } from './browser-operator';
+
+export class LocalBrowserOperator extends BrowserOperator {
+  private browserPath: string;
+  private browserType: BrowserType;
+  private searchEngine?: SearchEngine;
+
+  constructor(options?: LocalBrowserOperatorOptions) {
+    const {
+      highlightClickableElements = false,
+      showActionInfo = false,
+      showWaterFlow = false,
+      searchEngine,
+    } = options || {};
+
+    // Create logger with LocalBrowserOperator prefix
+    const logger = (options?.logger || new ConsoleLogger(undefined, LogLevel.DEBUG)).spawn(
+      '[Local]',
+    );
+
+    const browserFinder = new BrowserFinder(logger.spawn('[BrowserFinder]'));
+    const { path, type } = browserFinder.findBrowser();
+    logger.debug('ctor: browserData: ', { path, type });
+
+    const browser = new LocalBrowser({ logger: logger.spawn('[Browser]') });
+    const browserOptions = {
+      browser: browser,
+      browserType: type,
+      logger: logger,
+      highlightClickableElements: highlightClickableElements,
+      showActionInfo: showActionInfo,
+      showWaterFlow: showWaterFlow,
+    };
+    super(browserOptions);
+    logger.debug('super ctor done');
+
+    this.browserPath = path;
+    this.browserType = type;
+    this.searchEngine = searchEngine;
+  }
+
+  protected async initialize(): Promise<void> {
+    this.logger.debug('initialize: start');
+    await this.browser.launch({
+      executablePath: this.browserPath,
+      browserType: this.browserType,
+    });
+    this.logger.debug('initialize: browser launched');
+
+    const targetUrl = this.searchEngine ? searchEngineUrlMap[this.searchEngine] : undefined;
+    if (targetUrl) {
+      const openingPage = await this.browser?.getActivePage();
+      await openingPage?.goto(targetUrl, {
+        waitUntil: 'networkidle2',
+      });
+    }
+    this.logger.debug('initialize: search engine opened');
+
+    await super.initialize();
+  }
+}