feat(o-gui-agent): support agent mode and game operator with enhanced error handling (#1656)

Author: ZhaoHeh

multimodal/gui-agent/operator-aio/src/AIOComputer.ts

@@ -62,10 +62,21 @@ export class AIOComputer {
       };
     } catch (error) {
       logger.error('[AIOComputer] Action failed:', error);
-      return {
-        success: false,
-        message: error instanceof Error ? error.message : 'Unknown error',
-      };
+      // Extract key error message from HTTP 422 responses
+      let errorMessage = 'Unknown error';
+      if (error instanceof Error) {
+        const message = error.message;
+        logger.error('[AIOComputer] Action failed message:', message);
+        // Try to extract the key error message from HTTP 422 responses
+        const detailMatch = message.match(/"msg":"([^"]+)"/);
+        if (detailMatch && detailMatch[1]) {
+          errorMessage = detailMatch[1];
+        } else {
+          errorMessage = message;
+        }
+      }
+      // Throw a new error with the extracted key message
+      throw new Error(errorMessage);
     }
   }
 
@@ -287,3 +298,86 @@ export class AIOComputer {
     return this.request(action);
   }
 }
+
+// Comprehensive key name mapping for common key variations
+export const keyNameMap = {
+  // Arrow keys
+  arrowup: 'up',
+  arrowdown: 'down',
+  arrowleft: 'left',
+  arrowright: 'right',
+  // Common key aliases
+  space: ' ',
+  spacebar: ' ',
+  enter: 'enter',
+  return: 'return',
+  tab: 'tab',
+  escape: 'esc',
+  back: 'backspace',
+  backspace: 'backspace',
+  delete: 'del',
+  insert: 'insert',
+  home: 'home',
+  end: 'end',
+  pageup: 'pageup',
+  pagedown: 'pagedown',
+  // Function keys
+  // f1: 'f1', f2: 'f2', f3: 'f3', f4: 'f4', f5: 'f5', f6: 'f6',
+  // f7: 'f7', f8: 'f8', f9: 'f9', f10: 'f10', f11: 'f11', f12: 'f12',
+  // Modifier keys
+  ctrl: 'ctrl',
+  control: 'ctrl',
+  alt: 'alt',
+  shift: 'shift',
+  cmd: 'command',
+  command: 'command',
+  meta: 'command',
+  win: 'win',
+  windows: 'win',
+
+  // Number pad
+  numpad0: 'num0',
+  numpad1: 'num1',
+  numpad2: 'num2',
+  numpad3: 'num3',
+  numpad4: 'num4',
+  numpad5: 'num5',
+  numpad6: 'num6',
+  numpad7: 'num7',
+  numpad8: 'num8',
+  numpad9: 'num9',
+
+  // Special characters and punctuation
+  comma: ',',
+  period: '.',
+  semicolon: ';',
+  quote: "'",
+  doublequote: '"',
+  backquote: '`',
+  tilde: '~',
+  exclamation: '!',
+  at: '@',
+  hash: '#',
+  dollar: '$',
+  percent: '%',
+  caret: '^',
+  ampersand: '&',
+  asterisk: '*',
+  leftparen: '(',
+  rightparen: ')',
+  underscore: '_',
+  plus: '+',
+  minus: '-',
+  equal: '=',
+  leftbracket: '[',
+  rightbracket: ']',
+  backslash: '\\',
+  pipe: '|',
+  leftbrace: '{',
+  rightbrace: '}',
+  colon: ':',
+  less: '<',
+  greater: '>',
+  question: '?',
+  slash: '/',
+};

multimodal/gui-agent/operator-aio/src/AIOGameOperator.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { ConsoleLogger, LogLevel } from '@agent-infra/logger';
+import { AIOHybridOperator } from './AIOHybridOperator';
+import { AIOGameOptions } from './types';
+import { SupportedActionType } from '@gui-agent/shared/types';
+
+const defaultLogger = new ConsoleLogger(undefined, LogLevel.DEBUG);
+
+export class AIOGameOperator extends AIOHybridOperator {
+  readonly name = 'aio-game';
+  readonly description = 'Operator for game environment';
+  private targetUrl: string;
+
+  constructor(options: AIOGameOptions, logger: ConsoleLogger = defaultLogger) {
+    const operatorLogger = logger.spawn('[Game]');
+    super(options, operatorLogger);
+    this.targetUrl = options.targetUrl;
+  }
+
+  protected supportedActions(): Array<SupportedActionType> {
+    return [
+      'call_user',
+      'finished',
+      'wait',
+      'mouse_down',
+      'mouse_up',
+      'mouse_move',
+      'click',
+      'double_click',
+      'right_click',
+      'middle_click',
+      'drag',
+      'type',
+      'hotkey',
+      'press',
+      'release',
+      'scroll',
+    ];
+  }
+
+  protected async initialize(): Promise<void> {
+    await super.initialize();
+    await this.aioBrowser?.handleNavigate({
+      url: this.targetUrl,
+    });
+    this.logger.info(`initialize: goto ${this.targetUrl} successfully`);
+  }
+}

multimodal/gui-agent/operator-aio/src/AIOHybridOperator.ts

@@ -14,38 +14,23 @@ import { Operator, ScreenContext } from '@gui-agent/shared/base';
 import { ConsoleLogger, LogLevel } from '@agent-infra/logger';
 import { Base64ImageParser } from '@agent-infra/media-utils';
 
-import { AIOComputer } from './AIOComputer';
+import { AIOComputer, keyNameMap } from './AIOComputer';
 import { AIOBrowser } from './AIOBrowser';
 import type { AIOHybridOptions } from './types';
 
 const defaultLogger = new ConsoleLogger(undefined, LogLevel.DEBUG);
 
-const arrowKeyMap = {
-  arrowup: 'up',
-  arrowdown: 'down',
-  arrowleft: 'left',
-  arrowright: 'right',
-};
-
 export class AIOHybridOperator extends Operator {
-  private static currentInstance: AIOHybridOperator | null = null;
-  public static async create(options: AIOHybridOptions): Promise<AIOHybridOperator> {
-    defaultLogger.info('[AioHybridOperator.create]:', options.baseURL);
-    const instance = new AIOHybridOperator(options);
-    // await instance.initialize(options);
-    this.currentInstance = instance;
-    return instance;
-  }
-
   private options: AIOHybridOptions;
-  private logger: ConsoleLogger;
-  private aioBrowser: AIOBrowser | null = null;
-  private aioComputer: AIOComputer;
+
+  protected logger: ConsoleLogger;
+  protected aioBrowser: AIOBrowser | null = null;
+  protected aioComputer: AIOComputer;
 
   private screenshotWidth = 1280;
   private screenshotHeight = 1024;
 
-  private constructor(options: AIOHybridOptions, logger: ConsoleLogger = defaultLogger) {
+  constructor(options: AIOHybridOptions, logger: ConsoleLogger = defaultLogger) {
     super();
     this.options = options;
     this.logger = logger.spawn('[AIOHybridOperator]');
@@ -184,30 +169,30 @@ export class AIOHybridOperator extends Operator {
         case 'click':
         case 'left_click':
         case 'left_single': {
-          this.handleClick(actionInputs, 'left');
+          await this.handleClick(actionInputs, 'left');
           const { point } = actionInputs;
           startXPercent = (point as Coordinates)?.normalized?.x;
           startYPercent = (point as Coordinates)?.normalized?.y;
           break;
         }
         case 'left_double':
         case 'double_click': {
-          this.handleClick(actionInputs, 'left', 2);
+          await this.handleClick(actionInputs, 'left', 2);
           const { point } = actionInputs;
           startXPercent = (point as Coordinates)?.normalized?.x;
           startYPercent = (point as Coordinates)?.normalized?.y;
           break;
         }
         case 'right_click':
         case 'right_single': {
-          this.handleClick(actionInputs, 'right');
+          await this.handleClick(actionInputs, 'right');
           const { point } = actionInputs;
           startXPercent = (point as Coordinates)?.normalized?.x;
           startYPercent = (point as Coordinates)?.normalized?.y;
           break;
         }
         case 'middle_click': {
-          this.handleClick(actionInputs, 'middle');
+          await this.handleClick(actionInputs, 'middle');
           const { point } = actionInputs;
           startXPercent = (point as Coordinates)?.normalized?.x;
           startYPercent = (point as Coordinates)?.normalized?.y;
@@ -216,7 +201,7 @@ export class AIOHybridOperator extends Operator {
         case 'left_click_drag':
         case 'drag':
         case 'select': {
-          this.handleDrag(actionInputs);
+          await this.handleDrag(actionInputs);
           break;
         }
         case 'type': {
@@ -230,18 +215,26 @@ export class AIOHybridOperator extends Operator {
         }
         case 'hotkey':
         case 'press': {
-          let keyStr = actionInputs?.key || actionInputs?.hotkey;
+          const keyStr = actionInputs?.key || actionInputs?.hotkey;
           if (typeof keyStr !== 'string') {
             throw new Error('key string is required when press or hotkey');
           }
-          keyStr = keyStr.toLowerCase();
-          const keys = (keyStr as string).split(/[\s+]/).filter((k) => k.length > 0);
-          if (keys.length > 1) {
-            await this.aioComputer.hotkey(keys);
+          const lowerKeyStr: string = keyStr.toLowerCase();
+          const keys = lowerKeyStr.split(/\s+/).filter((k) => k.length > 0);
+
+          // Validate and map each key in the hotkey combination
+          const mappedKeys = keys.map((key) => {
+            return keyNameMap[key as keyof typeof keyNameMap] || key;
+          });
+          if (mappedKeys.length === 0) {
+            throw new Error('key string is required when press or hotkey');
+          }
+
+          this.logger.info('Press/hotkey action mappedKeys:', mappedKeys.join('+'));
+          if (mappedKeys.length > 1) {
+            await this.aioComputer.hotkey(mappedKeys);
           } else {
-            // Check if the key can be mapped using arrowKeyMap
-            const mappedKey = arrowKeyMap[keyStr as keyof typeof arrowKeyMap] || keyStr;
-            await this.aioComputer.press(mappedKey);
+            await this.aioComputer.press(mappedKeys[0]);
           }
           break;
         }
@@ -343,9 +336,9 @@ export class AIOHybridOperator extends Operator {
     }
     const { realX: startX, realY: startY } = await this.calculateRealCoords(startPoint);
     const { realX: endX, realY: endY } = await this.calculateRealCoords(endPoint);
-    if (startX > endX || startY > endY) {
-      throw new Error('start point must be top left of end point');
-    }
+    // if (startX > endX || startY > endY) {
+    //   throw new Error('start point must be top left of end point');
+    // }
     // Move to start position, press mouse, drag to end position, release mouse
     await this.aioComputer.moveTo(startX, startY);
     await this.aioComputer.mouseDown();

multimodal/gui-agent/operator-aio/src/index.ts

@@ -4,4 +4,5 @@
  */
 export { AIOComputer } from './AIOComputer';
 export { AIOHybridOperator } from './AIOHybridOperator';
+export { AIOGameOperator } from './AIOGameOperator';
 export * from './types';

multimodal/gui-agent/operator-aio/src/types.ts

@@ -125,3 +125,7 @@ export interface AIOHybridOptions {
   timeout?: number;
   headers?: Record<string, string>;
 }
+
+export interface AIOGameOptions extends AIOHybridOptions {
+  targetUrl: string;
+}

multimodal/gui-agent/operator-aio/test/aioComputer.test.ts

@@ -435,28 +435,19 @@ describe('AIOComputer', () => {
     it('should handle HTTP errors', async () => {
       mockAioClient.browserActions.mockRejectedValue(new Error('HTTP 404: Not Found'));
 
-      const result = await aioComputer.click(100, 200);
-
-      expect(result.success).toBe(false);
-      expect(result.message).toBe('HTTP 404: Not Found');
+      await expect(aioComputer.click(100, 200)).rejects.toThrow('HTTP 404: Not Found');
     });
 
     it('should handle network errors', async () => {
       mockAioClient.browserActions.mockRejectedValue(new Error('Network timeout'));
 
-      const result = await aioComputer.click(100, 200);
-
-      expect(result.success).toBe(false);
-      expect(result.message).toBe('Network timeout');
+      await expect(aioComputer.click(100, 200)).rejects.toThrow('Network timeout');
     });
 
     it('should handle unknown errors', async () => {
       mockAioClient.browserActions.mockRejectedValue('Unknown error');
 
-      const result = await aioComputer.click(100, 200);
-
-      expect(result.success).toBe(false);
-      expect(result.message).toBe('Unknown error');
+      await expect(aioComputer.click(100, 200)).rejects.toThrow('Unknown error');
     });
   });
 });

multimodal/gui-agent/operator-nutjs/src/NutJSOperator.ts

@@ -174,9 +174,9 @@ export class NutJSOperator extends Operator {
         const { realX: startX, realY: startY } = await this.calculateRealCoords(startPoint);
         const { realX: endX, realY: endY } = await this.calculateRealCoords(endPoint);
         this.logger.info(`drag: start(${startX},${startY}) -> end(${endX},${endY})`);
-        if (startX > endX || startY > endY) {
-          throw new Error('start point must be top left of end point');
-        }
+        // if (startX > endX || startY > endY) {
+        //   throw new Error('start point must be top left of end point');
+        // }
 
         await this.moveStraightTo(startX, startY);
         await sleep(100);