Support for VisualWebArena evaluation in OpenHands

.gitignore

@@ -228,3 +228,13 @@ runtime_*.tar
 containers/runtime/Dockerfile
 containers/runtime/project.tar.gz
 containers/runtime/code
+evaluation/gaia/temp.txt
+evaluation/gaia/temp2.txt
+evaluation/gaia/temp.png
+evaluation/gaia/b64_pil_image.py
+.gitignore
+evaluation/visualwebarena/scripts/run_evaluation.sh
+openhands/agenthub/browsing_agent/browsing_agent2.py
+openhands/agenthub/browsing_agent/som_example1_vwa.png
+openhands/agenthub/browsing_agent/som_example2_vwa.png
+openhands/agenthub/browsing_agent/som_example3_vwa.png

evaluation/visualwebarena/get_success_rate.py

@@ -0,0 +1,35 @@
+import argparse
+import json
+
+import browsergym.visualwebarena  # noqa F401 register visualwebarena tasks as gym environments
+import gymnasium as gym
+
+parser = argparse.ArgumentParser(description='Calculate average reward.')
+parser.add_argument('output_path', type=str, help='path to output.jsonl')
+
+args = parser.parse_args()
+
+if __name__ == '__main__':
+    env_ids = [
+        id
+        for id in gym.envs.registry.keys()
+        if id.startswith('browsergym/visualwebarena')
+    ]
+    total_num = len(env_ids)
+    print('Total number of tasks: ', total_num)
+    total_reward = 0
+    total_cost = 0
+    actual_num = 0
+    with open(args.output_path, 'r') as f:
+        for line in f:
+            data = json.loads(line)
+            actual_num += 1
+            total_cost += data['metrics']['accumulated_cost']
+            total_reward += data['test_result']['reward']
+    avg_reward = total_reward / total_num
+    print('Total reward: ', total_reward)
+    print('Success Rate: ', avg_reward)
+
+    avg_cost = total_cost / actual_num
+    print('Avg Cost: ', avg_cost)
+    print('Actual number of tasks finished: ', actual_num)

evaluation/visualwebarena/run_infer.py

@@ -0,0 +1,237 @@
+import asyncio
+import json
+import os
+from typing import Any
+
+import browsergym.visualwebarena  # noqa F401 register webarena tasks as gym environments
+import gymnasium as gym
+import pandas as pd
+
+from evaluation.utils.shared import (
+    EvalMetadata,
+    EvalOutput,
+    make_metadata,
+    prepare_dataset,
+    reset_logger_for_multiprocessing,
+    run_evaluation,
+)
+from openhands.controller.state.state import State
+from openhands.core.config import (
+    AppConfig,
+    SandboxConfig,
+    get_llm_config_arg,
+    parse_arguments,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.core.main import create_runtime, run_controller
+from openhands.events.action import (
+    BrowseInteractiveAction,
+    CmdRunAction,
+    MessageAction,
+)
+from openhands.events.observation import CmdOutputObservation
+from openhands.runtime.browser.browser_env import (
+    BROWSER_EVAL_GET_GOAL_ACTION,
+    BROWSER_EVAL_GET_REWARDS_ACTION,
+)
+from openhands.runtime.runtime import Runtime
+
+SUPPORTED_AGENT_CLS = {'BrowsingAgent'}
+
+
+def get_config(
+    metadata: EvalMetadata,
+    env_id: str,
+) -> AppConfig:
+    base_url = os.environ.get('VISUALWEBARENA_BASE_URL', None)
+    openai_api_key = os.environ.get('OPENAI_API_KEY', None)
+    assert base_url is not None, 'VISUALWEBARENA_BASE_URL must be set'
+    assert openai_api_key is not None, 'OPENAI_API_KEY must be set'
+
+    config = AppConfig(
+        default_agent=metadata.agent_class,
+        run_as_openhands=False,
+        runtime='eventstream',
+        max_iterations=metadata.max_iterations,
+        sandbox=SandboxConfig(
+            base_container_image='python:3.12-bookworm',
+            enable_auto_lint=True,
+            use_host_network=False,
+            browsergym_eval_env=env_id,
+            # TODO: how to initialize these urls?
+            runtime_startup_env_vars={
+                'BASE_URL': base_url,
+                'OPENAI_API_KEY': openai_api_key,
+                'VWA_CLASSIFIEDS': f'{base_url}:9980',
+                'VWA_CLASSIFIEDS_RESET_TOKEN': '4b61655535e7ed388f0d40a93600254c',
+                'VWA_SHOPPING': f'{base_url}:7770/',
+                'VWA_SHOPPING_ADMIN': f'{base_url}:7780/admin',
+                'VWA_REDDIT': f'{base_url}:9999',
+                'VWA_GITLAB': f'{base_url}:8023',
+                'VWA_WIKIPEDIA': f'{base_url}:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing',
+                'VWA_HOMEPAGE': f'{base_url}:4399',
+            },
+        ),
+        # do not mount workspace
+        workspace_base=None,
+        workspace_mount_path=None,
+    )
+    config.set_llm_config(metadata.llm_config)
+    return config
+
+
+def initialize_runtime(
+    runtime: Runtime,
+) -> tuple[str, list]:
+    """Initialize the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    # Set instance id
+    action = CmdRunAction(command='mkdir -p /workspace')
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    assert obs.exit_code == 0
+
+    action = BrowseInteractiveAction(browser_actions=BROWSER_EVAL_GET_GOAL_ACTION)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+    goal = obs.content
+    goal_image_urls = []
+    if hasattr(obs, 'goal_image_urls'):
+        goal_image_urls = obs.goal_image_urls
+    logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
+    return goal, goal_image_urls
+
+
+def complete_runtime(
+    runtime: Runtime,
+) -> dict[str, Any]:
+    """Complete the runtime for the agent.
+
+    This function is called before the runtime is used to run the agent.
+    If you need to do something in the sandbox to get the correctness metric after
+    the agent has run, modify this function.
+    """
+    logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
+    obs: CmdOutputObservation
+
+    action = BrowseInteractiveAction(browser_actions=BROWSER_EVAL_GET_REWARDS_ACTION)
+    logger.info(action, extra={'msg_type': 'ACTION'})
+    obs = runtime.run_action(action)
+    logger.info(obs, extra={'msg_type': 'OBSERVATION'})
+
+    logger.info(f"{'-' * 50} END Runtime Completion Fn {'-' * 50}")
+    return {
+        'rewards': json.loads(obs.content),
+    }
+
+
+def process_instance(
+    instance: pd.Series,
+    metadata: EvalMetadata,
+    reset_logger: bool = True,
+):
+    env_id = instance.instance_id
+
+    config = get_config(metadata, env_id)
+
+    # Setup the logger properly, so you can run multi-processing to parallelize the evaluation
+    if reset_logger:
+        log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs')
+        reset_logger_for_multiprocessing(logger, env_id, log_dir)
+    else:
+        logger.info(f'Starting evaluation for instance {env_id}.')
+    runtime_sid = env_id.replace('/', '_')
+    runtime = create_runtime(config, sid=runtime_sid)
+    task_str, goal_image_urls = initialize_runtime(runtime)
+    initial_user_action = MessageAction(content=task_str, images_urls=goal_image_urls)
+    state: State | None = asyncio.run(
+        run_controller(
+            config=config,
+            initial_user_action=initial_user_action,
+            runtime=runtime,
+        )
+    )
+    # ======= Attempt to evaluate the agent's environment impact =======
+
+    # If you are working on some simpler benchmark that only evaluates the final model output (e.g., in a MessageAction)
+    # You can simply get the LAST `MessageAction` from the returned `state.history` and parse it for evaluation.
+
+    if state is None:
+        raise ValueError('State should not be None.')
+
+    metrics = state.metrics.get() if state.metrics else None
+
+    # Instruction and image_urls obtained from the first message from the USER
+    instruction = ''
+    # image_urls = []
+    for event in state.history.get_events():
+        if isinstance(event, MessageAction):
+            instruction = event.content
+            # image_urls = event.images_urls
+            break
+
+    return_val = complete_runtime(runtime)
+    logger.info(f'Return value from complete_runtime: {return_val}')
+    reward = max(return_val['rewards'])
+
+    # history is now available as a stream of events, rather than list of pairs of (Action, Observation)
+    # for compatibility with the existing output format, we can remake the pairs here
+    # remove when it becomes unnecessary
+    histories = state.history.compatibility_for_eval_history_pairs()
+
+    # Save the output
+    output = EvalOutput(
+        instance_id=env_id,
+        instruction=instruction,
+        metadata=metadata,
+        history=histories,
+        metrics=metrics,
+        error=state.last_error if state and state.last_error else None,
+        test_result={
+            'reward': reward,
+        },
+    )
+    return output
+
+
+if __name__ == '__main__':
+    args = parse_arguments()
+
+    dataset = pd.DataFrame(
+        {
+            'instance_id': [
+                id
+                for id in gym.envs.registry.keys()
+                if id.startswith('browsergym/visualwebarena')
+            ]
+        }
+    )
+    llm_config = None
+    if args.llm_config:
+        llm_config = get_llm_config_arg(args.llm_config)
+    if llm_config is None:
+        raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
+    metadata = make_metadata(
+        llm_config,
+        'visualwebarena',
+        args.agent_cls,
+        args.max_iterations,
+        args.eval_note,
+        args.eval_output_dir,
+    )
+    output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
+    instances = prepare_dataset(dataset, output_file, args.eval_n_limit)
+
+    run_evaluation(
+        instances,
+        metadata,
+        output_file,
+        args.eval_num_workers,
+        process_instance,
+    )

evaluation/visualwebarena/scripts/run_infer.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -eo pipefail
+
+source "evaluation/utils/version_control.sh"
+
+# configure browsing agent
+export USE_NAV="true"
+export USE_CONCISE_ANSWER="true"
+
+MODEL_CONFIG=$1
+COMMIT_HASH=$2
+AGENT=$3
+EVAL_LIMIT=$4
+NUM_WORKERS=$5
+
+if [ -z "$NUM_WORKERS" ]; then
+  NUM_WORKERS=1
+  echo "Number of workers not specified, use default $NUM_WORKERS"
+fi
+checkout_eval_branch
+
+if [ -z "$AGENT" ]; then
+  echo "Agent not specified, use default BrowsingAgent"
+  AGENT="BrowsingAgent"
+fi
+
+get_agent_version
+
+echo "AGENT: $AGENT"
+echo "AGENT_VERSION: $AGENT_VERSION"
+echo "MODEL_CONFIG: $MODEL_CONFIG"
+
+EVAL_NOTE="$AGENT_VERSION"
+
+COMMAND="poetry run python evaluation/visualwebarena/run_infer.py \
+  --agent-cls $AGENT \
+  --llm-config $MODEL_CONFIG \
+  --max-iterations 15 \
+  --max-chars 10000000 \
+  --eval-num-workers $NUM_WORKERS \
+  --eval-note $EVAL_NOTE"
+
+if [ -n "$EVAL_LIMIT" ]; then
+  echo "EVAL_LIMIT: $EVAL_LIMIT"
+  COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT"
+fi
+
+# Run the command
+eval $COMMAND