diff --git a/minimal_example.py b/minimal_example.py new file mode 100644 index 0000000..8010054 --- /dev/null +++ b/minimal_example.py @@ -0,0 +1,133 @@ +import json +import os +import re +import subprocess +import time + +SLEEP = 1.5 +# set the URLs of each website, we use the demo sites as an example +os.environ["SHOPPING"] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:7770" +os.environ["SHOPPING_ADMIN"] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:7780/admin" +os.environ["REDDIT"] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:9999" +os.environ["GITLAB"] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8023" +os.environ["MAP"] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:3000" +os.environ["WIKIPEDIA"] = "http://ec2-3-131-244-37.us-east-2.compute.amazonaws.com:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing" +os.environ["HOMEPAGE"] = "PASS" # The home page is not currently hosted in the demo site +print("Done setting up URLs") + +# First, run `python scripts/generate_test_data.py` to generate the config files +p = subprocess.run(["python", "scripts/generate_test_data.py"], capture_output=True) + +# It will generate individual config file for each test example in config_files +assert os.path.exists("config_files/0.json") + +# Make sure the URLs in the config files are replaced properly +with open("config_files/0.json", "r") as f: + config = json.load(f) + assert os.environ["SHOPPING_ADMIN"] in config["start_url"], (os.environ["SHOPPING_ADMIN"], config["start_url"]) + +print("Done generating config files with the correct URLs") + +# run bash prepare.sh to save all account cookies +# subprocess.run(["bash", "prepare.sh"]) +# print("Done saving account cookies") + +# Init an environment +from browser_env import ( + Action, + ActionTypes, + ObservationMetadata, + ScriptBrowserEnv, + StateInfo, + action2str, + create_id_based_action, + create_stop_action, +) + +from agent.utils import Trajectory +from evaluation_harness.evaluators import evaluator_router + +# Init the environment +env = ScriptBrowserEnv( + headless=False, + slow_mo=100, + observation_type="accessibility_tree", + current_viewport_only=True, + viewport_size={ + "width": 1280, + "height": 720 + }) + +# example 156 as an example +config_file = "config_files/156.json" +# maintain a trajectory +trajectory: Trajectory = [] + +# set the environment for the current example +obs, info = env.reset(options={"config_file": config_file}) +actree_obs = obs["text"] +print(actree_obs) + +# You should see some output like this: +""" +[4] RootWebArea 'Projects · Dashboard · GitLab' focused: True + [12] link 'Skip to content' + [28] link 'Dashboard' + [2266] button '' hasPopup: menu expanded: False + [63] textbox 'Search GitLab' required: False + [61] generic 'Use the shortcut key / to start a search' + [79] link 'Create new...' + [95] link 'Issues' + [97] generic '13 assigned issues' + [101] link 'Merge requests' + [104] generic '8 merge requests'""" + +# save the state info to the trajectory +state_info: StateInfo = {"observation": obs, "info": info} +trajectory.append(state_info) + +# Now let's try to perform the action of clicking the "Merge request" link +# As the element ID is dynamic each time, we use regex to match the element as the demo +match = re.search(r"\[(\d+)\] link 'Merge requests'", actree_obs).group(1) +# Create the action click [ELEMENT_ID] +click_action = create_id_based_action(f"click [{match}]") +# Add the action to the trajectory +trajectory.append(click_action) + +# Step and get the new observation +obs, _, terminated, _, info = env.step(click_action) +# New observation +actree_obs = obs["text"] +print(actree_obs) +time.sleep(SLEEP) + +state_info = {"observation": obs, "info": info} +trajectory.append(state_info) + +# Next click "assign to you" +match = re.search(r"\[(\d+)\] link 'Assigned to you", actree_obs).group(1) +click_action = create_id_based_action(f"click [{match}]") +trajectory.append(click_action) + +obs, _, terminated, _, info = env.step(click_action) +actree_obs = obs["text"] +print(actree_obs) +time.sleep(SLEEP) +state_info = {"observation": obs, "info": info} +trajectory.append(state_info) + +# add a stop action to mark the end of the trajectory +trajectory.append(create_stop_action("")) + + +# Demo evaluation +evaluator = evaluator_router(config_file) +score = evaluator( + trajectory=trajectory, + config_file=config_file, + page=env.page, + client=env.get_page_client(env.page), +) + +# as we manually perform the task, the task should be judged as correct +assert score == 1.0