mirror of
https://github.com/web-arena-x/webarena.git
synced 2026-02-06 11:16:53 +00:00
fix black formatting
This commit is contained in:
oootttyyy
parent
9ccc2dc5ec
commit
772a5391b9
@ -68,9 +68,9 @@ def is_in_viewport(
|
||||
boxy1 = box["y"] + box["height"]
|
||||
viewportx0, viewporty0 = 0, 0
|
||||
viewportx1, viewporty1 = viewport["width"], viewport["height"]
|
||||
inter = max(0, min(boxx1, viewportx1) - max(boxx0, viewportx0)) * max(
|
||||
0, min(boxy1, viewporty1) - max(boxy0, viewporty0)
|
||||
)
|
||||
inter = max(
|
||||
0, min(boxx1, viewportx1) - max(boxx0, viewportx0)
|
||||
) * max(0, min(boxy1, viewporty1) - max(boxy0, viewporty0))
|
||||
ratio = inter / (box["width"] * box["height"])
|
||||
return ratio > threshold
|
||||
|
||||
@ -87,9 +87,9 @@ async def async_is_in_viewport(
|
||||
boxy1 = box["y"] + box["height"]
|
||||
viewportx0, viewporty0 = 0, 0
|
||||
viewportx1, viewporty1 = viewport["width"], viewport["height"]
|
||||
inter = max(0, min(boxx1, viewportx1) - max(boxx0, viewportx0)) * max(
|
||||
0, min(boxy1, viewporty1) - max(boxy0, viewporty0)
|
||||
)
|
||||
inter = max(
|
||||
0, min(boxx1, viewportx1) - max(boxx0, viewportx0)
|
||||
) * max(0, min(boxy1, viewporty1) - max(boxy0, viewporty0))
|
||||
ratio = inter / (box["width"] * box["height"])
|
||||
return ratio > threshold
|
||||
|
||||
@ -112,7 +112,9 @@ class Action(TypedDict):
|
||||
|
||||
|
||||
@beartype
|
||||
def action2str(action: Action, action_set_tag: str, semantic_element: str = "") -> str:
|
||||
def action2str(
|
||||
action: Action, action_set_tag: str, semantic_element: str = ""
|
||||
) -> str:
|
||||
"""Return the string representation of an action
|
||||
|
||||
sementic_element: the semantic information of the element
|
||||
@ -123,16 +125,12 @@ def action2str(action: Action, action_set_tag: str, semantic_element: str = "")
|
||||
match action["action_type"]:
|
||||
case ActionTypes.CLICK:
|
||||
# [ID=X] xxxxx
|
||||
action_str = (
|
||||
f"click [{element_id}] where [{element_id}] is {semantic_element}"
|
||||
)
|
||||
action_str = f"click [{element_id}] where [{element_id}] is {semantic_element}"
|
||||
case ActionTypes.TYPE:
|
||||
text = "".join([_id2key[i] for i in action["text"]])
|
||||
action_str = f"type [{element_id}] [{text}] where [{element_id}] is {semantic_element}"
|
||||
case ActionTypes.HOVER:
|
||||
action_str = (
|
||||
f"hover [{element_id}] where [{element_id}] is {semantic_element}"
|
||||
)
|
||||
action_str = f"hover [{element_id}] where [{element_id}] is {semantic_element}"
|
||||
case ActionTypes.SCROLL:
|
||||
action_str = f"scroll [{action['direction']}]"
|
||||
case ActionTypes.KEY_PRESS:
|
||||
@ -154,9 +152,13 @@ def action2str(action: Action, action_set_tag: str, semantic_element: str = "")
|
||||
case ActionTypes.NONE:
|
||||
action_str = "none"
|
||||
case _:
|
||||
raise ValueError(f"Unknown action type {action['action_type']}")
|
||||
raise ValueError(
|
||||
f"Unknown action type {action['action_type']}"
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(f"Unknown action set tag {action_set_tag}")
|
||||
raise NotImplementedError(
|
||||
f"Unknown action set tag {action_set_tag}"
|
||||
)
|
||||
|
||||
return action_str
|
||||
|
||||
@ -170,7 +172,9 @@ def action2create_function(action: Action) -> str:
|
||||
direction = "up" if "up" in action["direction"] else "down"
|
||||
return f"create_scroll_action({repr(direction)})"
|
||||
case ActionTypes.KEY_PRESS:
|
||||
return f"create_key_press_action({repr(action['key_comb'])})"
|
||||
return (
|
||||
f"create_key_press_action({repr(action['key_comb'])})"
|
||||
)
|
||||
# inter-page actions
|
||||
case ActionTypes.PAGE_FOCUS:
|
||||
return f"create_page_focus_action({action['page_number']})"
|
||||
@ -197,7 +201,9 @@ def action2create_function(action: Action) -> str:
|
||||
case ActionTypes.CLICK:
|
||||
args = []
|
||||
args.append(f"element_id={repr(action['element_id'])}")
|
||||
args.append(f"element_role={repr(_id2role[action['element_role']])}")
|
||||
args.append(
|
||||
f"element_role={repr(_id2role[action['element_role']])}"
|
||||
)
|
||||
args.append(f"element_name={repr(action['element_name'])}")
|
||||
args.append(f"pw_code={repr(action['pw_code'])}")
|
||||
args_str = ", ".join(args)
|
||||
@ -205,7 +211,9 @@ def action2create_function(action: Action) -> str:
|
||||
case ActionTypes.HOVER:
|
||||
args = []
|
||||
args.append(f"element_id={repr(action['element_id'])}")
|
||||
args.append(f"element_role={repr(_id2role[action['element_role']])}")
|
||||
args.append(
|
||||
f"element_role={repr(_id2role[action['element_role']])}"
|
||||
)
|
||||
args.append(f"element_name={repr(action['element_name'])}")
|
||||
args.append(f"pw_code={repr(action['pw_code'])}")
|
||||
args_str = ", ".join(args)
|
||||
@ -215,7 +223,9 @@ def action2create_function(action: Action) -> str:
|
||||
text = "".join(map(lambda x: _id2key[x], action["text"]))
|
||||
args.append(f"text={repr(text)}")
|
||||
args.append(f"element_id={repr(action['element_id'])}")
|
||||
args.append(f"element_role={repr(_id2role[action['element_role']])}")
|
||||
args.append(
|
||||
f"element_role={repr(_id2role[action['element_role']])}"
|
||||
)
|
||||
args.append(f"element_name={repr(action['element_name'])}")
|
||||
args.append(f"pw_code={repr(action['pw_code'])}")
|
||||
args_str = ", ".join(args)
|
||||
@ -326,7 +336,8 @@ _key2id: dict[str, int] = {
|
||||
}
|
||||
_id2key: list[str] = sorted(_key2id, key=_key2id.get) # type: ignore[arg-type]
|
||||
_role2id: dict[RolesType, int] = {
|
||||
cast(RolesType, role): i for i, role in enumerate(chain(ROLES, SPECIAL_LOCATORS))
|
||||
cast(RolesType, role): i
|
||||
for i, role in enumerate(chain(ROLES, SPECIAL_LOCATORS))
|
||||
}
|
||||
_id2role: list[RolesType] = sorted(_role2id, key=_role2id.get) # type: ignore[arg-type]
|
||||
|
||||
@ -335,7 +346,9 @@ _id2role: list[RolesType] = sorted(_role2id, key=_role2id.get) # type: ignore[a
|
||||
def _keys2ids(keys: list[int | str] | str) -> list[int]:
|
||||
return list(
|
||||
map(
|
||||
lambda key: _key2id[str(key)] if is_bearable(key, str) else int(key),
|
||||
lambda key: _key2id[str(key)]
|
||||
if is_bearable(key, str)
|
||||
else int(key),
|
||||
keys,
|
||||
)
|
||||
)
|
||||
@ -352,13 +365,19 @@ def get_action_space() -> spaces.Dict:
|
||||
np.array([1.0, 1.0], dtype=np.float32),
|
||||
),
|
||||
# element role is used for FOCUS_AND_CLICK and FOCUS_AND_TYPE
|
||||
"element_role": spaces.Discrete(len(ROLES) + len(SPECIAL_LOCATORS)),
|
||||
"element_role": spaces.Discrete(
|
||||
len(ROLES) + len(SPECIAL_LOCATORS)
|
||||
),
|
||||
# element name is used with element role
|
||||
"element_name": spaces.Text(TEXT_MAX_LENGTH),
|
||||
"element_id": spaces.Text(TEXT_MAX_LENGTH),
|
||||
# text is only used for TYPE and FOCUS_AND_TYPE
|
||||
"text": spaces.MultiDiscrete(
|
||||
[len(ASCII_CHARSET) + len(SPECIAL_KEYS) + len(FREQ_UNICODE_CHARSET)]
|
||||
[
|
||||
len(ASCII_CHARSET)
|
||||
+ len(SPECIAL_KEYS)
|
||||
+ len(FREQ_UNICODE_CHARSET)
|
||||
]
|
||||
* TYPING_MAX_LENGTH
|
||||
),
|
||||
"page_number": spaces.Discrete(MAX_PAGE_NUMBER),
|
||||
@ -378,9 +397,13 @@ def create_random_action() -> Action:
|
||||
return {
|
||||
"action_type": np.random.randint(len(ActionTypes)),
|
||||
"coords": np.random.rand(2).astype(np.float32),
|
||||
"element_role": np.random.randint(len(ROLES) + len(SPECIAL_LOCATORS)),
|
||||
"element_role": np.random.randint(
|
||||
len(ROLES) + len(SPECIAL_LOCATORS)
|
||||
),
|
||||
"element_name": "".join(
|
||||
random.choices(ASCII_CHARSET, k=np.random.randint(TEXT_MAX_LENGTH))
|
||||
random.choices(
|
||||
ASCII_CHARSET, k=np.random.randint(TEXT_MAX_LENGTH)
|
||||
)
|
||||
),
|
||||
"text": list(
|
||||
random.choices(
|
||||
@ -390,11 +413,15 @@ def create_random_action() -> Action:
|
||||
),
|
||||
"page_number": np.random.randint(MAX_PAGE_NUMBER),
|
||||
"url": "".join(
|
||||
random.choices(ASCII_CHARSET, k=np.random.randint(URL_MAX_LENGTH))
|
||||
random.choices(
|
||||
ASCII_CHARSET, k=np.random.randint(URL_MAX_LENGTH)
|
||||
)
|
||||
),
|
||||
"nth": np.random.randint(MAX_ELEMENT_INDEX_IN_VIEWPORT),
|
||||
"element_id": str(np.random.randint(MAX_ELEMENT_ID)),
|
||||
"key_comb": "+".join(random.choices(SPECIAL_KEYS, k=np.random.randint(3))),
|
||||
"key_comb": "+".join(
|
||||
random.choices(SPECIAL_KEYS, k=np.random.randint(3))
|
||||
),
|
||||
"direction": random.choice(["up", "down"]),
|
||||
"pw_code": "".join(
|
||||
random.choices(
|
||||
@ -581,7 +608,9 @@ def create_mouse_click_action(
|
||||
}
|
||||
)
|
||||
else:
|
||||
raise ValueError("left and top must be both None or both not None")
|
||||
raise ValueError(
|
||||
"left and top must be both None or both not None"
|
||||
)
|
||||
return action
|
||||
|
||||
|
||||
@ -785,7 +814,9 @@ async def aexecute_scroll(direction: str, page: APage) -> None:
|
||||
@beartype
|
||||
def execute_key_press(key: str, page: Page) -> None:
|
||||
"""Press a key."""
|
||||
if "Meta" in key and "Mac" not in page.evaluate("navigator.platform"):
|
||||
if "Meta" in key and "Mac" not in page.evaluate(
|
||||
"navigator.platform"
|
||||
):
|
||||
key = key.replace("Meta", "Control")
|
||||
page.keyboard.press(key)
|
||||
|
||||
@ -793,7 +824,9 @@ def execute_key_press(key: str, page: Page) -> None:
|
||||
@beartype
|
||||
async def aexecute_key_press(key: str, page: APage) -> None:
|
||||
"""Press a key."""
|
||||
if "Meta" in key and "Mac" not in page.evaluate("navigator.platform"):
|
||||
if "Meta" in key and "Mac" not in page.evaluate(
|
||||
"navigator.platform"
|
||||
):
|
||||
key = key.replace("Meta", "Control")
|
||||
await page.keyboard.press(key)
|
||||
|
||||
@ -803,30 +836,42 @@ def execute_mouse_hover(left: float, top: float, page: Page) -> None:
|
||||
"""Click at coordinates (left, top)."""
|
||||
viewport_size = page.viewport_size
|
||||
assert viewport_size
|
||||
page.mouse.move(left * viewport_size["width"], top * viewport_size["height"])
|
||||
page.mouse.move(
|
||||
left * viewport_size["width"], top * viewport_size["height"]
|
||||
)
|
||||
|
||||
|
||||
@beartype
|
||||
async def aexecute_mouse_hover(left: float, top: float, page: APage) -> None:
|
||||
async def aexecute_mouse_hover(
|
||||
left: float, top: float, page: APage
|
||||
) -> None:
|
||||
"""Click at coordinates (left, top)."""
|
||||
viewport_size = page.viewport_size
|
||||
assert viewport_size
|
||||
await page.mouse.move(left * viewport_size["width"], top * viewport_size["height"])
|
||||
await page.mouse.move(
|
||||
left * viewport_size["width"], top * viewport_size["height"]
|
||||
)
|
||||
|
||||
|
||||
def execute_mouse_click(left: float, top: float, page: Page) -> None:
|
||||
"""Click at coordinates (left, top)."""
|
||||
viewport_size = page.viewport_size
|
||||
assert viewport_size
|
||||
page.mouse.click(left * viewport_size["width"], top * viewport_size["height"])
|
||||
page.mouse.click(
|
||||
left * viewport_size["width"], top * viewport_size["height"]
|
||||
)
|
||||
|
||||
|
||||
@beartype
|
||||
async def aexecute_mouse_click(left: float, top: float, page: APage) -> None:
|
||||
async def aexecute_mouse_click(
|
||||
left: float, top: float, page: APage
|
||||
) -> None:
|
||||
"""Click at coordinates (left, top)."""
|
||||
viewport_size = page.viewport_size
|
||||
assert viewport_size
|
||||
await page.mouse.click(left * viewport_size["width"], top * viewport_size["height"])
|
||||
await page.mouse.click(
|
||||
left * viewport_size["width"], top * viewport_size["height"]
|
||||
)
|
||||
|
||||
|
||||
@beartype
|
||||
@ -883,11 +928,15 @@ async def aexecute_type(keys: list[int], page: APage) -> None:
|
||||
|
||||
|
||||
@beartype
|
||||
def execute_focus(element_role: int, element_name: str, nth: int, page: Page) -> None:
|
||||
def execute_focus(
|
||||
element_role: int, element_name: str, nth: int, page: Page
|
||||
) -> None:
|
||||
"""Click the specified DOM element."""
|
||||
element_role_str = _id2role[element_role]
|
||||
if page.viewport_size is None:
|
||||
raise ValueError("Viewport size is not set for the current page")
|
||||
raise ValueError(
|
||||
"Viewport size is not set for the current page"
|
||||
)
|
||||
element_location_list: list[tuple[Locator, float, float]] = []
|
||||
for frame in page.frames:
|
||||
match element_role_str:
|
||||
@ -898,7 +947,9 @@ def execute_focus(element_role: int, element_name: str, nth: int, page: Page) ->
|
||||
case "placeholder":
|
||||
locators = frame.get_by_placeholder(element_name)
|
||||
case _:
|
||||
locators = frame.get_by_role(role=element_role_str, name=element_name)
|
||||
locators = frame.get_by_role(
|
||||
role=element_role_str, name=element_name
|
||||
)
|
||||
for locator_idx in range(locators.count()):
|
||||
locator = locators.nth(locator_idx)
|
||||
if is_in_viewport(locator, page.viewport_size):
|
||||
@ -911,7 +962,9 @@ def execute_focus(element_role: int, element_name: str, nth: int, page: Page) ->
|
||||
raise ValueError(
|
||||
f"There are only {len(element_location_list)} elements found in viewport, but {nth + 1} is requested"
|
||||
)
|
||||
element_location_list.sort(key=lambda x: (x[2], x[1])) # row major order
|
||||
element_location_list.sort(
|
||||
key=lambda x: (x[2], x[1])
|
||||
) # row major order
|
||||
element_location_list[nth][0].focus()
|
||||
|
||||
|
||||
@ -922,7 +975,9 @@ async def aexecute_focus(
|
||||
"""Click the specified DOM element."""
|
||||
element_role_str = _id2role[element_role]
|
||||
if page.viewport_size is None:
|
||||
raise ValueError("Viewport size is not set for the current page")
|
||||
raise ValueError(
|
||||
"Viewport size is not set for the current page"
|
||||
)
|
||||
element_location_list: list[tuple[ALocator, float, float]] = []
|
||||
for frame in page.frames:
|
||||
match element_role_str:
|
||||
@ -933,7 +988,9 @@ async def aexecute_focus(
|
||||
case "placeholder":
|
||||
locators = frame.get_by_placeholder(element_name)
|
||||
case _:
|
||||
locators = frame.get_by_role(role=element_role_str, name=element_name)
|
||||
locators = frame.get_by_role(
|
||||
role=element_role_str, name=element_name
|
||||
)
|
||||
for locator_idx in range(await locators.count()):
|
||||
locator = locators.nth(locator_idx)
|
||||
if await async_is_in_viewport(locator, page.viewport_size):
|
||||
@ -946,29 +1003,39 @@ async def aexecute_focus(
|
||||
raise ValueError(
|
||||
f"There are only {len(element_location_list)} elements found in viewport, but {nth + 1} is requested"
|
||||
)
|
||||
element_location_list.sort(key=lambda x: (x[2], x[1])) # row major order
|
||||
element_location_list.sort(
|
||||
key=lambda x: (x[2], x[1])
|
||||
) # row major order
|
||||
await element_location_list[nth][0].focus()
|
||||
|
||||
|
||||
@beartype
|
||||
def locate(locator_calls: list[ParsedPlaywrightCode], page: Page) -> Locator:
|
||||
def locate(
|
||||
locator_calls: list[ParsedPlaywrightCode], page: Page
|
||||
) -> Locator:
|
||||
locator = page
|
||||
for call in locator_calls:
|
||||
function_name = call["function_name"]
|
||||
arguments = call["arguments"]
|
||||
keywords = call["keywords"]
|
||||
locator = getattr(locator, function_name)(*arguments, **keywords)
|
||||
locator = getattr(locator, function_name)(
|
||||
*arguments, **keywords
|
||||
)
|
||||
return locator # type: ignore[return-value]
|
||||
|
||||
|
||||
@beartype
|
||||
async def alocate(locator_calls: list[ParsedPlaywrightCode], page: APage) -> ALocator:
|
||||
async def alocate(
|
||||
locator_calls: list[ParsedPlaywrightCode], page: APage
|
||||
) -> ALocator:
|
||||
locator = page
|
||||
for call in locator_calls:
|
||||
function_name = call["function_name"]
|
||||
arguments = call["arguments"]
|
||||
keywords = call["keywords"]
|
||||
locator = await getattr(locator, function_name)(*arguments, **keywords)
|
||||
locator = await getattr(locator, function_name)(
|
||||
*arguments, **keywords
|
||||
)
|
||||
return locator # type: ignore[return-value]
|
||||
|
||||
|
||||
@ -1028,7 +1095,9 @@ def execute_playwright_type(
|
||||
) -> None:
|
||||
locator = locate(locator_code, page)
|
||||
# perform the action
|
||||
pw_action_args = [text] + pw_action_args # text is the first argument
|
||||
pw_action_args = [
|
||||
text
|
||||
] + pw_action_args # text is the first argument
|
||||
locator.type(*pw_action_args, **pw_action_kwargs)
|
||||
|
||||
|
||||
@ -1042,7 +1111,9 @@ async def aexecute_playwright_type(
|
||||
) -> None:
|
||||
locator = await alocate(locator_code, page)
|
||||
# perform the action
|
||||
pw_action_args = [text] + pw_action_args # text is the first argument
|
||||
pw_action_args = [
|
||||
text
|
||||
] + pw_action_args # text is the first argument
|
||||
await locator.type(*pw_action_args, **pw_action_kwargs)
|
||||
|
||||
|
||||
@ -1109,9 +1180,13 @@ def execute_action(
|
||||
execute_key_press(keys, page)
|
||||
|
||||
case ActionTypes.MOUSE_CLICK:
|
||||
execute_mouse_click(action["coords"][0], action["coords"][1], page)
|
||||
execute_mouse_click(
|
||||
action["coords"][0], action["coords"][1], page
|
||||
)
|
||||
case ActionTypes.MOUSE_HOVER:
|
||||
execute_mouse_hover(action["coords"][0], action["coords"][1], page)
|
||||
execute_mouse_hover(
|
||||
action["coords"][0], action["coords"][1], page
|
||||
)
|
||||
case ActionTypes.KEYBOARD_TYPE:
|
||||
execute_type(action["text"], page)
|
||||
|
||||
@ -1121,7 +1196,9 @@ def execute_action(
|
||||
if action["element_id"]:
|
||||
element_id = action["element_id"]
|
||||
element_center = obseration_processor.get_element_center(element_id) # type: ignore[attr-defined]
|
||||
execute_mouse_click(element_center[0], element_center[1], page)
|
||||
execute_mouse_click(
|
||||
element_center[0], element_center[1], page
|
||||
)
|
||||
elif action["element_role"] and action["element_name"]:
|
||||
element_role = int(action["element_role"])
|
||||
element_name = action["element_name"]
|
||||
@ -1132,14 +1209,20 @@ def execute_action(
|
||||
parsed_code = parse_playwright_code(action["pw_code"])
|
||||
locator_code = parsed_code[:-1]
|
||||
# [shuyanzh], don't support action args and kwargs now
|
||||
execute_playwright_click(locator_code=locator_code, page=page)
|
||||
execute_playwright_click(
|
||||
locator_code=locator_code, page=page
|
||||
)
|
||||
else:
|
||||
raise ValueError("No proper locator found for click action")
|
||||
raise ValueError(
|
||||
"No proper locator found for click action"
|
||||
)
|
||||
case ActionTypes.HOVER:
|
||||
if action["element_id"]:
|
||||
element_id = action["element_id"]
|
||||
element_center = obseration_processor.get_element_center(element_id) # type: ignore[attr-defined]
|
||||
execute_mouse_hover(element_center[0], element_center[1], page)
|
||||
execute_mouse_hover(
|
||||
element_center[0], element_center[1], page
|
||||
)
|
||||
elif action["element_role"] and action["element_name"]:
|
||||
element_role = int(action["element_role"])
|
||||
element_name = action["element_name"]
|
||||
@ -1149,14 +1232,20 @@ def execute_action(
|
||||
parsed_code = parse_playwright_code(action["pw_code"])
|
||||
locator_code = parsed_code[:-1]
|
||||
# [shuyanzh], don't support action args and kwargs now
|
||||
execute_playwright_hover(locator_code=locator_code, page=page)
|
||||
execute_playwright_hover(
|
||||
locator_code=locator_code, page=page
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError("No proper locator found for hover action")
|
||||
raise NotImplementedError(
|
||||
"No proper locator found for hover action"
|
||||
)
|
||||
case ActionTypes.TYPE:
|
||||
if action["element_id"]:
|
||||
element_id = action["element_id"]
|
||||
element_center = obseration_processor.get_element_center(element_id) # type: ignore[attr-defined]
|
||||
execute_mouse_click(element_center[0], element_center[1], page)
|
||||
execute_mouse_click(
|
||||
element_center[0], element_center[1], page
|
||||
)
|
||||
execute_type(action["text"], page)
|
||||
elif action["element_role"] and action["element_name"]:
|
||||
element_role = int(action["element_role"])
|
||||
@ -1169,9 +1258,13 @@ def execute_action(
|
||||
locator_code = parsed_code[:-1]
|
||||
text = parsed_code[-1]["arguments"][0]
|
||||
# [shuyanzh], don't support action args and kwargs now
|
||||
execute_playwright_type(text=text, locator_code=locator_code, page=page)
|
||||
execute_playwright_type(
|
||||
text=text, locator_code=locator_code, page=page
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError("No proper locator found for type action")
|
||||
raise NotImplementedError(
|
||||
"No proper locator found for type action"
|
||||
)
|
||||
|
||||
case ActionTypes.PAGE_FOCUS:
|
||||
page = browser_ctx.pages[action["page_number"]]
|
||||
@ -1234,9 +1327,13 @@ async def aexecute_action(
|
||||
await aexecute_key_press(keys, page)
|
||||
|
||||
case ActionTypes.MOUSE_CLICK:
|
||||
await aexecute_mouse_click(action["coords"][0], action["coords"][1], page)
|
||||
await aexecute_mouse_click(
|
||||
action["coords"][0], action["coords"][1], page
|
||||
)
|
||||
case ActionTypes.MOUSE_HOVER:
|
||||
await aexecute_mouse_hover(action["coords"][0], action["coords"][1], page)
|
||||
await aexecute_mouse_hover(
|
||||
action["coords"][0], action["coords"][1], page
|
||||
)
|
||||
case ActionTypes.KEYBOARD_TYPE:
|
||||
await aexecute_type(action["text"], page)
|
||||
|
||||
@ -1249,15 +1346,21 @@ async def aexecute_action(
|
||||
element_role = int(action["element_role"])
|
||||
element_name = action["element_name"]
|
||||
nth = action["nth"]
|
||||
await aexecute_focus(element_role, element_name, nth, page)
|
||||
await aexecute_focus(
|
||||
element_role, element_name, nth, page
|
||||
)
|
||||
await aexecute_click_current(page)
|
||||
elif action["pw_code"]:
|
||||
parsed_code = parse_playwright_code(action["pw_code"])
|
||||
locator_code = parsed_code[:-1]
|
||||
# [shuyanzh], don't support action args and kwargs now
|
||||
await aexecute_playwright_click(locator_code=locator_code, page=page)
|
||||
await aexecute_playwright_click(
|
||||
locator_code=locator_code, page=page
|
||||
)
|
||||
else:
|
||||
raise ValueError("No proper locator found for click action")
|
||||
raise ValueError(
|
||||
"No proper locator found for click action"
|
||||
)
|
||||
case ActionTypes.HOVER:
|
||||
if action["element_id"]:
|
||||
raise NotImplementedError
|
||||
@ -1265,14 +1368,20 @@ async def aexecute_action(
|
||||
element_role = int(action["element_role"])
|
||||
element_name = action["element_name"]
|
||||
nth = action["nth"]
|
||||
await aexecute_focus(element_role, element_name, nth, page)
|
||||
await aexecute_focus(
|
||||
element_role, element_name, nth, page
|
||||
)
|
||||
elif action["pw_code"]:
|
||||
parsed_code = parse_playwright_code(action["pw_code"])
|
||||
locator_code = parsed_code[:-1]
|
||||
# [shuyanzh], don't support action args and kwargs now
|
||||
await aexecute_playwright_hover(locator_code=locator_code, page=page)
|
||||
await aexecute_playwright_hover(
|
||||
locator_code=locator_code, page=page
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError("No proper locator found for hover action")
|
||||
raise NotImplementedError(
|
||||
"No proper locator found for hover action"
|
||||
)
|
||||
case ActionTypes.TYPE:
|
||||
if action["element_id"]:
|
||||
raise NotImplementedError
|
||||
@ -1280,7 +1389,9 @@ async def aexecute_action(
|
||||
element_role = int(action["element_role"])
|
||||
element_name = action["element_name"]
|
||||
nth = action["nth"]
|
||||
await aexecute_focus(element_role, element_name, nth, page)
|
||||
await aexecute_focus(
|
||||
element_role, element_name, nth, page
|
||||
)
|
||||
await aexecute_type(action["text"], page)
|
||||
elif action["pw_code"]:
|
||||
parsed_code = parse_playwright_code(action["pw_code"])
|
||||
@ -1291,7 +1402,9 @@ async def aexecute_action(
|
||||
text=text, locator_code=locator_code, page=page
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError("No proper locator found for type action")
|
||||
raise NotImplementedError(
|
||||
"No proper locator found for type action"
|
||||
)
|
||||
|
||||
case ActionTypes.PAGE_FOCUS:
|
||||
page = browser_ctx.pages[action["page_number"]]
|
||||
@ -1315,7 +1428,9 @@ async def aexecute_action(
|
||||
if action["pw_code"]:
|
||||
parsed_code = parse_playwright_code(action["pw_code"])
|
||||
locator_code = parsed_code[:-1]
|
||||
await aexecute_playwright_select_option(locator_code, page)
|
||||
await aexecute_playwright_select_option(
|
||||
locator_code, page
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"No proper locator found for select option action"
|
||||
@ -1340,7 +1455,9 @@ async def aexecute_action(
|
||||
def parse_playwright_code(code: str) -> list[ParsedPlaywrightCode]:
|
||||
# extract function calls
|
||||
if not code.startswith("page."):
|
||||
raise ValueError(f'Playwright action must start with "page.", but got {code}')
|
||||
raise ValueError(
|
||||
f'Playwright action must start with "page.", but got {code}'
|
||||
)
|
||||
|
||||
regex = r"\.(?![^\(\)]*\))"
|
||||
chain = re.split(regex, code)[1:]
|
||||
@ -1354,11 +1471,14 @@ def parse_playwright_code(code: str) -> list[ParsedPlaywrightCode]:
|
||||
if isinstance(node, ast.Call):
|
||||
function_name = node.func.id # type: ignore[attr-defined]
|
||||
arguments = [
|
||||
ast.literal_eval(arg) if isinstance(arg, ast.Str) else arg
|
||||
ast.literal_eval(arg)
|
||||
if isinstance(arg, ast.Str)
|
||||
else arg
|
||||
for arg in node.args
|
||||
]
|
||||
keywords = {
|
||||
str(kw.arg): ast.literal_eval(kw.value) for kw in node.keywords
|
||||
str(kw.arg): ast.literal_eval(kw.value)
|
||||
for kw in node.keywords
|
||||
}
|
||||
funcs.append(
|
||||
ParsedPlaywrightCode(
|
||||
@ -1373,7 +1493,10 @@ def parse_playwright_code(code: str) -> list[ParsedPlaywrightCode]:
|
||||
if len(funcs) != 1:
|
||||
raise ValueError(f"Fail to parse {item} in {code}")
|
||||
|
||||
if funcs[0]["function_name"] not in PLAYWRIGHT_LOCATORS + PLAYWRIGHT_ACTIONS:
|
||||
if (
|
||||
funcs[0]["function_name"]
|
||||
not in PLAYWRIGHT_LOCATORS + PLAYWRIGHT_ACTIONS
|
||||
):
|
||||
raise ValueError(
|
||||
f"Invalid playwright code {item}, ",
|
||||
f"the function needs to be one of {PLAYWRIGHT_LOCATORS + PLAYWRIGHT_ACTIONS}",
|
||||
@ -1429,7 +1552,9 @@ def create_playwright_action(playwright_code: str) -> Action:
|
||||
f"Invalid type/fill action, required to be page.type(TEXT)"
|
||||
)
|
||||
text = match.group(1)
|
||||
return create_type_action(text=text, pw_code=playwright_code)
|
||||
return create_type_action(
|
||||
text=text, pw_code=playwright_code
|
||||
)
|
||||
case "select_option":
|
||||
return create_select_option_action(pw_code=playwright_code)
|
||||
case "check":
|
||||
@ -1448,7 +1573,9 @@ def create_playwright_action(playwright_code: str) -> Action:
|
||||
p = r"page_focus\((\d+)\)"
|
||||
match = re.search(p, playwright_code)
|
||||
if not match:
|
||||
raise ActionParsingError("page focus requires a page number")
|
||||
raise ActionParsingError(
|
||||
"page focus requires a page number"
|
||||
)
|
||||
page_num = int(match.group(1))
|
||||
return create_page_focus_action(page_num)
|
||||
case "new_tab":
|
||||
@ -1484,23 +1611,33 @@ def create_id_based_action(action_str: str) -> Action:
|
||||
case "click":
|
||||
match = re.search(r"click ?\[(\d+)\]", action_str)
|
||||
if not match:
|
||||
raise ActionParsingError(f"Invalid click action {action_str}")
|
||||
raise ActionParsingError(
|
||||
f"Invalid click action {action_str}"
|
||||
)
|
||||
element_id = match.group(1)
|
||||
return create_click_action(element_id=element_id)
|
||||
case "hover":
|
||||
match = re.search(r"hover ?\[(\d+)\]", action_str)
|
||||
if not match:
|
||||
raise ActionParsingError(f"Invalid hover action {action_str}")
|
||||
raise ActionParsingError(
|
||||
f"Invalid hover action {action_str}"
|
||||
)
|
||||
element_id = match.group(1)
|
||||
return create_hover_action(element_id=element_id)
|
||||
case "type":
|
||||
# add default enter flag
|
||||
if not (action_str.endswith("[0]") or action_str.endswith("[1]")):
|
||||
if not (
|
||||
action_str.endswith("[0]") or action_str.endswith("[1]")
|
||||
):
|
||||
action_str += " [1]"
|
||||
|
||||
match = re.search(r"type ?\[(\d+)\] ?\[(.+)\] ?\[(\d+)\]", action_str)
|
||||
match = re.search(
|
||||
r"type ?\[(\d+)\] ?\[(.+)\] ?\[(\d+)\]", action_str
|
||||
)
|
||||
if not match:
|
||||
raise ActionParsingError(f"Invalid type action {action_str}")
|
||||
raise ActionParsingError(
|
||||
f"Invalid type action {action_str}"
|
||||
)
|
||||
element_id, text, enter_flag = (
|
||||
match.group(1),
|
||||
match.group(2),
|
||||
@ -1512,20 +1649,26 @@ def create_id_based_action(action_str: str) -> Action:
|
||||
case "press":
|
||||
match = re.search(r"press ?\[(.+)\]", action_str)
|
||||
if not match:
|
||||
raise ActionParsingError(f"Invalid press action {action_str}")
|
||||
raise ActionParsingError(
|
||||
f"Invalid press action {action_str}"
|
||||
)
|
||||
key_comb = match.group(1)
|
||||
return create_key_press_action(key_comb=key_comb)
|
||||
case "scroll":
|
||||
# up or down
|
||||
match = re.search(r"scroll ?\[?(up|down)\]?", action_str)
|
||||
if not match:
|
||||
raise ActionParsingError(f"Invalid scroll action {action_str}")
|
||||
raise ActionParsingError(
|
||||
f"Invalid scroll action {action_str}"
|
||||
)
|
||||
direction = match.group(1)
|
||||
return create_scroll_action(direction=direction)
|
||||
case "goto":
|
||||
match = re.search(r"goto ?\[(.+)\]", action_str)
|
||||
if not match:
|
||||
raise ActionParsingError(f"Invalid goto action {action_str}")
|
||||
raise ActionParsingError(
|
||||
f"Invalid goto action {action_str}"
|
||||
)
|
||||
url = match.group(1)
|
||||
return create_goto_url_action(url=url)
|
||||
case "new_tab":
|
||||
@ -1537,7 +1680,9 @@ def create_id_based_action(action_str: str) -> Action:
|
||||
case "tab_focus":
|
||||
match = re.search(r"tab_focus ?\[(\d+)\]", action_str)
|
||||
if not match:
|
||||
raise ActionParsingError(f"Invalid tab_focus action {action_str}")
|
||||
raise ActionParsingError(
|
||||
f"Invalid tab_focus action {action_str}"
|
||||
)
|
||||
page_number = int(match.group(1))
|
||||
return create_page_focus_action(page_number)
|
||||
case "close_tab":
|
||||
|
||||
@ -24,7 +24,9 @@ def test_frame_locator(script_browser_env: ScriptBrowserEnv) -> None:
|
||||
env.reset()
|
||||
for action in seq.split("\n"):
|
||||
action = action.strip()
|
||||
_, success, _, _, info = env.step(create_playwright_action(action))
|
||||
_, success, _, _, info = env.step(
|
||||
create_playwright_action(action)
|
||||
)
|
||||
assert success
|
||||
|
||||
|
||||
@ -47,7 +49,9 @@ def test_basic(script_browser_env: ScriptBrowserEnv) -> None:
|
||||
env.reset()
|
||||
for action in seq.split("\n"):
|
||||
action = action.strip()
|
||||
_, success, _, _, info = env.step(create_playwright_action(action))
|
||||
_, success, _, _, info = env.step(
|
||||
create_playwright_action(action)
|
||||
)
|
||||
assert success
|
||||
|
||||
|
||||
@ -59,7 +63,9 @@ def test_hover(script_browser_env: ScriptBrowserEnv) -> None:
|
||||
env.reset()
|
||||
for action in seq.split("\n"):
|
||||
action = action.strip()
|
||||
_, success, _, _, info = env.step(create_playwright_action(action))
|
||||
_, success, _, _, info = env.step(
|
||||
create_playwright_action(action)
|
||||
)
|
||||
assert success
|
||||
|
||||
|
||||
@ -70,7 +76,9 @@ def test_select_option(script_browser_env: ScriptBrowserEnv) -> None:
|
||||
env.reset()
|
||||
for action in seq.split("\n"):
|
||||
action = action.strip()
|
||||
_, success, _, _, info = env.step(create_playwright_action(action))
|
||||
_, success, _, _, info = env.step(
|
||||
create_playwright_action(action)
|
||||
)
|
||||
assert success
|
||||
|
||||
|
||||
@ -90,11 +98,15 @@ def test_xpath(script_browser_env: ScriptBrowserEnv) -> None:
|
||||
env.reset()
|
||||
for action in seq.split("\n"):
|
||||
action = action.strip()
|
||||
_, success, _, _, info = env.step(create_playwright_action(action))
|
||||
_, success, _, _, info = env.step(
|
||||
create_playwright_action(action)
|
||||
)
|
||||
assert success
|
||||
|
||||
|
||||
def test_inter_page_actions(script_browser_env: ScriptBrowserEnv) -> None:
|
||||
def test_inter_page_actions(
|
||||
script_browser_env: ScriptBrowserEnv,
|
||||
) -> None:
|
||||
env = script_browser_env
|
||||
seq = """page.goto("https://demo.playwright.dev/todomvc/")
|
||||
browser.new_tab()
|
||||
@ -108,12 +120,16 @@ def test_inter_page_actions(script_browser_env: ScriptBrowserEnv) -> None:
|
||||
env.reset()
|
||||
for action in seq.split("\n"):
|
||||
action = action.strip()
|
||||
_, success, _, _, info = env.step(create_playwright_action(action))
|
||||
_, success, _, _, info = env.step(
|
||||
create_playwright_action(action)
|
||||
)
|
||||
assert success
|
||||
assert "https://demo.playwright.dev/todomvc" in info["page"].url
|
||||
|
||||
|
||||
def test_scroll(current_viewport_script_browser_env: ScriptBrowserEnv) -> None:
|
||||
def test_scroll(
|
||||
current_viewport_script_browser_env: ScriptBrowserEnv,
|
||||
) -> None:
|
||||
env = current_viewport_script_browser_env
|
||||
env.reset()
|
||||
_, success, _, _, _ = env.step(create_scroll_action("down"))
|
||||
@ -138,21 +154,33 @@ def test_id_click(
|
||||
# get the id of the link
|
||||
element_id = re.search(r"\[(\d+)\] link 'McKenna/Bell'", obs["text"]).group(1) # type: ignore
|
||||
|
||||
obs, success, _, _, info = env.step(create_id_based_action(f"click [{element_id}]"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_id_based_action(f"click [{element_id}]")
|
||||
)
|
||||
assert success
|
||||
assert info["page"].url == "https://russmaxdesign.github.io/exercise/#link-four"
|
||||
assert (
|
||||
info["page"].url
|
||||
== "https://russmaxdesign.github.io/exercise/#link-four"
|
||||
)
|
||||
|
||||
obs, success, _, _, info = env.step(create_scroll_action("down"))
|
||||
assert "link 'Classification'" in obs["text"]
|
||||
element_id = re.search(r"\[(\d+)\] link 'Classification'", obs["text"]).group(1) # type: ignore
|
||||
|
||||
obs, success, _, _, info = env.step(create_id_based_action(f"click [{element_id}]"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_id_based_action(f"click [{element_id}]")
|
||||
)
|
||||
assert success
|
||||
assert info["page"].url == "https://russmaxdesign.github.io/exercise/#link-two"
|
||||
assert (
|
||||
info["page"].url
|
||||
== "https://russmaxdesign.github.io/exercise/#link-two"
|
||||
)
|
||||
assert "radio 'Weekly'" in obs["text"]
|
||||
element_id = re.search(r"\[(\d+)\] radio 'Weekly'", obs["text"]).group(1) # type: ignore
|
||||
|
||||
obs, success, _, _, info = env.step(create_id_based_action(f"click [{element_id}]"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_id_based_action(f"click [{element_id}]")
|
||||
)
|
||||
assert success
|
||||
assert "radio 'Weekly'" in obs["text"]
|
||||
|
||||
@ -164,13 +192,17 @@ def test_id_hover(
|
||||
env.reset()
|
||||
|
||||
obs, success, _, _, info = env.step(
|
||||
create_playwright_action('page.goto("https://ianlunn.github.io/Hover/")')
|
||||
create_playwright_action(
|
||||
'page.goto("https://ianlunn.github.io/Hover/")'
|
||||
)
|
||||
)
|
||||
assert success
|
||||
assert "link 'Download on GitHub'" in obs["text"]
|
||||
element_id = re.search(r"\[(\d+)\] link 'Download on GitHub'", obs["text"]).group(1) # type: ignore
|
||||
|
||||
obs, success, _, _, info = env.step(create_id_based_action(f"hover [{element_id}]"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_id_based_action(f"hover [{element_id}]")
|
||||
)
|
||||
assert success
|
||||
|
||||
|
||||
@ -198,13 +230,17 @@ def test_key_press(
|
||||
expect(env.page.get_by_label("Full name")).to_be_focused()
|
||||
expect(env.page.get_by_label("Full name")).to_have_value(s)
|
||||
|
||||
obs, success, _, _, info = env.step(create_id_based_action("press [meta+a]"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_id_based_action("press [meta+a]")
|
||||
)
|
||||
assert success
|
||||
|
||||
env.page.get_by_label("Full name").type(s)
|
||||
expect(env.page.get_by_label("Full name")).to_have_value(s)
|
||||
|
||||
obs, success, _, _, info = env.step(create_key_press_action("Enter"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_key_press_action("Enter")
|
||||
)
|
||||
assert success
|
||||
expect(env.page.get_by_label("Email")).to_be_focused()
|
||||
|
||||
@ -238,12 +274,18 @@ def test_e2e_id_based_actions(
|
||||
env = accessibility_tree_script_browser_env
|
||||
env.reset()
|
||||
obs, *_ = env.step(
|
||||
create_id_based_action("goto [https://russmaxdesign.github.io/exercise/]")
|
||||
create_id_based_action(
|
||||
"goto [https://russmaxdesign.github.io/exercise/]"
|
||||
)
|
||||
)
|
||||
element_id = re.search(r"\[(\d+)\] link 'What are mammals\?'", obs["text"]).group(1) # type: ignore
|
||||
obs, *_ = env.step(create_id_based_action(f"click [{element_id}]"))
|
||||
element_id = re.search(r"\[(\d+)\] textbox 'Email'", obs["text"]).group(1) # type: ignore
|
||||
env.step(create_id_based_action(f"type [{element_id}] [test@gmail.com] [0]"))
|
||||
env.step(
|
||||
create_id_based_action(
|
||||
f"type [{element_id}] [test@gmail.com] [0]"
|
||||
)
|
||||
)
|
||||
env.step(create_id_based_action("scroll [down]"))
|
||||
env.step(create_id_based_action("scroll [up]"))
|
||||
env.step(create_id_based_action("new_tab"))
|
||||
@ -254,7 +296,10 @@ def test_e2e_id_based_actions(
|
||||
x = env.step(create_id_based_action("go_forward"))
|
||||
assert x[-1]["page"].url == "https://example.com/"
|
||||
x = env.step(create_id_based_action("tab_focus [0]"))
|
||||
assert x[-1]["page"].url == "https://russmaxdesign.github.io/exercise/#link-one"
|
||||
assert (
|
||||
x[-1]["page"].url
|
||||
== "https://russmaxdesign.github.io/exercise/#link-one"
|
||||
)
|
||||
|
||||
|
||||
def test_id_delete_input(
|
||||
@ -279,13 +324,19 @@ def test_id_delete_input(
|
||||
locator = env.page.get_by_label("Full name")
|
||||
expect(locator).to_have_value(s)
|
||||
|
||||
obs, success, _, _, info = env.step(create_id_based_action(f"click [{element_id}]"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_id_based_action(f"click [{element_id}]")
|
||||
)
|
||||
assert success
|
||||
|
||||
obs, success, _, _, info = env.step(create_id_based_action(f"press [Meta+a]"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_id_based_action(f"press [Meta+a]")
|
||||
)
|
||||
assert success
|
||||
|
||||
obs, success, _, _, info = env.step(create_id_based_action("press [backspace]"))
|
||||
obs, success, _, _, info = env.step(
|
||||
create_id_based_action("press [backspace]")
|
||||
)
|
||||
assert success
|
||||
|
||||
new_s = "NEW"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user