[
{
"path": "LICENSE",
"content": "MIT License\n\nCopyright (c) 2023 gravelBridge\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
},
{
"path": "README.md",
"content": "# AutoGPT Web Interaction Plugin\n\n\n\nThe AutoGPT Web Interaction Plugin enables Auto-GPT to interact with websites.\n\nNote: The plugin is very flakey on GPT-3.5, I recommend using GPT-4. However, it can still perform basic tasks on GPT-3.5.\n\n## Key Features:\n- Allows Auto-GPT to click elements.\n- Allows Auto-GPT to type text.\n- Allows Auto-GPT select elements.\n- Allows Auto-GPT to scroll\n\n## Installation\n\nFollow these steps to configure the Auto-GPT Email Plugin:\n\n### 1. Clone this repository.\n\n### 2. cd into the directory, and run pip install -r requirements.txt\n\n### 3. pip install playwright\n\n### 3. Zip/Compress the web_interaction folder\n\n### 4. Drag the new zip file into the Auto-GPT plugins folder.\n\n### 5. Set `ALLOWLISTED_PLUGINS=AutoGPTWebInteraction,example-plugin1,example-plugin2,etc` in your AutoGPT `.env` file.\n\n### 6. Edit goals\nWhen using Auto-GPT please set one of the goals to \"Remember to use the Web Interaction Plugin possible\".\n"
},
{
"path": "requirements.txt",
"content": "playwright\ncolorama\ntyping\n"
},
{
"path": "web_interaction/__init__.py",
"content": "\"\"\"This is the web interaction plugin for Auto-GPT.\"\"\"\nfrom typing import Any, Dict, List, Optional, Tuple, TypeVar, TypedDict\nfrom auto_gpt_plugin_template import AutoGPTPluginTemplate\nfrom colorama import Fore\n\nPromptGenerator = TypeVar(\"PromptGenerator\")\n\n\nclass Message(TypedDict):\n role: str\n content: str\n\n\nclass AutoGPTWebInteraction(AutoGPTPluginTemplate):\n \"\"\"\n This is the Auto-GPT web interaction plugin.\n \"\"\"\n\n def __init__(self):\n super().__init__()\n self._name = \"Auto-Web-Interaction-Plugin\"\n self._version = \"0.1.0\"\n self._description = \"Auto-GPT Web Interaction Plugin: Interact with websites.\"\n\n def post_prompt(self, prompt: PromptGenerator) -> PromptGenerator:\n from .web_interaction import (\n start_browser,\n go_to_page,\n scroll,\n click,\n type,\n enter,\n crawl,\n type_and_enter,\n get_current_url,\n )\n\n prompt.add_resource(\"\"\"\n Ability to interact with websites via the web_interaction plugin. Information for the web_interaction plugin: \n The format of the browser content is highly simplified; all formatting elements are stripped.\n Interactive elements such as links, inputs, buttons are represented like this:\n\n\t\t text\n\t\t \n\t\t text\n\n Images are rendered as their alt text like this:\n\n\t\t \n\n Don't try to interact with elements that you can't see.\n\n CRITICAL: The id parameter specified in , text, etc.. MUST be used for all web_interaction commands that require an id.\n CRITICAL: Use the command get_dom every time before executing any web_interaction plugin command.\n CRITICAL: When trying to search something on Google, don't call the dom function. Instead, just use the id value of 3\n \"\"\")\n\n prompt.add_command(\"start_browser\", \"Starts the browser for web interaction. Must be ran before attempting to perform any other web interaction plugins.\", {}, start_browser)\n\n prompt.add_command(\"go_to_website\", \"Goes to a website in the web interaction plugin. Must be ran after starting the browser and before attempting to interact with a website.\", {\"url\":\"\"}, go_to_page)\n\n prompt.add_command(\"get_dom\", \"Returns a simplified DOM of the current web page. The id is specific to this plugin and will be needed to interact with elements. Make sure to run this before interacting with any elements on a webpage. Re-run this each time you're on a new webpage and want to interact with elements.\", {}, crawl)\n\n prompt.add_command(\"click_element_by_id\", \"Clicks an element. Specify the id with the unique id received from the get_dom command. CRITICAL: The ID must be the integer id from the get_dom command.\", {\"id\":\"\"}, click)\n\n prompt.add_command(\"input_text_by_id\", \"Inputs text to an element. Specify the id with the unique id received from the get_dom command. CRITICAL: The ID must be the integer id from the get_dom command.\", {\"id\":\"\", \"text\":\"\"}, type)\n\n prompt.add_command(\"input_text_by_id_and_press_enter\", \"Inputs text to an element. Specify the id with the unique id received from the get_dom command. Also presses enter after finishing inputting text. CRITICAL: The ID must be the integer id from the get_dom command.\", {\"id\":\"\", \"text\":\"\"}, type_and_enter)\n\n prompt.add_command(\"scroll\", \"Scrolls the current website up or down one page. In the arguments, use either \\\"up\\\" or \\\"down\\\"\", {\"direction\":\"\"}, scroll)\n\n prompt.add_command(\"enter\", \"Presses enter on the keyboard on the current website.\", {}, enter)\n\n prompt.add_command(\"get_url\", \"Retrieves the current url that the web_interaction plugin is on.\", {}, get_current_url)\n\n return prompt\n\n\n def can_handle_post_prompt(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the post_prompt method.\n\n Returns:\n bool: True if the plugin can handle the post_prompt method.\"\"\"\n return True\n\n def can_handle_on_response(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the on_response method.\n\n Returns:\n bool: True if the plugin can handle the on_response method.\"\"\"\n return False\n\n def on_response(self, response: str, *args, **kwargs) -> str:\n \"\"\"This method is called when a response is received from the model.\"\"\"\n pass\n\n def can_handle_on_planning(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the on_planning method.\n\n Returns:\n bool: True if the plugin can handle the on_planning method.\"\"\"\n return False\n\n def on_planning(\n self, prompt: PromptGenerator, messages: List[Message]\n ) -> Optional[str]:\n \"\"\"This method is called before the planning chat completion is done.\n\n Args:\n prompt (PromptGenerator): The prompt generator.\n messages (List[str]): The list of messages.\n \"\"\"\n pass\n\n def can_handle_post_planning(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the post_planning method.\n\n Returns:\n bool: True if the plugin can handle the post_planning method.\"\"\"\n return False\n\n def post_planning(self, response: str) -> str:\n \"\"\"This method is called after the planning chat completion is done.\n\n Args:\n response (str): The response.\n\n Returns:\n str: The resulting response.\n \"\"\"\n pass\n\n def can_handle_pre_instruction(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the pre_instruction method.\n\n Returns:\n bool: True if the plugin can handle the pre_instruction method.\"\"\"\n return False\n\n def pre_instruction(self, messages: List[Message]) -> List[Message]:\n \"\"\"This method is called before the instruction chat is done.\n\n Args:\n messages (List[Message]): The list of context messages.\n\n Returns:\n List[Message]: The resulting list of messages.\n \"\"\"\n pass\n\n def can_handle_on_instruction(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the on_instruction method.\n\n Returns:\n bool: True if the plugin can handle the on_instruction method.\"\"\"\n return False\n\n def on_instruction(self, messages: List[Message]) -> Optional[str]:\n \"\"\"This method is called when the instruction chat is done.\n\n Args:\n messages (List[Message]): The list of context messages.\n\n Returns:\n Optional[str]: The resulting message.\n \"\"\"\n pass\n\n def can_handle_post_instruction(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the post_instruction method.\n\n Returns:\n bool: True if the plugin can handle the post_instruction method.\"\"\"\n return False\n\n def post_instruction(self, response: str) -> str:\n \"\"\"This method is called after the instruction chat is done.\n\n Args:\n response (str): The response.\n\n Returns:\n str: The resulting response.\n \"\"\"\n pass\n\n def can_handle_pre_command(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the pre_command method.\n\n Returns:\n bool: True if the plugin can handle the pre_command method.\"\"\"\n return False\n\n def pre_command(\n self, command_name: str, arguments: Dict[str, Any]\n ) -> Tuple[str, Dict[str, Any]]:\n \"\"\"This method is called before the command is executed.\n\n Args:\n command_name (str): The command name.\n arguments (Dict[str, Any]): The arguments.\n\n Returns:\n Tuple[str, Dict[str, Any]]: The command name and the arguments.\n \"\"\"\n pass\n\n def can_handle_post_command(self) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the post_command method.\n\n Returns:\n bool: True if the plugin can handle the post_command method.\"\"\"\n return False\n\n def post_command(self, command_name: str, response: str) -> str:\n \"\"\"This method is called after the command is executed.\n\n Args:\n command_name (str): The command name.\n response (str): The response.\n\n Returns:\n str: The resulting response.\n \"\"\"\n pass\n\n def can_handle_chat_completion(\n self, messages: Dict[Any, Any], model: str, temperature: float, max_tokens: int\n ) -> bool:\n \"\"\"This method is called to check that the plugin can\n handle the chat_completion method.\n\n Args:\n messages (List[Message]): The messages.\n model (str): The model name.\n temperature (float): The temperature.\n max_tokens (int): The max tokens.\n\n Returns:\n bool: True if the plugin can handle the chat_completion method.\"\"\"\n return False\n\n def handle_chat_completion(\n self, messages: List[Message], model: str, temperature: float, max_tokens: int\n ) -> str:\n \"\"\"This method is called when the chat completion is done.\n\n Args:\n messages (List[Message]): The messages.\n model (str): The model name.\n temperature (float): The temperature.\n max_tokens (int): The max tokens.\n\n Returns:\n str: The resulting response.\n \"\"\"\n pass\n \n def can_handle_text_embedding(\n self, text: str\n ) -> bool:\n return False\n\n def handle_text_embedding(\n self, text: str\n ) -> list:\n pass\n\n def can_handle_user_input(self, user_input: str) -> bool:\n return False\n\n def user_input(self, user_input: str) -> str:\n return user_input\n\n def can_handle_report(self) -> bool:\n return False\n\n def report(self, message: str) -> None:\n pass\n\n"
},
{
"path": "web_interaction/web_interaction.py",
"content": "from playwright.sync_api import sync_playwright\nfrom sys import argv, exit, platform\n\nblack_listed_elements = set([\"html\", \"head\", \"title\", \"meta\", \"iframe\", \"body\", \"style\", \"script\", \"path\", \"svg\", \"br\", \"::marker\",])\n\n\ndef start_browser():\n global browser\n global page\n\n browser = (\n sync_playwright()\n .start()\n .chromium.launch(\n headless=False,\n )\n )\n\n page = browser.new_page()\n page.set_viewport_size({\"width\": 1280, \"height\": 1080})\n\n return \"Browser successfully started!\"\n\n\ndef go_to_page(url):\n global client\n global page_element_buffer\n\n try:\n page.goto(url=url if \"://\" in url else \"http://\" + url)\n client = page.context.new_cdp_session(page)\n page_element_buffer = {}\n except:\n return \"Failed to go to url, please try again and make sure the url is correct.\"\n\n return \"Now on url \" + url\n\ndef scroll(direction):\n if direction == \"up\":\n page.evaluate(\n \"(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop - window.innerHeight;\"\n )\n return \"Scrolled!\"\n elif direction == \"down\":\n page.evaluate(\n \"(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop + window.innerHeight;\"\n )\n\n return \"Scrolled!\"\n\n else:\n return \"Scroll direction invalid.\"\n\ndef click(id):\n # Inject javascript into the page which removes the target= attribute from all links\n js = \"\"\"\n links = document.getElementsByTagName(\"a\");\n for (var i = 0; i < links.length; i++) {\n links[i].removeAttribute(\"target\");\n }\n \"\"\"\n page.evaluate(js)\n\n element = page_element_buffer.get(int(id))\n if element:\n x = element.get(\"center_x\")\n y = element.get(\"center_y\")\n \n page.mouse.click(x, y)\n else:\n return \"Could not find element\"\n\n return \"Successfully clicked!\"\n\ndef type(id, text):\n click(int(id))\n page.keyboard.type(text)\n\n return \"Typed \" + text + \" into \" + id\n\ndef enter():\n page.keyboard.press(\"Enter\")\n\n return \"Pressed enter!\"\n\ndef type_and_enter(id, text):\n click(int(id))\n page.keyboard.type(text)\n page.keyboard.press(\"Enter\")\n\n return \"Inputted text, and pressed enter!\"\n\ndef get_current_url():\n try:\n current_url = page.url\n return current_url\n except:\n return \"Error retrieving current URL.\"\n\ndef crawl():\n page_state_as_text = []\n\n device_pixel_ratio = page.evaluate(\"window.devicePixelRatio\")\n if platform == \"darwin\" and device_pixel_ratio == 1: # lies\n device_pixel_ratio = 2\n\n win_scroll_x = page.evaluate(\"window.scrollX\")\n win_scroll_y = page.evaluate(\"window.scrollY\")\n win_upper_bound = page.evaluate(\"window.pageYOffset\")\n win_left_bound = page.evaluate(\"window.pageXOffset\") \n win_width = page.evaluate(\"window.screen.width\")\n win_height = page.evaluate(\"window.screen.height\")\n win_right_bound = win_left_bound + win_width\n win_lower_bound = win_upper_bound + win_height\n document_offset_height = page.evaluate(\"document.body.offsetHeight\")\n document_scroll_height = page.evaluate(\"document.body.scrollHeight\")\n\n# percentage_progress_start = (win_upper_bound / document_scroll_height) * 100\n# percentage_progress_end = (\n# (win_height + win_upper_bound) / document_scroll_height\n# ) * 100\n percentage_progress_start = 1\n percentage_progress_end = 2\n\n page_state_as_text.append(\n {\n \"x\": 0,\n \"y\": 0,\n \"text\": \"[scrollbar {:0.2f}-{:0.2f}%]\".format(\n round(percentage_progress_start, 2), round(percentage_progress_end)\n ),\n }\n )\n\n tree = client.send(\n \"DOMSnapshot.captureSnapshot\",\n {\"computedStyles\": [], \"includeDOMRects\": True, \"includePaintOrder\": True},\n )\n strings = tree[\"strings\"]\n document = tree[\"documents\"][0]\n nodes = document[\"nodes\"]\n backend_node_id = nodes[\"backendNodeId\"]\n attributes = nodes[\"attributes\"]\n node_value = nodes[\"nodeValue\"]\n parent = nodes[\"parentIndex\"]\n node_types = nodes[\"nodeType\"]\n node_names = nodes[\"nodeName\"]\n is_clickable = set(nodes[\"isClickable\"][\"index\"])\n\n text_value = nodes[\"textValue\"]\n text_value_index = text_value[\"index\"]\n text_value_values = text_value[\"value\"]\n\n input_value = nodes[\"inputValue\"]\n input_value_index = input_value[\"index\"]\n input_value_values = input_value[\"value\"]\n\n input_checked = nodes[\"inputChecked\"]\n layout = document[\"layout\"]\n layout_node_index = layout[\"nodeIndex\"]\n bounds = layout[\"bounds\"]\n\n cursor = 0\n html_elements_text = []\n\n child_nodes = {}\n elements_in_view_port = []\n\n anchor_ancestry = {\"-1\": (False, None)}\n button_ancestry = {\"-1\": (False, None)}\n\n def convert_name(node_name, has_click_handler):\n if node_name == \"a\":\n return \"link\"\n if node_name == \"input\" or node_name == \"textarea\":\n return \"input\"\n if node_name == \"img\":\n return \"img\"\n if (\n node_name == \"button\" or has_click_handler\n ): # found pages that needed this quirk\n return \"button\"\n else:\n return \"text\"\n\n def find_attributes(attributes, keys):\n values = {}\n\n for [key_index, value_index] in zip(*(iter(attributes),) * 2):\n if value_index < 0:\n continue\n key = strings[key_index]\n value = strings[value_index]\n\n if key in keys:\n values[key] = value\n keys.remove(key)\n\n if not keys:\n return values\n\n return values\n\n def add_to_hash_tree(hash_tree, tag, node_id, node_name, parent_id):\n parent_id_str = str(parent_id)\n if not parent_id_str in hash_tree:\n parent_name = strings[node_names[parent_id]].lower()\n grand_parent_id = parent[parent_id]\n\n add_to_hash_tree(\n hash_tree, tag, parent_id, parent_name, grand_parent_id\n )\n\n is_parent_desc_anchor, anchor_id = hash_tree[parent_id_str]\n\n # even if the anchor is nested in another anchor, we set the \"root\" for all descendants to be ::Self\n if node_name == tag:\n value = (True, node_id)\n elif (\n is_parent_desc_anchor\n ): # reuse the parent's anchor_id (which could be much higher in the tree)\n value = (True, anchor_id)\n else:\n value = (\n False,\n None,\n ) # not a descendant of an anchor, most likely it will become text, an interactive element or discarded\n\n hash_tree[str(node_id)] = value\n\n return value\n\n for index, node_name_index in enumerate(node_names):\n node_parent = parent[index]\n node_name = strings[node_name_index].lower()\n\n is_ancestor_of_anchor, anchor_id = add_to_hash_tree(\n anchor_ancestry, \"a\", index, node_name, node_parent\n )\n\n is_ancestor_of_button, button_id = add_to_hash_tree(\n button_ancestry, \"button\", index, node_name, node_parent\n )\n\n try:\n cursor = layout_node_index.index(\n index\n ) # todo replace this with proper cursoring, ignoring the fact this is O(n^2) for the moment\n except:\n continue\n\n if node_name in black_listed_elements:\n continue\n\n [x, y, width, height] = bounds[cursor]\n x /= device_pixel_ratio\n y /= device_pixel_ratio\n width /= device_pixel_ratio\n height /= device_pixel_ratio\n\n elem_left_bound = x\n elem_top_bound = y\n elem_right_bound = x + width\n elem_lower_bound = y + height\n\n partially_is_in_viewport = (\n elem_left_bound < win_right_bound\n and elem_right_bound >= win_left_bound\n and elem_top_bound < win_lower_bound\n and elem_lower_bound >= win_upper_bound\n )\n\n if not partially_is_in_viewport:\n continue\n\n meta_data = []\n\n # inefficient to grab the same set of keys for kinds of objects but its fine for now\n element_attributes = find_attributes(\n attributes[index], [\"type\", \"placeholder\", \"aria-label\", \"title\", \"alt\"]\n )\n\n ancestor_exception = is_ancestor_of_anchor or is_ancestor_of_button\n ancestor_node_key = (\n None\n if not ancestor_exception\n else str(anchor_id)\n if is_ancestor_of_anchor\n else str(button_id)\n )\n ancestor_node = (\n None\n if not ancestor_exception\n else child_nodes.setdefault(str(ancestor_node_key), [])\n )\n\n if node_name == \"#text\" and ancestor_exception:\n text = strings[node_value[index]]\n if text == \"|\" or text == \"•\":\n continue\n ancestor_node.append({\n \"type\": \"type\", \"value\": text\n })\n else:\n if (\n node_name == \"input\" and element_attributes.get(\"type\") == \"submit\"\n ) or node_name == \"button\":\n node_name = \"button\"\n element_attributes.pop(\n \"type\", None\n ) # prevent [button ... (button)..]\n \n for key in element_attributes:\n if ancestor_exception:\n ancestor_node.append({\n \"type\": \"attribute\",\n \"key\": key,\n \"value\": element_attributes[key]\n })\n else:\n meta_data.append(element_attributes[key])\n\n element_node_value = None\n\n if node_value[index] >= 0:\n element_node_value = strings[node_value[index]]\n if element_node_value == \"|\": #commonly used as a seperator, does not add much context - lets save ourselves some token space\n continue\n elif (\n node_name == \"input\"\n and index in input_value_index\n and element_node_value is None\n ):\n node_input_text_index = input_value_index.index(index)\n text_index = input_value_values[node_input_text_index]\n if node_input_text_index >= 0 and text_index >= 0:\n element_node_value = strings[text_index]\n\n # remove redudant elements\n if ancestor_exception and (node_name != \"a\" and node_name != \"button\"):\n continue\n\n elements_in_view_port.append(\n {\n \"node_index\": str(index),\n \"backend_node_id\": backend_node_id[index],\n \"node_name\": node_name,\n \"node_value\": element_node_value,\n \"node_meta\": meta_data,\n \"is_clickable\": index in is_clickable,\n \"origin_x\": int(x),\n \"origin_y\": int(y),\n \"center_x\": int(x + (width / 2)),\n \"center_y\": int(y + (height / 2)),\n }\n )\n\n # lets filter further to remove anything that does not hold any text nor has click handlers + merge text from leaf#text nodes with the parent\n elements_of_interest= []\n id_counter = 0\n\n for element in elements_in_view_port:\n node_index = element.get(\"node_index\")\n node_name = element.get(\"node_name\")\n node_value = element.get(\"node_value\")\n is_clickable = element.get(\"is_clickable\")\n origin_x = element.get(\"origin_x\")\n origin_y = element.get(\"origin_y\")\n center_x = element.get(\"center_x\")\n center_y = element.get(\"center_y\")\n meta_data = element.get(\"node_meta\")\n\n inner_text = f\"{node_value} \" if node_value else \"\"\n meta = \"\"\n \n if node_index in child_nodes:\n for child in child_nodes.get(node_index):\n entry_type = child.get('type')\n entry_value= child.get('value')\n\n if entry_type == \"attribute\":\n entry_key = child.get('key')\n meta_data.append(f'{entry_key}=\"{entry_value}\"')\n else:\n inner_text += f\"{entry_value} \"\n\n if meta_data:\n meta_string = \" \".join(meta_data)\n meta = f\" {meta_string}\"\n\n if inner_text != \"\":\n inner_text = f\"{inner_text.strip()}\"\n\n converted_node_name = convert_name(node_name, is_clickable)\n\n # not very elegant, more like a placeholder\n if (\n (converted_node_name != \"button\" or meta == \"\")\n and converted_node_name != \"link\"\n and converted_node_name != \"input\"\n and converted_node_name != \"img\"\n and converted_node_name != \"textarea\"\n ) and inner_text.strip() == \"\":\n continue\n\n page_element_buffer[id_counter] = element\n\n if inner_text != \"\": \n elements_of_interest.append(\n f\"\"\"<{converted_node_name} id={id_counter}{meta}>{inner_text}\"\"\"\n )\n else:\n elements_of_interest.append(\n f\"\"\"<{converted_node_name} id={id_counter}{meta}/>\"\"\"\n )\n id_counter += 1\n\n if len(elements_of_interest) > 125:\n idCounter =0\n divided_elements_of_interest = []\n while idCounter <= 125:\n divided_elements_of_interest.append(elements_of_interest[idCounter])\n idCounter+=1\n\n return repr(divided_elements_of_interest) + \"This is not part of the DOM, note that not the entire DOM was returned as it exceeded the context limit. If you're sure what you're need is not included in this DOM, you should find a workaround.\"\n\n return repr(elements_of_interest)\n"
}
]