""" title: Website Summarizer description: A tool that returns a summary of the content of a passed in URL. author: ekatiyar author_url: https://github.com/ekatiyar github: https://github.com/ekatiyar/open-webui-tools funding_url: https://github.com/open-webui version: 0.0.1 license: MIT """ import requests from bs4 import BeautifulSoup from typing import Callable, Any import unittest class EventEmitter: def __init__(self, event_emitter: Callable[[dict], Any] = None): self.event_emitter = event_emitter async def progress_update(self, description): await self.emit(description) async def error_update(self, description): await self.emit(description, "error", True) async def success_update(self, description): await self.emit(description, "success", True) async def emit(self, description="Unknown State", status="in_progress", done=False): if self.event_emitter: await self.event_emitter( { "type": "status", "data": { "status": status, "description": description, "done": done, }, } ) class Tools: def __init__(self): self.citation = True async def get_website_summary( self, url: str, __event_emitter__: Callable[[dict], Any] = None ) -> str: """ Provides a summary of the content of a website. Only use if the user supplied a valid URL. :param url: The URL of the website that you want the summary for. :return: A summary of the website content, or an error message. """ emitter = EventEmitter(__event_emitter__) try: await emitter.progress_update(f"Getting content for {url}") error_message = f"Error: Invalid URL: {url}" if not url or url == "": await emitter.error_update(error_message) return error_message response = requests.get(url) if response.status_code != 200: error_message = ( f"Error: Received status code {response.status_code} for URL: {url}" ) await emitter.error_update(error_message) return error_message soup = BeautifulSoup(response.content, "html.parser") paragraphs = soup.find_all("p") content = "\n".join([para.get_text() for para in paragraphs]) if len(content) == 0: error_message = f"Error: Failed to find content for {url}" await emitter.error_update(error_message) return error_message await emitter.success_update(f"Content for {url} retrieved!") return f"Content:\n{content}" except Exception as e: error_message = f"Error: {str(e)}" await emitter.error_update(error_message) return error_message class WebsiteSummarizerTest(unittest.IsolatedAsyncioTestCase): async def assert_content_length(self, url: str, expected_length: int): self.assertEqual(len(await Tools().get_website_summary(url)), expected_length) async def assert_content_error(self, url: str): response = await Tools().get_website_summary(url) self.assertTrue("Error" in response) async def test_get_website_summary(self): url = "https://docs.openwebui.com/features/plugin/tools/" await self.assert_content_length(url, 3812) # Updated expected length async def test_get_website_summary_with_invalid_url(self): invalid_url = "https://www.invalidurl.com" await self.assert_content_error(invalid_url) async def test_get_website_summary_with_none_arg(self): await self.assert_content_error(None) await self.assert_content_error("") if __name__ == "__main__": print("Running tests...") unittest.main()