proxy

proxy#

  • __all__ = ['ProxyWebScraperDriver'] module-attribute

Bases: BaseWebScraperDriver “)

Source Code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py

@define
class ProxyWebScraperDriver(BaseWebScraperDriver):
    proxies: dict = field(kw_only=True, metadata={"serializable": False})
    params: dict = field(default=Factory(dict), kw_only=True, metadata={"serializable": True})

    def fetch_url(self, url: str) -> str:
        response = requests.get(url, proxies=self.proxies, **self.params)

        return response.text

    def extract_page(self, page: str) -> TextArtifact:
        return TextArtifact(page)

  • params = field(default=Factory(dict), kw_only=True, metadata={'serializable': True}) class-attribute instance-attribute

  • proxies = field(kw_only=True, metadata={'serializable': False}) class-attribute instance-attribute

extract_page(page)#

Source Code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py

def extract_page(self, page: str) -> TextArtifact:
    return TextArtifact(page)

fetch_url(url)#

Source Code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py

def fetch_url(self, url: str) -> str:
    response = requests.get(url, proxies=self.proxies, **self.params)

    return response.text
Contents