
    (Gg	                         d Z ddlZddlmZmZmZ ddlmZ ddlm	Z	 ddl
mZ  ej                  e      Z G d de      Zy)	zScrapingAnt Web Extractor.    N)IteratorListOptional)
BaseLoader)Document)get_from_envc                   Z    e Zd ZdZdddddee   dee   dee   ded	df
d
Z	d	e
e   fdZy)ScrapingAntLoaderzTurn an url to LLM accessible markdown with `ScrapingAnt`.

    For further details, visit: https://docs.scrapingant.com/python-client
    NT)api_keyscrape_configcontinue_on_failureurlsr   r   r   returnc                    	 ddl m} |st        d      |xs t	        dd      } ||      | _        || _        || _        || _        y# t        $ r t        d      w xY w)	at  Initialize client.

        Args:
            urls: List of urls to scrape.
            api_key: The ScrapingAnt API key. If not specified must have env var
                SCRAPINGANT_API_KEY set.
            scrape_config: The scraping config from ScrapingAntClient.markdown_request
            continue_on_failure: Whether to continue if scraping an url fails.
        r   )ScrapingAntClientzL`scrapingant-client` package not found, run `pip install scrapingant-client`zURLs must be provided.r   SCRAPINGANT_API_KEY)tokenN)	scrapingant_clientr   ImportError
ValueErrorr   clientr   r   r   )selfr   r   r   r   r   s         /home/kushmeetdev/apache_webroot/langgraph_flaskproject/venv/lib/python3.12/site-packages/langchain_community/document_loaders/scrapingant.py__init__zScrapingAntLoader.__init__   ss    "	< 566K\)5JK'g6	*#6   	8 	s   A	 	Ac              #   j  K   | j                   | j                   ni }| j                  D ]F  }	  | j                  j                  dd|i|}t	        |j
                  d|j                  i       H y# t        $ r4}| j                  rt        j                  d| d|        n|Y d}~d}~ww xY ww)zFetch data from ScrapingAnt.Nurl)page_contentmetadatazError fetching data from z, exception:  )r   r   r   markdown_requestr   markdownr   	Exceptionr   loggererror)r   r   r   resultes        r   	lazy_loadzScrapingAntLoader.lazy_load3   s      /3.@.@.L**RT99 	C
555O#OO!'#VZZ0 	  ++LL#<SEqc!RSG Ts/   *B3AA30B33	B0<*B+&B3+B00B3)__name__
__module____qualname____doc__r   strr   dictboolr   r   r   r'   r       r   r
   r
      se     "&(,$(73i7 #	7
  ~7 "7 
7@8H- r/   r
   )r+   loggingtypingr   r   r   langchain_core.document_loadersr   langchain_core.documentsr   langchain_core.utilsr   	getLogger__file__r#   r
   r   r/   r   <module>r7      s8       + + 6 - -			8	$5
 5r/   