hh&dZddlZddlZddlmZmZddlmZmZm Z m Z m Z m Z m Z ddlmZddlmZerddlmZddlmZdd lmZdd lmZmZmZejeZGd d eZGd deZGddeZdS)zQLoader that uses Playwright to load a page, then uses unstructured to parse html.N)ABCabstractmethod) TYPE_CHECKING AsyncIteratorDictIteratorListOptionalUnion)Document) BaseLoader)Browser)Page)Response)rrrc beZdZdZedddddddefd Zedd dd dd defd ZdS)PlaywrightEvaluatorzAbstract base class for all evaluators. Each evaluator should take a page, a browser instance, and a response object, process the page as necessary, and return the resulting text. pagerbrowserrresponserreturncdS)aSynchronously process the page and return the resulting text. Args: page: The page to process. browser: The browser instance. response: The response from page.goto(). Returns: text: The text content of the page. Nselfrrrs p/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/langchain_community/document_loaders/url_playwright.pyevaluatezPlaywrightEvaluator.evaluates   AsyncPage AsyncBrowser AsyncResponsec KdS)aAsynchronously process the page and return the resulting text. Args: page: The page to process. browser: The browser instance. response: The response from page.goto(). Returns: text: The text content of the page. Nrrs revaluate_asyncz"PlaywrightEvaluator.evaluate_async+s  rN)__name__ __module__ __qualname____doc__rstrrr"rrrrrs   V  i  :  RU    ^      *8  DS      ^    rrcheZdZdZddeeefdZddddd d d efd Zdd ddd dd efdZ dS)UnstructuredHtmlEvaluatorz@Evaluate the page HTML content using the `unstructured` library.Nremove_selectorscZ ddl}n#t$rtdwxYw||_dS)z%Initialize UnstructuredHtmlEvaluator.rNzQunstructured package not found, please install it with `pip install unstructured`) unstructured ImportErrorr*)rr*r,s r__init__z"UnstructuredHtmlEvaluator.__init__?sX         -   !1!rrrrrrrcNddlm}|jpgD]W}||}|D]+}|r|d,X|}||}dd|DS)z3Synchronously process the HTML content of the page.rpartition_htmlelement => element.remove()text c,g|]}t|Srr'.0els r z6UnstructuredHtmlEvaluator.evaluate..W777CGG777r unstructured.partition.htmlr2r*locatorall is_visiblercontentjoin rrrrr2selectorelementselement page_sources rrz"UnstructuredHtmlEvaluator.evaluateKs>>>>>>-3 D DH||H--1133H# D D%%''D$$%BCCC Dllnn !>{333{{77h777888rrrr cKddlm}|jpgD]i}||d{V}|D]7}|d{Vr|dd{V8j|d{V}||}dd|DS)z4Asynchronously process the HTML content of the page.rr1Nr3r4r6c,g|]}t|Srr8r9s rr<z.gr=rr>rEs rr"z(UnstructuredHtmlEvaluator.evaluate_asyncYs ?>>>>>-3 J JH!\\(337799999999H# J J ++--------J!**+HIIIIIIIII J!LLNN****** !>{333{{77h777888rN) r#r$r%r&r r r'r.rr"rrrr)r)<sJJ 1 1$s))< 1 1 1 1 9V 9i 9: 9RU 9 9 9 999*89DS9 999999rr)c eZdZdZ ddeedededeeedeed ee eefd ee ee j effd Z d eefd Zd eefdZd eefdZdS)PlaywrightURLLoaderadLoad `HTML` pages with `Playwright` and parse with `Unstructured`. This is useful for loading pages that require javascript to render. Attributes: urls (List[str]): List of URLs to load. continue_on_failure (bool): If True, continue loading other URLs on failure. headless (bool): If True, the browser will run in headless mode. proxy (Optional[Dict[str, str]]): If set, the browser will access URLs through the specified proxy. browser_session (Optional[Union[str, os.PathLike[str]]]): Path to a file with browser session data that can be used to restore the browser session. Example: .. code-block:: python from langchain_community.document_loaders import PlaywrightURLLoader urls = ["https://api.ipify.org/?format=json",] proxy={ "server": "https://xx.xx.xx:15818", # https://: "username": "username", "password": "password" } loader = PlaywrightURLLoader(urls, proxy=proxy) data = loader.load() TNurlscontinue_on_failureheadlessr* evaluatorproxybrowser_sessionc ddl}n#t$rtdwxYw||_||_||_||_||_|r|rtd|pt||_ dS)z%Load a list of URLs using Playwright.rNzMplaywright package not found, please install it with `pip install playwright`z:`remove_selectors` and `evaluator` cannot be both not None) playwrightr-rOrPrQrSrT ValueErrorr)rR) rrOrPrQr*rRrSrTrVs rr.zPlaywrightURLLoader.__init__s         +    #6    .    L  #Q&?@P&Q&Qr/rc #Kddlm}|5}|j|j|j}d}|jrbtj |jr| |j}n"t d|j|| }|j D]} |}||}|t!d||d|j|||}|d |i} t+|| V#t,$r4} |jr!td |d | n| Yd} ~ d} ~ wwxYw|ddddS#1swxYwYdS) zLoad the specified URLs using Playwright and create Document instances. Returns: A list of Document instances with loaded content. r)sync_playwrightrQrSN storage_stateSession file not found: "page.goto() returned None for url loadsource page_contentmetadataError fetching or processing , exception: )playwright.sync_apirYchromiumlaunchrQrSrTospathexists new_contextloggerwarningrOnew_pagegotorWwait_for_load_staterRrcloser ExceptionrPerror) rrYprcontexturlrrr5rces r lazy_loadzPlaywrightURLLoader.lazy_loads[ 877777 _  ! !j''dj'QQGG# V7>>$"677V%11@T1UUGGNN#Td>R#T#TUUU!--//y   "++--D#yy~~H'()Sc)S)STTT,,V444>224(KKDJJLLL (#H"xHHHHHHH    /  QCQQaQQ    MMOOOC! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! s=B1GBEG F)*FGFGGGcNKd|2d{VS)Load the specified URLs with Playwright and create Documents asynchronously. Use this function when in a jupyter notebook environment. Returns: A list of Document instances with loaded content. c"Kg|3d{V}| 6SrLr)r:docs rr<z-PlaywrightURLLoader.aload..s.77777777c7777sN) alazy_load)rs raloadzPlaywrightURLLoader.aloads887T__%6%67777777777rc .Kddlm}|4d{V}|j|j|jd{V}d}|jrhtj |jr"| |jd{V}n"t d|j|| d{V}|j D]} |d{V}||d{V}|t!d||dd{V|j|||d{V}|d{Vd |i} t+|| WV#t,$r4} |jr!td |d | n| Yd} ~ d} ~ wwxYw|d{Vdddd{VdS#1d{VswxYwYdS) r{r)async_playwrightNrZr[r]r^r_r`rardre)playwright.async_apirrgrhrQrSrTrirjrkrlrmrnrOrorprWrqrRr"rrr rsrPrt) rrrurrvrwrrr5rcrxs rr~zPlaywrightURLLoader.alazy_loadsr :99999##%%# "# "# "# "# "# "# "J--t}DJ-WWWWWWWWGG# V7>>$"677V$+$7$7&*&:%8%%GGNN#Td>R#T#TUUU ' 3 3 5 5555555y   !(!1!1!3!3333333D%)YYs^^333333H'()Sc)S)STTT226:::::::::!%!>!>tWh!W!WWWWWWWD**,,&&&&&&& (#H"xHHHHHHHH    /  QCQQaQQ   --// ! ! ! ! ! ! !G# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "# "s=CHB8FH G *G HGH HH)TTNNNN)r#r$r%r&r r'boolr rrr riPathLiker.rr ryrrr~rrrrNrNjs2>%)0437*.BFRR3iR"R R #49- R /0 RS#X'R"%R[-=(=">?RRRRB)8H-))))V8T(^8888,"-"9,",",",",","rrN) r&loggingriabcrrtypingrrrrr r r langchain_core.documentsr )langchain_community.document_loaders.baser rrrrrrr rf getLoggerr#rmrr)rNrrrrsWW ########VVVVVVVVVVVVVVVVVV------@@@@@@<<<<<<<666666>>>>>>;;;;;;;;;;  8 $ $# # # # # ## # # L+9+9+9+9+9 3+9+9+9\^"^"^"^"^"*^"^"^"^"^"r