'GgaddlZddlZddlZddlmZmZmZddlmZddl m Z m Z m Z m Z mZmZmZddlZddlZddlmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z0m1Z1dd l2m3Z3dd l4m5Z5edZ6e e7ee8efZ9e e)e efZ:ee#jve jxZ=GddZ>Gdde,Z?e@dk(rddlAmBZBeBeeCjjdz dz ZEe?ZF eFjdZHeHjeEdz d z ZLeFjeHjeOeLyy#eI$reFjddddiddiddZHY_wxYw)!N)BufferedIOBaseBufferedReaderBytesIO)Path)ListOptionalTypeUnion CoroutineAnyTypeVar) BaseModel) ExtractAgentExtractAgentCreate ExtractConfig ExtractJobExtractJobCreate ExtractRunExtractSchemaValidateRequestExtractAgentUpdateFile ExtractMode StatusEnumProject ExtractTargetLlamaExtractSettingsPaginatedExtractRunsResponse)AsyncLlamaCloud)JSONObjectTypeaugment_async_errors) BaseComponent)run_jobs)Field PrivateAttr)DEFAULT_BASE_URL)ThreadPoolExecutorT)extraction_targetextraction_modeceZdZdZ d-dededeedeededed ed e d e d ee d ee fdZ de e e efdefdZedefdZedefdZedefdZej*deddfdZedefdZej*deddfdZdedefdZdedeefdZd.dZdeee efd e!dee"e e"ffd!Z#deee efdee"e e"ffd"Z$deee efdeee effd#Z%deee efdeee effd$Z&dede"fd%Z'dedefd&Z(d'eddfd(Z) d/d)ed*ede*fd+Z+defd,Z,y)0ExtractionAgentzTClass representing a single extraction agent with methods for extraction operations.Nclientagent project_idorganization_idcheck_interval max_timeout num_workers show_progressverboseverify httpx_timeoutc "||_||_||_||_||_||_||_||_| |_| |_ | |_ d|_ d|_ ttdtj xsddz|_y)N  max_workers)_client_agent _project_id_organization_idr0r1r2r3r5r6_verbose _data_schema_configr&minos cpu_count _thread_pool) selfr,r-r.r/r0r1r2r3r4r5r6s /home/kushmeetdev/apache_webroot/langgraph_flaskproject/venv/lib/python3.12/site-packages/llama_cloud_services/extract/extract.py__init__zExtractionAgent.__init__1s  % /,&&* * 9=37 .B!41 9: cororeturncrdtffd }jj|jS)=Run coroutine in a separate thread to avoid event loop issuesrMcRdtffd }tj|S)NrMcKjjj}tjj j 4d{}|jj_ d{|jj_cdddd{S7V727 #|jj_wxYw#1d{7swYyxYww)Nr5timeout)r=_client_wrapper httpx_clienthttpx AsyncClientr5r6)original_clientr,rLrHs rI wrapped_corozFExtractionAgent._run_in_thread..run_coro..wrapped_coroTs"&,,">">"K"K!,,;; .. T T@FDLL00=T%)zET 44A T T T * TET 44A T T TsrAC%B*C%C4B09B,:B0=C C%$B.%C%,B0.C%0C  CC"C C"C%r'asynciorunrYrLrHs rIrun_coroz0ExtractionAgent._run_in_thread..run_coroSs$ T T";;|~. .rKr'rGsubmitresultrHrLr^s`` rI_run_in_threadzExtractionAgent._run_in_threadPs1 /! /(  ''188::rKc.|jjSN)r>idrHs rIrfzExtractionAgent.idis{{~~rKc.|jjSre)r>namergs rIrizExtractionAgent.namems{{rKc^|js|jjS|jSre)rBr> data_schemargs rIrkzExtractionAgent.data_schemaqs%.2.?.?t{{&&VTEVEVVrKrkcBt|tr|}n M  .. LL & & A A4AQR B   -88rKc^|js|jjS|jSre)rCr>configrgs rIr{zExtractionAgent.configs!)-t{{!!G4<<GrKr{c||_yre)rC)rHr{s rIr{zExtractionAgent.configs  rK file_inputcKt|tr|}nJt|tr t|}n.t|tt fr t |d}n td |jjj|j|d{t|tr|jSS7%#t|tr|jwwxYww)zUpload a file for extraction.rbzJfile_input must be either a file path string, file bytes, or buffer object)r. upload_fileN)rprbytesrstrropenrur=filesrr?rclose)rHr}rs rI _upload_filezExtractionAgent._upload_files j. 1$K  E *!*-K  S$K 0z40K\  $++77++8+~6!!#7 +~6!!#7s0AC#!4B=B;B="C#;B==#C  C#job_idc@Ktj}d} tj|jd{|dz }|j j j|d{}|jtjk(r.|j j j|d{S|jtjk(rXtj}||z |jkDrtd||jr|dzdk(rt!dd d t#j$d |d |jd |j&|j j j|d{S7X7(77 w)z5Wait for and return the results of an extraction job.rTNr9rz#Timeout while extracting the file: r8.)endflushzFailure in job: z , status: z , error: )time perf_counterr[sleepr0r=rvget_jobstatusrSUCCESSget_run_by_job_idPENDINGr1 ExceptionrAprintwarningswarnerror)rHrstarttriesjobrs rI_wait_for_job_resultz$ExtractionAgent._wait_for_job_resultsx!!#-- 3 34 4 4 QJE 22::;CzzZ///!\\77II!Jz111'');!1!11#&I&$RSS==URZ1_#2T2 &vhj Icii[Y"\\77II!J+ 5 sG:FF2F/F0A F:F;CFFFFFFc |j|jjj|jt |j |j|_y)zPersist the extraction agent's schema and config to the database. Returns: ExtractionAgent: The updated extraction agent )rkr{)extraction_agent_idroN) rcr=rvupdate_extraction_agentrfrrkr{r>rgs rIsavezExtractionAgent.savesR )) LL & & > >$(GG* $ 0 0;; ?   rKrextract_settingscKt|ts|g}d}nd}|Dcgc]}j|}}t5t |j dj d{}ddddtdtffd }Dcgc] }|| }}t5t |j dj d{} dddjrRt| D]C\}} t|ttfr t|nd } td | d | jE|r d S Scc}w7#1swYxYwcc}w7#1swYxYww) NTFUploading filesworkersdescr3filerMc"Kjjjtj|jj j d{}j|jd{S7'7w)Nrfile_iddata_schema_overrideconfig_override) job_creater)r=rvrun_job_test_userrrfrkr{r)r job_queuedrrHs rIrun_jobz7ExtractionAgent._queue_extraction_test..run_jobs}#||99KK+(, GG)-)9)9$(KK  "2 L J22:==AA ABs$A B#B $"BB B BzRunning extraction jobs Queued file extraction for file  under job_id r)rplistrr r"r2r3rrrAziprrrrf) rHrr single_filer upload_tasksuploaded_filesr job_tasks extract_jobsr file_reprs ` ` rI_queue_extraction_testz&ExtractionAgent._queue_extraction_tests~ %&GEKK>tWT]> > ! # !)((."00 "L  ==  5  c!+D3+!>CIDT6ykPSPVPVxX  #.|A?<?OC  (?  spE,E E,'E ,E -E 1E,E E,+'E EE A4E, E  E E,E  E)%E,c ^K t|ts|g}d}nd}|Dcgc]}|j|}}t5t ||j d|j d{}dddDcgc]]}|jjjt|j|j|j|j_}}t5t ||j d|j d{}ddd|jrRt|D]C\}}t|t t"fr t!|nd } t%d | d |jE|rd SScc}w7#1swYxYwcc}w7#1swYxYww) z Queue multiple files for extraction. Args: files (Union[FileInput, List[FileInput]]): The files to extract Returns: Union[ExtractJob, List[ExtractJob]]: The queued extraction jobs TFrrNrrnzCreating extraction jobsrrrr)rprrr r"r2r3r=rvrrrfrkr{rArrrr) rHrrrrrrrrrs rIqueue_extractionz ExtractionAgent.queue_extractions C%&GEKKCIDT6ykPSPVPVxX  #.|A?<?KC      sqF-F F-'F +F ,F 0 F-;9S#q[#9;a;2C c  WTWW9{9t99$H HH ]]]t$Y$4$**9M< 2@YY/02@/2@ z4 ++ , 2@h9@YY/09@ z4 ++ ,9@v69d9o566 z4 ++ ,6> 99d9o56 9 z4 ++ , 9 V V V    C D +.  $' % B#BrKr+ceZdZUdZedZeed<edZeed<edd Z e ed <ed d Z e ed <eddddZ e ed<edd Z eed<edd Zeed<edd Zeeed<edd Zeeed<eZeed<eZeed <eZeeed!<eZeeed"< d3deedeed e d e de ded$eed%eedeedeedeffd& Zd'eeeefd(efd)Z d4d*ed+e d,ee!d(e"fd-Z# d5d*eed.eed(e"fd/Z$d(e%e"fd0Z&d1ed(d#fd2Z'xZ(S)6 LlamaExtractz:Factory class for creating and managing extraction agents.z%The API key for the LlamaExtract API.) descriptionapi_keyz%The base URL of the LlamaExtract API.base_urlr9z;The interval in seconds to check if the extraction is done.)defaultrr0rzDThe maximum timeout in seconds to wait for the extraction to finish.r1r:rr8zAThe number of workers to use sending API requests for extraction.)rgtltrr2Tz-Show progress when extracting multiple files.r3Fz*Show verbose output when extracting files.r4zSimple SSL verification option.r5rzTimeout for the httpx client.r6 _async_clientrGr?r@Nr.r/c $|s#tjdd}| td|stjddxst}t|||||||| | |  t j| | |_| |_ | |_ t|j|j|j|_tt!dtj"xsdd z |_|stjd d}|svt'd |j)|jj*j-} | D cgc]} | j.s| }} |s td |dj0}||_||_ycc} w)NLLAMA_CLOUD_API_KEYzThe API key is required.LLAMA_CLOUD_BASE_URL) rrr0r1r2r3r5r6r4rR)tokenrrUr8r9r:r;LLAMA_CLOUD_PROJECT_IDz1No project_id provided, fetching default project.z6No default project found. Please provide a project_id.r)rEgetenvrur%superrJrVrW _httpx_clientr5r6rrrrr&rDrFrGrrcprojects list_projects is_defaultrfr?r@)rHrrr0r1r2r3r.r/r5r6r4rpdefault_project __class__s rIrJzLlamaExtract.__init__sii 5tRBRH )##''  #..fmT *,,,]]++  /B!41 9: #;TBJIJ*.*=*=&&//==?+/7"G!,,1"G"G&$P-Q/22 % /#Hs F F rLrMcrdtffd }jj|jS)rOrMcRdtffd }tj|S)NrMcKjJdtjjj4d{}|j j _ d{jj j _cdddd{S7`7<7 #jj j _wxYw#1d{7swYyxYww)Nz"httpx_client should be initializedrR)rrVrWr5r6rrTrU)r,rLrHs rIrYzCLlamaExtract._run_in_thread..run_coro..wrapped_coros&&28782!,,;; ..  FLD&&66C%)z!..**::G    * !..**::G   srAC,B'C, C'B-,B)-B-0%C C,!B+"C,)B-+C,-'CCC)C C)%C,rZr]s rIr^z-LlamaExtract._run_in_thread..run_coros"  &;;|~. .rKr_rbs`` rIrczLlamaExtract._run_in_thread s1 /! /.  ''188::rKrirkr{c |H|jtjk(r1tjdtj |_nt }t|tr|}n,t|tr|j}n td|j|jjj!|j"|j$t'|||}t)|j||j"|j$|j*|j,|j.|j0|j2 S)a]Create a new extraction agent. Args: name (str): The name of the extraction agent data_schema (SchemaInput): The data schema for the extraction agent config (Optional[ExtractConfig]): The extraction config for the agent Returns: ExtractionAgent: The created extraction agent z?ACCURATE extraction mode is deprecated. Using BALANCED instead.rm)rirkr{)r.r/ro r,r-r.r/r0r1r2r3r4)r)rACCURATErrBALANCEDDEFAULT_EXTRACT_CONFIGrprqrsrrtrurcrrvcreate_extraction_agentr?r@rr+r0r1r2r3r4)rHrirkr{r-s rI create_agentzLlamaExtract.create_agent's&  %%)=)== U*5)=)=&+F k4 (%K  Y /%779KM ##    , , D D++ $ 5 5* +! E   %%'' 11..((((,,LL  rKrfc 4||tjd|r6|j|jjj |}nN|rA|j|jjj ||j}n tdt|j||j|j|j|j|j|j|j|j |j" S)aGet extraction agents by name or extraction agent ID. Args: name (Optional[str]): Filter by name extraction_agent_id (Optional[str]): Filter by extraction agent ID Returns: ExtractionAgent: The extraction agent zJBoth name and extraction_agent_id are provided. Using extraction_agent_id.r)rir.z4Either name or extraction_agent_id must be provided.) r,r-r.r/r0r1r2r3r4r5r6)rrrcrrvget_extraction_agentget_extraction_agent_by_namer?rur+r@r0r1r2r3r4r5r6)rHrirfr-s rI get_agentzLlamaExtract.get_agentas  >d. MM\  ''""00EE(*FE ''""00MM#//NEST T%%'' 11..((((,,LL;;,,  rKcn|j|jjj|j}|Dcgc]f}t |j||j|j |j|j|j|j|j hc}Scc}w)z%List all available extraction agents.)r.r) rcrrvlist_extraction_agentsr?r+r@r0r1r2r3r4)rHagentsr-s rI list_agentszLlamaExtract.list_agentss$$    , , C C++ D  $    ))++ $ 5 5#22 ,, ,,"00     sA+B2agent_idcn|j|jjj|y)zzDelete an extraction agent by ID. Args: agent_id (str): ID of the extraction agent to delete r N)rcrrvdelete_extraction_agent)rHrs rI delete_agentzLlamaExtract.delete_agents5     , , D D$, E  rK) NNr9rr:TNNTrFre)NN))rrrrr#rr__annotations__rr0rr1r2r3rr4r5rr6rr$rrrGr&r?r@rJr r r'rcrrr+r rrrr __classcell__)rs@rIrrsuD%LMGSM&MNHcNQNCZK W K  "QM4#OGT#"CFHTN&+ ?&M8E?&1]M?2'2}L$4!,K#.&1mhsm3"&"&"$()-!%)+=0#=03-=0 =0  =0  =0=0SM=0"#=0=0 =0=0~;9S#q[#9;a;@+/ 8 8 !8 ' 8  8 x# 0 sm0  SM0   0 d T/2 .  S  T  rKr__main__) load_dotenvtestsdataz test-agent)riobjectrrstring)titlesummary)rr propertiesslidezconocophilips.pdf)Pr[rEriorrrpathlibrtypingrrr r r r r rrVpydanticr llama_cloudrrrrrrrrrrrrrrrrllama_cloud.clientr"llama_cloud_services.extract.utilsrr llama_index.core.schemar!llama_index.core.async_utilsr" llama_index.core.bridge.pydanticr#r$llama_index.core.constantsr%concurrent.futuresr&r'rrrrPER_DOCrrr+rrdotenvr__file__parentdata_dir extractorrr-rr rrrrfrrrKrIr5s 66GGG "/S11?71 CL #tUN2 3 NDO34 &#++(( wBwBt L =L ^ z"MH~$$++g5>HI  ###6mmHw.1DDEG 588$ 'N-   &&  $h/ &1     sE%%!F F