hh3ddlmZddlZddlmZmZmZmZmZm Z m Z m Z m Z m Z ddlZddlZddlmZmZddlmZddlmZerddlmZmZddlmZGd d eZdS) ) annotationsN) TYPE_CHECKINGAny AsyncIteratorDict GeneratorIteratorListMappingOptionalUnion)AsyncCallbackManagerForLLMRunCallbackManagerForLLMRun)LLM)GenerationChunk)RESTfulChatModelHandleRESTfulGenerateModelHandle)LlamaCppGenerateConfigceZdZUdZdZded<ded< ded< ded < d.d/fd Zed0dZed1dZ d2dZ d3d4dZ d3d5d Z d3d6d"Z d7d8d%Zed9d(Z d3d:d+Z d7d;d-ZxZS)< Xinferencea `Xinference` large-scale model inference service. To use, you should have the xinference library installed: .. code-block:: bash pip install "xinference[all]" If you're simply using the services provided by Xinference, you can utilize the xinference_client package: .. code-block:: bash pip install xinference_client Check out: https://github.com/xorbitsai/inference To run, you need to start a Xinference supervisor on one server and Xinference workers on the other servers Example: To start a local instance of Xinference, run .. code-block:: bash $ xinference You can also deploy Xinference in a distributed cluster. Here are the steps: Starting the supervisor: .. code-block:: bash $ xinference-supervisor Starting the worker: .. code-block:: bash $ xinference-worker Then, launch a model using command line interface (CLI). Example: .. code-block:: bash $ xinference launch -n orca -s 3 -q q4_0 It will return a model UID. Then, you can use Xinference with LangChain. Example: .. code-block:: python from langchain_community.llms import Xinference llm = Xinference( server_url="http://0.0.0.0:9997", model_uid = {model_uid} # replace model_uid with the model UID return from launching the model ) llm.invoke( prompt="Q: where can we visit in the capital of France? A:", generate_config={"max_tokens": 1024, "stream": True}, ) Example: .. code-block:: python from langchain_community.llms import Xinference from langchain.prompts import PromptTemplate llm = Xinference( server_url="http://0.0.0.0:9997", model_uid={model_uid}, # replace model_uid with the model UID return from launching the model stream=True ) prompt = PromptTemplate( input=['country'], template="Q: where can we visit in the capital of {country}? A:" ) chain = prompt | llm chain.stream(input={'country': 'France'}) To view all the supported builtin models, run: .. code-block:: bash $ xinference list --all Nz Optional[Any]client Optional[str] server_url model_uidzDict[str, Any] model_kwargsapi_keyrc  ddlm}n:#t$r- ddlm}n"#t$r}td|d}~wwxYwYnwxYw|pi}t jd i|||d|jtd|jtdi|_ d|_ | ||j r d||j d <||||_ dS) Nr) RESTfulClientzCould not import RESTfulClient from xinference. Please install it with `pip install xinference` or `pip install xinference_client`.rrrzPlease provide server URLzPlease provide the model UIDFzBearer Authorization) xinference.clientr ImportErrorxinference_clientsuper__init__r ValueErrorr_headers_cluster_authed_check_cluster_authenticatedr)selfrrrrre __class__s `/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/langchain_community/llms/xinference.pyr&zXinference.__init__sd  7 7 7 7 7 7 7    ;;;;;;;   !Y <; $)r   (& ,    ? "899 9 > !;<< <(* $ ))+++  4#7 -@w-@-@DM/ *#mJ88 s) AA ;6;AAreturnstrcdS)zReturn type of llm. xinferencer!r+s r. _llm_typezXinference._llm_types |Mapping[str, Any]c<id|jid|jid|jiS)zGet the identifying parameters.rrrrr3s r._identifying_paramszXinference._identifying_paramss< T_- DN+ t01  r5Nonec0|jd}tj|}|jdkr d|_dS|jdkr*t d|d|}t|d|_dS)Nz/v1/cluster/authFz+Failed to get cluster information, detail: detailauth)rrequestsget status_coder) RuntimeErrorjsonbool)r+urlresponse response_datas r.r*z'Xinference._check_cluster_authenticateds222<$$  3 & &#(D #s**";'}}x8;;%MMOOM#' f(=#>#>D r5promptstopOptional[List[str]] run_manager"Optional[CallbackManagerForLLMRun]kwargsc |jtd|j|j}|di}i|j|}|r||d<|r:|dr%d}|||||D]}||z }|S|||} | d d d S) aqCall the xinference model and return the output. Args: prompt: The prompt to use for generation. stop: Optional list of stop words to use when generating. generate_config: Optional dictionary for the configuration used for generation. Returns: The generated string by the model. NClient is not initialized!generate_configrIstream)modelrHrKrPrHrPchoicesrtext)rr' get_modelrr@r_stream_generategenerate) r+rHrIrKrMrSrPcombined_text_outputtoken completions r._callzXinference._calls$ ; 9:: : %%dn554:JJ?PRT4U4UBT.B/B  +&*OF #  4228<< 4#% ..' / / . . %-$$' 'vWWJi(+F3 3r5rS=Union['RESTfulGenerateModelHandle', 'RESTfulChatModelHandle']rP"Optional['LlamaCppGenerateConfig']Generator[str, None, None]c#rK|||}|D]}t|tr|dg}|rk|d}t|trN|dd} |d} |r|| |j| | VdS) a^ Args: prompt: The prompt to use for generation. model: The model used for generation. stop: Optional list of stop words to use when generating. generate_config: Optional dictionary for the configuration used for generation. Yields: A string token. rTrUrrVrRlogprobs)r[verbose log_probsN)rY isinstancedictr@on_llm_new_tokenrc) r+rSrHrKrPstreaming_responsechunkrUchoicer[rds r.rXzXinference._stream_generates$#^^?,  ( $ $E%&& $))Ir22 $$QZF!&$//$ & 62 6 6$*JJz$:$: &'88&+T\Y9$  $ $r5Iterator[GenerationChunk]c+K|di}i|j|}|r||d<|||D]@}|r<||}|r!||j|j|VAdSNrPrI)rc)r@r_create_generate_stream$_stream_response_to_generation_chunkrgrVrcr+rHrIrKrMrP stream_respris r._streamzXinference._streams!**%6;;BT.B/B  +&*OF #77PP  K AA+NN00 $ 1   r5Optional[Dict[str, List[str]]] Iterator[str]c#K|jtd|j|j}|||Ed{VdS)NrOrT)rr'rWrrY)r+rHrPrSs r.rnz"Xinference._create_generate_stream.sd ; 9:: : %%dn55>>>QQQQQQQQQQQr5stream_responserc d}t|tr|dg}|r|d}t|tr^|dd}t|t|dd|dd St d t| St d ) z0Convert a stream response to a generation chunk.rRrUrrV finish_reasonNrb)rxrb)rVgeneration_infozchoice type error!)rVzstream_response type error!)rerfr@r TypeError)rvr[rUrjs r.roz/Xinference._stream_response_to_generation_chunk6s  ot , , ;%)))R88G 3 fd++ :"JJvr22E*"(,*0**_d*K*K%+ZZ D%A%A)))$$8999&E22229:: :r5'Optional[AsyncCallbackManagerForLLMRun]AsyncIterator[GenerationChunk]c "K|di}i|j|}|r||d<|||23d{V}|rC||}|r'||j|jd{V|WVM6dSrm)r@r_acreate_generate_streamrorgrVrcrps r._astreamzXinference._astreamQs!**%6;;BT.B/B  +&*OF #!%!>!>v!W!W       + AA+NN%66 $ 7 "X!W!WsBAsyncIterator[str]ctK|j|d}||D] \}}|||< t|o|d}t j4d{V}||jd|4d{V }|jdkr;|jdkrtd|j } td|jd | |j 23d{V} |stj| WV!| d } | d rC| t#d d} | swtj| WV6 dddd{Vn#1d{VswxYwYdddd{VdS#1d{VswxYwYdS) N)rSrHrQz/v1/completions)rErCr<r;z)astream call failed with status code 404.z%astream call failed with status code z . Details: zutf-8sdata:)ritemsrDr@aiohttp ClientSessionpostrstatusFileNotFoundErrorrVr'contentrCloadsdecode startswithlenstrip) r+rHrP request_bodykeyvaluerQsessionrFoptional_detaillinejson_strs r.r~z#Xinference._acreate_generate_streamfs>266'R'R  &-3355 * * U$) S!!oG/*=*=h*G*GHH(** 7 7 7 7 7 7 7g||777!$ 7 7 7 7 7 7 7 7?c))#--/G+3-(;HO;;)8;; #+"2 7 7 7 7 7 7 7$!7"j.......#';;w#7#7??8447'/H '@'F'F'H'HH#+) ("&*X"6"66666#3"2! 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7sD+&F'AF E/&B F1F' F F'F F'' F14F1)NNN)rrrrrrrr)r/r0)r/r6)r/r9)NN) rHr0rIrJrKrLrMrr/r0) rSr^rHr0rKrLrPr_r/r`) rHr0rIrJrKrLrMrr/rk)N)rHr0rPrsr/rt)rvr0r/r) rHr0rIrJrKr{rMrr/r|)rHr0rPrsr/r)__name__ __module__ __qualname____doc__r__annotations__r&propertyr4r8r*r]rXrrrn staticmethodrorr~ __classcell__)r-s@r.rrsZZx!F    &#    :%)#'!% (9(9(9(9(9(9(9TX   X  ? ? ? ?"%):> *4*4*4*4*4`;?>B !$!$!$!$!$L%):> ,NRRRRRR;;;\;:%)?C ,NR#7#7#7#7#7#7#7#7#7r5r) __future__rrCtypingrrrrrr r r r r rr?langchain_core.callbacksrr#langchain_core.language_models.llmsrlangchain_core.outputsrr"rrxinference.model.llm.corerrr!r5r.rsk""""""                         433333222222ATTTTTTTT@@@@@@j7j7j7j7j7j7j7j7j7j7r5