ghGddZddlmZddlZddlmZmZddlmZddl m Z ddl m Z m Z mZddlmZdd lmZdd lmZdd lmZdd lmZejeZGd dee ZGddeZGddZGddeeZ GddeeZ!GddeeZ"dS)z3Interfaces to be implemented by general evaluators.) annotationsN)ABCabstractmethod)Sequence)Enum)AnyOptionalUnion)warn) AgentAction)BaseLanguageModel)run_in_executor)ChainceZdZdZdZ dZ dZ dZ dZ dZ dZ d Z d Z d Z d Z d Z dZ dZ dZ dZ dZ dZ dZ dZdS) EvaluatorTypezThe types of the evaluators.qacot_qa context_qapairwise_string score_stringlabeled_pairwise_stringlabeled_score_string trajectorycriterialabeled_criteriastring_distance exact_match regex_matchpairwise_string_distanceembedding_distancepairwise_embedding_distance json_validity json_equalityjson_edit_distancejson_schema_validationN)__name__ __module__ __qualname____doc__QACOT_QA CONTEXT_QAPAIRWISE_STRING SCORE_STRINGLABELED_PAIRWISE_STRINGLABELED_SCORE_STRINGAGENT_TRAJECTORYCRITERIALABELED_CRITERIASTRING_DISTANCE EXACT_MATCH REGEX_MATCHPAIRWISE_STRING_DISTANCEEMBEDDING_DISTANCEPAIRWISE_EMBEDDING_DISTANCE JSON_VALIDITY JSON_EQUALITYJSON_EDIT_DISTANCEJSON_SCHEMA_VALIDATIONX/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/langchain/evaluation/schema.pyrrs&& B F%JS'O!L7H1@#VH<)7'OPKIKN9=-M"?;#M.#M=-T5IIr?rc:eZdZdZeed dZdS) LLMEvalChainz,A base class for evaluators that use an LLM.llmr kwargsrreturnc dS)z#Create a new evaluator from an LLM.Nr>)clsrCrDs r@from_llmzLLMEvalChain.from_llmNr?N)rCr rDrrErB)r&r'r(r) classmethodrrHr>r?r@rBrBKsB66222^[222r?rBceZdZdZeddZeddZeddZeddZ dddZ d S)_EvalArgsMixinz(Mixin for checking evaluation arguments.rEboolcdSz2Whether this evaluator requires a reference label.Fr>selfs r@requires_referencez!_EvalArgsMixin.requires_referenceW ur?cdS)0Whether this evaluator requires an input string.Fr>rPs r@requires_inputz_EvalArgsMixin.requires_input\rSr?strc"d|jjdS)z&Warning to show when input is ignored.zIgnoring input in , as it is not expected. __class__r&rPs r@_skip_input_warningz"_EvalArgsMixin._skip_input_warningasVDN$;UUUUr?c"d|jjdS)z*Warning to show when reference is ignored.zIgnoring reference in rYrZrPs r@_skip_reference_warningz&_EvalArgsMixin._skip_reference_warningfs WT^%< V V V r?N reference Optional[str]inputNonec|jr|t|jjd||jst |j|jr|t|jjd||jst |jdSdSdS)aCheck if the evaluation arguments are valid. Args: reference (Optional[str], optional): The reference label. input (Optional[str], optional): The input string. Raises: ValueError: If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided. Nz requires an input string.z requires a reference string.)rV ValueErrorr[r&r r\rRr^)rQr_ras r@_check_evaluation_argsz%_EvalArgsMixin._check_evaluation_argsms   +5= 7SSSTT T  t':  ) * * *  " /y'8 7VVVWW W  "4+B " - . . . . .# " " "r?rErMrErW)NN)r_r`rar`rErb) r&r'r(r)propertyrRrVr\r^rer>r?r@rLrLTs22 XXVVVXV   X $(#///////r?rLceZdZdZeddZeddZedddddZdddddZ dddddZ dddddZ dS)StringEvaluatorzcGrade, tag, or otherwise evaluate predictions relative to their inputs and/or reference labels.rErWc|jjS)zThe name of the evaluation.rZrPs r@evaluation_namezStringEvaluator.evaluation_names~&&r?rMcdSrOr>rPs r@rRz"StringEvaluator.requires_referencerSr?Nr_ra predictionUnion[str, Any]r_Optional[Union[str, Any]]rarDrdictc dS)a:Evaluate Chain or LLM output, based on optional input and label. Args: prediction (str): The LLM or chain prediction to evaluate. reference (Optional[str], optional): The reference label to evaluate against. input (Optional[str], optional): The input to consider during evaluation. kwargs: Additional keyword arguments, including callbacks, tags, etc. Returns: dict: The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys: - score: the score of the evaluation, if applicable. - value: the string value of the evaluation, if applicable. - reasoning: the reasoning for the evaluation, if applicable. Nr>rQror_rarDs r@_evaluate_stringsz!StringEvaluator._evaluate_stringsrIr?c@Ktd|jf|||d|d{VS)aIAsynchronously evaluate Chain or LLM output, based on optional input and label. Args: prediction (str): The LLM or chain prediction to evaluate. reference (Optional[str], optional): The reference label to evaluate against. input (Optional[str], optional): The input to consider during evaluation. kwargs: Additional keyword arguments, including callbacks, tags, etc. Returns: dict: The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys: - score: the score of the evaluation, if applicable. - value: the string value of the evaluation, if applicable. - reasoning: the reasoning for the evaluation, if applicable. Nror_ra)rrurts r@_aevaluate_stringsz"StringEvaluator._aevaluate_stringss`,%   " "             r?r`c R||||jd|||d|S)aEvaluate Chain or LLM output, based on optional input and label. Args: prediction (str): The LLM or chain prediction to evaluate. reference (Optional[str], optional): The reference label to evaluate against. input (Optional[str], optional): The input to consider during evaluation. kwargs: Additional keyword arguments, including callbacks, tags, etc. Returns: dict: The evaluation results containing the score or value. rnrwr>)rerurts r@evaluate_stringsz StringEvaluator.evaluate_stringssM$ ##iu#EEE%t% !Ye  GM   r?cbK||||jd|||d|d{VS)a Asynchronously evaluate Chain or LLM output, based on optional input and label. Args: prediction (str): The LLM or chain prediction to evaluate. reference (Optional[str], optional): The reference label to evaluate against. input (Optional[str], optional): The input to consider during evaluation. kwargs: Additional keyword arguments, including callbacks, tags, etc. Returns: dict: The evaluation results containing the score or value. rnrwNr>)rerxrts r@aevaluate_stringsz!StringEvaluator.aevaluate_stringsso$ ##iu#EEE,T, !Ye  GM         r?rgrf) rorpr_rqrarqrDrrErr) rorWr_r`rar`rDrrErr) r&r'r(r)rhrlrRrrurxrzr|r>r?r@rjrjs  '''X'X 04+/      ^ 604+/       F$(#       6$(#         r?rjcbeZdZdZedddddZdddddZdddddZdddddZdS)PairwiseStringEvaluatorzDCompare the output of two models (or two outputs of the same model).NrnrorW prediction_br_r`rarDrrErrc dS)1Evaluate the output string pairs. Args: prediction (str): The output string from the first model. prediction_b (str): The output string from the second model. reference (Optional[str], optional): The expected output / reference string. input (Optional[str], optional): The input string. kwargs: Additional keyword arguments, such as callbacks and optional reference strings. Returns: dict: A dictionary containing the preference, scores, and/or other information. Nr>rQrorr_rarDs r@_evaluate_string_pairsz.PairwiseStringEvaluator._evaluate_string_pairsrIr?cBKtd|jf||||d|d{VS)@Asynchronously evaluate the output string pairs. Args: prediction (str): The output string from the first model. prediction_b (str): The output string from the second model. reference (Optional[str], optional): The expected output / reference string. input (Optional[str], optional): The input string. kwargs: Additional keyword arguments, such as callbacks and optional reference strings. Returns: dict: A dictionary containing the preference, scores, and/or other information. Nrorr_ra)rrrs r@_aevaluate_string_pairsz/PairwiseStringEvaluator._aevaluate_string_pairssc(%   ' "%            r?c T||||jd||||d|S)rrnrr>)rerrs r@evaluate_string_pairsz-PairwiseStringEvaluator.evaluate_string_pairs0sT( ##iu#EEE*t* !%       r?cdK||||jd||||d|d{VS)rrnrNr>)rerrs r@aevaluate_string_pairsz.PairwiseStringEvaluator.aevaluate_string_pairsMsv( ##iu#EEE1T1 !%             r?) rorWrrWr_r`rar`rDrrErr) r&r'r(r)rrrrrr>r?r@r~r~sNN $(#      ^ 4$(#       F$(#       D$(#         r?r~creZdZdZeddZeddddZddddZddddZ ddddZ dS)AgentTrajectoryEvaluatorz,Interface for evaluating agent trajectories.rErMcdS)rUTr>rPs r@rVz'AgentTrajectoryEvaluator.requires_inputns tr?N)r_rorWagent_trajectory!Sequence[tuple[AgentAction, str]]rar_r`rDrrrc dS)Evaluate a trajectory. Args: prediction (str): The final predicted response. agent_trajectory (List[Tuple[AgentAction, str]]): The intermediate steps forming the agent trajectory. input (str): The input to the agent. reference (Optional[str]): The reference answer. Returns: dict: The evaluation result. Nr>rQrorrar_rDs r@_evaluate_agent_trajectoryz3AgentTrajectoryEvaluator._evaluate_agent_trajectorysrIr?cBKtd|jf||||d|d{VS)Asynchronously evaluate a trajectory. Args: prediction (str): The final predicted response. agent_trajectory (List[Tuple[AgentAction, str]]): The intermediate steps forming the agent trajectory. input (str): The input to the agent. reference (Optional[str]): The reference answer. Returns: dict: The evaluation result. N)rorr_ra)rrrs r@_aevaluate_agent_trajectoryz4AgentTrajectoryEvaluator._aevaluate_agent_trajectorysc*%   + "-            r?c T||||jd||||d|S)rrnrorarr_r>)rerrs r@evaluate_agent_trajectoryz2AgentTrajectoryEvaluator.evaluate_agent_trajectorysT* ##iu#EEE.t. !-       r?cdK||||jd||||d|d{VS)rrnrNr>)rerrs r@aevaluate_agent_trajectoryz3AgentTrajectoryEvaluator.aevaluate_agent_trajectorysv* ##iu#EEE5T5 !-             r?rf) rorWrrrarWr_r`rDrrErr) r&r'r(r)rhrVrrrrrr>r?r@rrks66 X$(      ^ 8$(       J$(       H$(         r?r)#r) __future__rloggingabcrrcollections.abcrenumrtypingrr r warningsr langchain_core.agentsr langchain_core.language_modelsr langchain_core.runnables.configrlangchain.chains.baser getLoggerr&loggerrWrrBrLrjr~rr>r?r@rs99""""""########$$$$$$''''''''''------<<<<<<;;;;;;''''''  8 $ $3J3J3J3J3JC3J3J3Jl222225222././././././././bq q q q q ncq q q ho o o o o nco o o dx x x x x ~sx x x x x r?