
    _jܖ                       d dl mZmZ d dlmZmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZ d dlmZ d dlmZmZmZmZmZ d dlmZmZ d dlmZmZ d dlm Z  d dl!m"Z" d d	l#m$Z$ d d
l%m&Z&m'Z' d dl(m Z) d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2 d dl3m4Z4 d dl5Z6d dl7Z7e G d dee.e8                                        Z9e G d dee.e8                                        Z:	 	 	 	 d[dZ;	 	 	 	 d[dZ<e G d dee.e8                                        Z=	 	 d\ddde0e         de0ee+                  de1e ef         fdZ>de+de.e+         fdZ?	 	 	 	 d[dZ@	 	 	 	 d[dZA	 	 	 	 d[d ZBe7jC        D                    e e!          	 	 	 	 d[d"            ZEe7jC        D                    ee!          	 	 	 	 d[d#            ZF	 	 	 	 d[d$ZG	 	 	 	 d[d%ZHe7jC        D                    ee!          	 	 	 	 d[d&            ZIe7jC        D                    ee!          	 	 	 	 d[d'            ZJe7jC        D                    ee!          	 	 	 	 d[d(            ZK	 	 	 	 d[d)ZL	 	 	 	 d[d*ZMe7jC        D                    ee!          	 	 	 	 d[d+            ZN	 	 	 	 d[d,ZO	 	 	 	 d[d-ZP e            e7jC        D                    ee!          	 	 	 	 d[d.                        ZQe7jC        D                    ee!          	 	 	 	 d[d/            ZR	 	 	 	 d[d0ZSe7jC        D                    ee!          	 	 	 	 d[d1            ZTe7jC        D                    ee!          	 	 	 	 d[d2            ZUe7jC        D                    ee!          	 	 	 	 d[d3            ZVe7jC        D                    ee!          	 	 	 	 d[d4            ZWe7jC        D                    ee!          	 	 	 	 d[d5            ZXe7jC        D                    ee!          	 	 	 	 d[d6            ZYe7jC        D                    ee!          	 	 	 	 d[d7            ZZe7jC        D                    ee!          	 	 	 	 d[d8            Z[e7jC        D                    ee!          	 	 	 	 d[d9            Z\e7jC        D                    ee!          	 	 	 	 d[d:            Z]e7jC        D                    ee!          	 	 	 	 d[d;            Z^e7jC        D                    ee!          	 	 	 	 d[d<            Z_	 	 	 	 d[d=Z`	 	 	 	 d[d>Za	 	 	 	 d[d?Zb	 	 	 	 d[d@Zce7jC        D                    ee!          	 	 	 	 d[dA            Zde7jC        D                    ee!          	 	 	 	 d[dB            Zee7jC        D                    ee!          	 	 	 	 d[dC            Zfe7jC        D                    ee!          	 	 	 	 d[dD            Zgd]dEZhd]dFZid]dGZje7jC        D                    ee!          	 	 	 	 d[dH            Zke7jC        D                    ee!          	 	 	 	 d[dI            Zle7jC        D                    ee!          dJeddfdK            Zme7jC        D                    e dL!          dJeddfdM            Zne7jC        D                    ee!          dJeddfdN            Zoe7jC        D                    ee!          dJeddfdO            Zpe7jC        D                    ee!          dJeddfdP            Zqd]dQZre G dR dSee.e8                                        Zse7jC        D                    ee!          deddfdT            Zt	 	 	 	 d[dUZu	 	 	 	 d[dVZv	 	 	 	 d[dWZw	 	 	 	 d[dXZx	 	 	 	 d[dYZye7jC        D                    ee!          	 	 	 	 d[dZ            ZzdS )^    )	ClientAPI	ServerAPI)SchemaFtsIndexConfigSparseVectorIndexConfigSparseEmbeddingFunctionSparseVectorStringInvertedIndexConfigIntInvertedIndexConfigFloatInvertedIndexConfigBoolInvertedIndexConfigVectorIndexConfigSpannIndexConfigEmbeddingFunction
Embeddings)Key)ClientFactoriesis_spann_disabled_modeskip_if_not_clusterskip_reason_spann_disabledskip_reason_spann_enabled)get_collection_versionwait_for_version_increase)register_embedding_function"register_sparse_embedding_function)
Collection)CollectionCommon)InvalidArgumentError)KnnSearch)AnyCallableDictListMappingOptionalTuplecast)uuid4Nc                       e Zd ZdZddefdZdee         defdZ	e
defd            Zdeeef         fd	Ze
d
eeef         dd fd            ZdefdZdS )SimpleEmbeddingFunctionzJSimple embedding function with stable configuration for persistence tests.   dimc                     || _         d S N_dim)selfr-   s     ^/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/chromadb/test/api/test_schema_e2e.py__init__z SimpleEmbeddingFunction.__init__0   s    			    inputreturnc                     d t          | j                  D             t          t          fd|D                       S )Nc                 ,    g | ]}t          |          S  )float.0is     r3   
<listcomp>z4SimpleEmbeddingFunction.__call__.<locals>.<listcomp>4   s    555q%((555r5   c                     g | ]}S r:   r:   )r=   _vectors     r3   r?   z4SimpleEmbeddingFunction.__call__.<locals>.<listcomp>5   s     7 7 7A 7 7 7r5   )ranger1   r(   r   )r2   r6   rB   s     @r3   __call__z SimpleEmbeddingFunction.__call__3   sE    55E$)$4$4555J 7 7 7 7 7 7 7888r5   c                      dS )N	simple_efr:   r:   r5   r3   namezSimpleEmbeddingFunction.name7   s    {r5   c                     d| j         iS )Nr-   r0   r2   s    r3   
get_configz"SimpleEmbeddingFunction.get_config;   s    ty!!r5   configc                 .    t          | d                   S )Nr-   r-   )r+   rK   s    r3   build_from_configz)SimpleEmbeddingFunction.build_from_config>   s    &6%=9999r5   c                     dS )Ncosiner:   rI   s    r3   default_spacez%SimpleEmbeddingFunction.default_spaceB   s    xr5   N)r,   )__name__
__module____qualname____doc__intr4   r$   strr   rD   staticmethodrG   r#   r!   rJ   rO   rR   r:   r5   r3   r+   r+   ,   s        TT C    9d3i 9J 9 9 9 9 #    \"DcN " " " " :$sCx. :5N : : : \:s      r5   r+   c                       e Zd ZdZddeddfdZdee         defdZdee         defd	Z	e
defd
            Zdeeef         fdZe
deeef         dd fd            ZdS ) RecordingSearchEmbeddingFunctionzBEmbedding function that records inputs for search embedding tests.defaultlabelr7   Nc                 0    || _         g | _        g | _        d S r/   )_labelcall_inputsquery_inputsr2   r]   s     r3   r4   z)RecordingSearchEmbeddingFunction.__init__J   s    ,.-/r5   r6   c                     | j                             t          |                     d |D             }t          t          |          S )Nc                     g | ]=}t          t          |                    t          t          |                    d z   g>S )g      ?r;   lenr=   texts     r3   r?   z=RecordingSearchEmbeddingFunction.__call__.<locals>.<listcomp>Q   >    PPP$E#d))$$eCII&6&6&<=PPPr5   )r`   appendlistr(   r   r2   r6   vectorss      r3   rD   z)RecordingSearchEmbeddingFunction.__call__O   sB    U,,,PP%PPPJ(((r5   c                     | j                             t          |                     d |D             }t          t          |          S )Nc                     g | ]=}t          t          |                    t          t          |                    d z   g>S )g      ?re   rg   s     r3   r?   z@RecordingSearchEmbeddingFunction.embed_query.<locals>.<listcomp>V   ri   r5   )ra   rj   rk   r(   r   rl   s      r3   embed_queryz,RecordingSearchEmbeddingFunction.embed_queryT   sB      e---PP%PPPJ(((r5   c                      dS )Nrecording_search_efr:   r:   r5   r3   rG   z%RecordingSearchEmbeddingFunction.nameY   s    $$r5   c                     d| j         iS Nr]   r_   rI   s    r3   rJ   z+RecordingSearchEmbeddingFunction.get_config]       %%r5   rK   c                 H    t          |                     dd                    S )Nr]   r\   )r[   getrN   s    r3   rO   z2RecordingSearchEmbeddingFunction.build_from_config`   s    /

7I0N0NOOOr5   )r\   )rS   rT   rU   rV   rX   r4   r$   r   rD   rp   rY   rG   r#   r!   rJ   rO   r:   r5   r3   r[   r[   F   s       LL0 0c 0$ 0 0 0 0
)d3i )J ) ) ) )
)c )z ) ) ) )
 %# % % % \%&DcN & & & & P$sCx. P5W P P P \P P Pr5   r[   client_factoriesr   r7   c                 8   |                                  }|                                 dt                      j         }t	                      }|                    t          dt          dddd                    	           |                    ||
          }|j	        }|J t          |                                           |j        d         j        }|J |j        }|J |j        du sJ |j        J |j        j        J |j        j        dk    sJ |                                  }|                    |          }	|	j	        }
|
J |
j        d         j        }|J |j        }|J |j        J |j        j        J |j        j        dk    sJ dS )EEnsure schema-provided SPANN settings persist across client restarts.schema_spann_rQ          x      search_nprobewrite_nprobeef_constructionmax_neighbors)spacespannrN   rG   schemaN
#embeddingTrG   )create_client_from_systemresetr)   hexr   create_indexr   r   get_or_create_collectionr   printserialize_to_jsonkeys
float_listvector_indexenabledrK   r   get_collection)ry   clientcollection_namer   
collectionpersisted_schemaembedding_overrider   client_reloadedreloaded_collectionreloaded_schemareloaded_embedding_overridereloaded_vector_indexs                r3   %test_schema_vector_config_persistencer   e   s   
 7799F
LLNNN3eggk33OXXF
 "  # 	  
 
 
  
 
 
 00 1  J
 "('''	

,
,
.
.///).|<G)))%2L###4''''***$000$0000&@@BBO)88 9   *0O&&&"1"6|"D"O&2227D ,,, '333 '-999 '-999999r5   c                    |                                  }|                                 dt                      j         }t	                      }t          d          }|                    t          d|t          dddd	          
                     |	                    ||          }|j
        }|J t          |                                           |j        d         j        }|J |j        }|J |j        du sJ |j        J |j        j        J |j        j        dk    sJ t&          sO|j        j        J |j        j        }	|	j        dk    sJ |	j        dk    sJ |	j        dk    sJ |	j        dk    sJ n\|j        j        J |j        j        J |j        j        }
|
j        dk    sJ |
j        dk    sJ |
j        dk    sJ |
j        dk    sJ |j        j        }|J |                                dk    sJ |                                ddik    sJ |                                }t&          sC|d         d         d         d         d         d         }|d         dk    sJ |d         dk    sJ nP|d         d         d         d         d         d         }|d         dk    sJ |d         dk    sJ |d         dk    sJ |                                  }|                    |          }|j
        }|J |j        d         j        }|J |j        }|J |j        J |j        j        J |j        j        dk    sJ t&          s=|j        j        J |j        j        j        dk    sJ |j        j        j        dk    sJ nj|j        j        J |j        j        j        dk    sJ |j        j        j        dk    sJ |j        j        j        dk    sJ |j        j        j        dk    sJ |j         }|J |!                    d           }|J |                                dk    sJ |                                ddik    sJ dS )!r{   r|      rM   rQ   r}   r~   r   r   r   )r   embedding_functionr   rN   r   Nr   Td   g333333?rF   r-   r   r   r   rK   r   r   r   hnswr   	ef_searchr   r   r   )"r   r   r)   r   r   r+   r   r   r   r   r   r   r   r   r   r   r   rK   r   r   r   r   r   r   r   r   r   resize_factorr   rG   rJ   r   configurationrx   )ry   r   r   r   r   r   r   r   r   spann_confighnsw_configefpersisted_json
spann_json	hnsw_jsonr   r   r   r   r   rK   	config_efs                         r3   -test_schema_vector_config_persistence_with_efr      s   
 7799F
LLNNN3eggk33OXXF0Q777
 1"  # 	  	
 	
 	
     00 1  J
 "('''	

,
,
.
.///).|<G)))%2L###4''''***$000$0000! 0"(444#*0)R////(B....+s2222)R/////"(000"'333").*c1111$++++(B....(C////			/B>>>7799####==??uaj((((%7799N! 0#F+L9,GW


 /*b0000.)R/////"6*<8F~V

	 *+s2222%,,,,)R////&@@BBO)88 9   *0O&&&"1"6|"D"O&2227D ,,, '333 '-999 '-9999! 	F$+1===$+1?2EEEE$+1>"DDDDD$+0<<<$+0@CGGGG$+0:cAAAA$+0>"DDDD$+0>#EEEE .F

/00I   >>{****!!eQZ//////r5   c                       e Zd ZdZddefdZdee         dee         fdZe	defd            Z
deeef         fd	Ze	d
eeef         dd fd            ZdS )$DeterministicSparseEmbeddingFunctionzCSparse embedding function that emits predictable token/value pairs.
det_sparser]   c                     || _         d S r/   ru   rb   s     r3   r4   z-DeterministicSparseEmbeddingFunction.__init__      r5   r6   r7   c                 4    d t          |          D             S )Nc           
      t    g | ]5\  }}t          |gt          t          |          |z             g           6S indicesvaluesr	   r;   rf   r=   idxrh   s      r3   r?   zADeterministicSparseEmbeddingFunction.__call__.<locals>.<listcomp>  P     
 
 
T #c$ii#o0F0F/GHHH
 
 
r5   	enumerater2   r6   s     r3   rD   z-DeterministicSparseEmbeddingFunction.__call__  +    
 
&u--
 
 
 	
r5   c                      dS )Nr   r:   r:   r5   r3   rG   z)DeterministicSparseEmbeddingFunction.name  s    |r5   c                     d| j         iS rt   ru   rI   s    r3   rJ   z/DeterministicSparseEmbeddingFunction.get_config!  rv   r5   rK   c                 H    t          |                     dd                    S )Nr]   r   )r   rx   rN   s    r3   rO   z6DeterministicSparseEmbeddingFunction.build_from_config$  s      4FJJw4U4UVVVr5   N)r   )rS   rT   rU   rV   rX   r4   r$   r	   rD   rY   rG   r#   r!   rJ   rO   r:   r5   r3   r   r     s        MM c    
d3i 
D,> 
 
 
 
 #    \&DcN & & & & WS#XW	/W W W \W W Wr5   r   r   r   c                    |                                  }|                                 dt                      j         }||                    ||          }n0||                    ||          }n|                    |          }||fS )zKProvision a new temporary collection and return it with the backing client.schema_e2e_Nr   rG   r   r   )r   r   r)   r   r   )ry   r   r   r   r   r   s         r3   _create_isolated_collectionr   +  s     7799F
LLNNN1EGGK11O44  5 
 



 )88$#5 9  JJ
  88$ 9  J vr5   rankc                    | g S t          | t                    r| j        gS g }t          | dd           }|"|                    t          |                     t          | dd           }|"|                    t          |                     t          | dd           }|"|                    t          |                     t          | dd           }|r'|D ]$}|                    t          |                     %|S )Nr   leftrightranks)
isinstancer   querygetattrextend_collect_knn_queries)r   queries
child_rank	left_rank
right_rankr   childs          r3   r   r   H  s   |	$ 
|Gvt,,J+J77888fd++I+I66777w--J+J77888D'4((E 8 	8 	8ENN/667777Nr5   c                 $   t          |           \  }}|j        }|J |j        J |j        j        J |j        j        j        }|J |j        du sJ |j        j        J |j        j        j        }|J |j        du sJ |j        j        J |j        j        j	        }|J |j        du sJ |j        j
        J |j        j
        j        }|J |j        du sJ |j        d         j        }|J |j        }	|	J |	j        du sJ |j        d         j        }
|
J |
j        }|J |j        du sJ g d}g d}ddd	dd
ddddd
ddddd
g}|                    |||           |                    ddi          }t%          |d                   dhk    sJ |                    ddi          }t%          |d                   dhk    sJ |                    ddi          }t%          |d                   dhk    sJ |                    dgdddg           |                    ddi          }t%          |d                   dhk    sJ |                    dgdgdddg           |                    dgd g!          }|d          J t+          |d                    d"k    sJ |                    |j                  }|j        J t0          s1|j                                        |                                k    sJ dS dS )#zKValidate default schema indexes support filtering, updates, and embeddings.NT	#documentr   )doc-1doc-2doc-3)alphabetagammanews   g      #@)categoryratingprice	is_activescience   g      @F   g      @ids	documents	metadatasr   wherer   r   r   r   r   r   r   updates)r   r   r   r   zbeta-updated)r   r   
embeddingsr   include   )r   r   defaultsstringstring_inverted_indexr   	int_valueint_inverted_indexfloat_valuefloat_inverted_indexbooleanbool_inverted_indexr   	fts_indexr   r   addrx   setupdateupsertrf   r   rG   r   r   )ry   r   r   r   string_index	int_indexfloat_index
bool_indexdocument_overrider  r   r   r   r   r   filterednumeric_filterbool_filterrating_after_updateembeddings_payloadreloadeds                        r3   .test_schema_defaults_enable_indexed_operationsr  e  s    55EFFJF?&&&?!---?)?L###4''''?$000)<I   $$$$?&222/-BK"""$&&&&?"...(<J!!!%%%%K07(((!+I   $$$$\2=)))%2L###4''''
%
%
%C***Iq3TJJ!cNNq3TJJ*I NNsi9NEEE~~Z$;~<<HxG9,,,,^^8Q-^88N~e$%%'2222..U';.<<K{5!""wi////7)1)0T0T/UVVV$..!}.=="5)**wi7777I!" i889     $WI~NNl+777!,/00A5555 $$Z_55H?&&&! Q0022f6N6N6P6PPPPPQ QPPr5   c                    t                      }|                    dt                                 |                    dt                                 t	          | |          \  }}|j        J |j                                        }d|d         v sJ d|d         v sJ |                    |j                  }|j        J |j                                        |k    sJ |	                    |j                  }|j        J |j                                        |k    sJ |
                    dgd	gd
ddg           |                    dd
i          }t          |d                   dhk    sJ dS )zIEnsure repeated collection lookups reuse the persisted schema definition.
custom_tagkeyrK   
importancer   Nr   r   zschema-preservedocr   
   )r  r  r   r   r   )r   r   r
   r   r   r   r   r   rG   r   r  rx   r  )ry   base_schemar   r   initial_schema_jsonsecond_referencefetchedstoreds           r3   5test_get_or_create_and_get_collection_preserve_schemar     s    ((K(**     %''    
 5  J
 ((($+==??.v66666.v6666666JO6LL"..."4466:MMMMM###99G>%%%>++--1DDDDD'")<<=     [[g6[77Fve}"3!4444444r5   c                 n   t                      }|                    dt                                 t          | |          \  }}|j        J d|j        j        v sJ |                    |j                   |                    |j                  }|j        J |j        	                                }t                      	                                }d|d         vsJ t          |d                                                   t          |d                                                   k    sJ dS )zHDeleting and recreating a collection should drop prior schema overrides.transient_keyr  r  Nr   r   )r   r   r
   r   r   r   delete_collectionrG   create_collectionr   r  )ry   r   r   r   	recreatedrecreated_jsonbaseline_jsons          r3   2test_delete_collection_resets_schema_configurationr(    sG    XXF
(**    
 5  J
 (((j/44444
*/222((jo(>>I'''%7799NHH..00M."88888~f%**,,--]65J5O5O5Q5Q1R1RRRRRRRr5   )reasonc                     t                      }|                    dt                                 t          j        t
          d          5  t          | |           ddd           dS # 1 swxY w Y   dS )z<Sparse vector configs are not allowed to be created locally.sparse_metadatar  z.Sparse vector indexing is not enabled in localmatchr  N)r   r   r   pytestraisesr   r   )ry   r   s     r3   &test_sparse_vector_not_allowed_locallyr0    s    
 XXF
-6M6O6OPPP	$T
 
 
 E E 	$$4VDDDDE E E E E E E E E E E E E E E E E Es   A,,A03A0c                 F   t          d          }t                      }|                    dt          d|                     |                    dt	                                 |                    dt	                                 t          | |	          \  }}|j        J d|j        j        v sJ d|j        j        v sJ d|j        j        v sJ |                    dgdgddddg           |	                    dgdg          }|d         J |d         d         }|J |d         dk    sJ |d         dk    sJ |d         dk    sJ d|v sJ t          t          |d                   }| |dg          d         k    sJ |                    t                                          t          dt          t           |                                        }t#          |d                   dk    sJ d|d         d         v sJ t%          j        t(                    5  |j                            dt          d                     d
d
d
           n# 1 swxY w Y   |	                    ddi          }	t+          |	d                   dhk    sJ d
S )zNSparse vector configs honor source key embedding and single-index enforcement.zsource-testr]   r+  raw_text
source_keyr   r  tag_atag_br  Nzsparse-1zsource documentorangescitrusfruit)r3  r6  r7  r   r   r   r   r  r   r   r   another_sparser5  r   )r   r   r   r   r
   r   r   r   r  rx   r(   r	   searchr    r   r   r!   rf   r.  r/  
ValueErrorr  )
ry   	sparse_efr   r   rA   r  metadatasparse_payloadsearch_resultstring_filters
             r3   3test_sparse_vector_source_key_and_index_constraintsrE    sZ   
 5=IIIIXXF
&!(
 
 
     G,E,G,GHHH
G,E,G,GHHH/0@PPPMJ(((
 1 66666j',,,,,j',,,,,NNL$% )HwOOP     ^^{m^DDF+***k"1%HG((((G''''J9,,,,((((,1B(CDDNYY	{33A66666%%c/tC7P7PQQQRR M }U#$$))))u-a00000	z	"	" 
 
&& *jAAA 	' 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 NN'7);N<<M}U#$$444444s   ++I""I&)I&c           	      
   t                      }|                    dt                                 |                    dt                                 |                    dt	                                 |                    dt                                 t          | |          \  }}|                    dgdgd	d
dddg           |j        J |j        	                                }| 
                                }|                    |j                  }|j        J t          s|j        	                                |k    sJ |                    dd	i          }t          |d                   dhk    sJ dS )z<Custom schema overrides persist across new client instances.titler  published_yearscoreis_featuredr  z	persist-1zpersistent doczSchema Persistencei  g      @T)rG  rH  rI  rJ  r   Nr   r   r   )r   r   r
   r   r   r   r   r  r   r   r   r   rG   r   rx   r  )ry   r   r   r   expected_schema_jsonreloaded_clientr   r  s           r3   -test_schema_persistence_with_custom_overridesrM  @  s    XXF
G,E,G,GHHH
,5K5M5MNNN
G,D,F,FGGG
M2I2K2KLLL4  J
 NNM#$ ."&#	 
     (((%,>>@@&@@BBO)88jo8NN%111! V");;==AUUUUU!%%W6J,K%LLGwu~;-//////r5   c                    t          d          }t                                          t          |                    }t	          | |          \  }}|                    dg          }|J t          j        |d         g d	          sJ t          d
          }t	          | |          \  }}|                    dg          }|J t          j        |d         g d          sJ dS )zE_embed should respect schema-provided and direct embedding functions.r   rM   r   rN   r  zschema documentNr                 ?       @      @g      @r   zdirect document)rQ  rR  rS  )r+   r   r   r   r   _embednpallclose)	ry   schema_emb_fnr   schema_collectionrA   schema_embeddingsdirect_emb_fndirect_collectiondirect_embeddingss	            r3   Btest_collection_embed_uses_schema_or_collection_embedding_functionr^  i  s+   
 ,222MXX"" MBBB #  F 7  q
 *002C1DEE(((;(+-F-F-FGGGGG+222M6(  q
 *002C1DEE(((;(+___=======r5   c                    t          d          }t          | |          \  }}t                                          t	          d                    }t          |j                   |                    |          }|j        dggk    sJ |j	        rJ t          |j        t                    sJ |j        j        dk    sJ |j        }t          |t                    sJ |j        ddgk    sJ dS )	Q_embed_search_string_queries should embed string KNN queries using collection EF.primaryr2  rO  hello worldr         &@g      )@N)r[   r   r    r   r   r   r   _embed_search_string_queriesra   r`   r   _rankr   )ry   embedding_fnr   rA   r>  embedded_searchembedded_ranks          r3   %test_search_embeds_string_knn_queriesrj    s	    4)DDDL/\  MJ XX]]3]33344F	*
 ==fEEO$-(99999''''flC(((((<....#)MmS)))))4,......r5   c                 v   t          d          }t                                          dt          d|                    }t	          | |          \  }}t                                          t          dd	                    }|                    |          }t          |j
        t                    sJ |j
        j        dk    sJ |j
        j        dk    sJ |j
        }t          |t                    sJ |j        dk    sJ t          |j                   |j        t          d
gdg          k    sJ dS )r`  sparser2  r+  r3  r4  r  r  rb  r;  r   rd  r   N)r   r   r   r   r   r    r   r   re  r   rf  r  r   r   r	   )ry   r@  r   r   rA   r>  rh  ri  s           r3   Dtest_search_embeds_string_knn_queries_with_sparse_embedding_functionrm    sO    58DDDIXX""&!i
 
 
 #  F 00@PPPMJXX]]3#4MJJJKKF ==fEEOflC(((((<00000<....#)MmS))))) 11111	-
,sD6"J"J"JJJJJJJr5   c                   
 t          d          }t          | |          \  
}t          d          t          d          z                       t          d                    }t          d          d	z                                  }t                                          |          t                                          |          g}
fd
|D             }dgdgdgdgg}|j        |k    sJ g }|D ])}	|                    t          |	j
                             *|sJ t          d |D                       sJ t          d |D                       sJ dS )z>String queries in composite rank trees should all be embedded.nestedr2  rO  r   rc  r   r   delta   c                 :    g | ]}                     |          S r:   )re  )r=   sr   s     r3   r?   zEtest_search_embeds_string_queries_in_nested_ranks.<locals>.<listcomp>  s'    VVV@@CCVVVr5   c              3   B   K   | ]}t          |t                     V  d S r/   )r   rX   r=   r   s     r3   	<genexpr>zDtest_search_embeds_string_queries_in_nested_ranks.<locals>.<genexpr>  s/      CCe:eS)))CCCCCCr5   c              3   @   K   | ]}t          |t                    V  d S r/   )r   rk   ru  s     r3   rv  zDtest_search_embeds_string_queries_in_nested_ranks.<locals>.<genexpr>  s,      @@5z%&&@@@@@@r5   N)r[   r   r   maxabsr    r   ra   r   r   rf  all)ry   rg  rA   rank_onerank_twosearchesembedded_searchesexpected_queriesall_queriesrh  r   s             @r3   1test_search_embeds_string_queries_in_nested_ranksr    s    4(CCCL/\  MJ '"""Sv%6%6%66;;Cg<N<N<NOOH'"""Q&++--Hh''x)@)@AHVVVVXVVV 	F8gY	B$(88888K, H H/0EFFGGGGCC{CCCCCCCC@@K@@@@@@@@@@r5   c                    t                                          t                                }t          | |          \  }}|                    dgdgddig           t          j        t                    5  |                    ddi           d	d	d	           n# 1 swxY w Y   |	                    |j
        
           t                                          t                                                    dt                                }|                    |j
        |          }|                    dgdgddig           t          j        t                    5  |                    ddi           d	d	d	           n# 1 swxY w Y   |	                    |j
        
           t                                          dt                                }|                    |j
        |          }|                    dgdgddig           |                    ddi          }t          |d                   dhk    sJ d	S )z>Toggling inverted index enablement reflects in query behavior.rN   r  zno-indexr  global_fieldvaluer   r   Nr   r   r  r   zkey-disabledr   zkey-enabledr   )r   delete_indexr
   r   r  r.  r/  	Exceptionrx   r#  rG   r   r   r  )	ry   disabled_defaultsr   r   disabled_key_schemar%  restored_schemarestoredr>  s	            r3   $test_schema_delete_index_and_restorer    s    --5N5P5P-QQ4   J
 NNL'"G,-     
y	!	! 8 8ng67778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 */222 		688	9	9	*-F-H-H	I	I 
 //_%8 0  I MM''(     
y	!	! 2 2Z01112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 */222hh++8:: ,  O ..__ /  H LLO''(     \\V 4\55Fve}-000000s$   9BB"%B"%F

FFc                   	 t                                          dt                                }t          | |          \  }                    dgt          t          g dg          ddigdg           j        J j        j        d         j	        }|J |j
        }|J |j        d
u sJ ddi	dt          g t          f         dd	fd}	fd	fd	fdg}|D ]} ||           d	S )zDisabled metadata inverted index should block filter-based operations in get, query, and delete for local, single node, and distributed.restricted_tagr  r  restricted-docg?g?g333333?g?blockedr  r   r   r   r   NF	operationr7   c                     t          j        t                    5 } |              d d d            n# 1 swxY w Y   dt          |j                  v sJ d S Nz1Cannot filter using metadata key 'restricted_tag'r.  r/  r   rX   r  r  exc_infos     r3   _expect_disabled_errorzetest_disabled_metadata_index_filters_raise_invalid_argument_all_modes.<locals>._expect_disabled_error.      ]/00 	HIKKK	 	 	 	 	 	 	 	 	 	 	 	 	 	 	BcNG
 G
 
 
 
 
 
 
   155c                  0                                    S Nr   rx   r   filter_payloads   r3   <lambda>zWtest_disabled_metadata_index_filters_raise_invalid_argument_all_modes.<locals>.<lambda>6      
^44 r5   c                  `                          t          t          g dg          d          S Nr  r   )query_embeddings	n_resultsr   r   r(   r   r  s   r3   r  zWtest_disabled_metadata_index_filters_raise_invalid_argument_all_modes.<locals>.<lambda>7  9    
  !*/C/C/C.DEE  ! 
 
 r5   c                  0                                    S r  deleter  s   r3   r  zWtest_disabled_metadata_index_filters_raise_invalid_argument_all_modes.<locals>.<lambda><      
!!!77 r5   )r   r  r
   r   r  r(   r   r   r   r   r   r   r"   r!   )
ry   r   rA   schema_entryindex_configr  
operationsr  r   r  s
           @@r3   Etest_disabled_metadata_index_filters_raise_invalid_argument_all_modesr    s    XX""%>%@%@ #  F 00@PPPMJNN
%9%9%9$:;;$i01'	     ((($)*:;BL###5L###5((((&6	%BN
(2s7*; 
 
 
 
 
 	54444	
 	
 	
 	
 	

 	87777+J   * *	y))))* *r5   c                   	
 t                                          dt                                }t          | |          \  }                    dgt          t          g dg          ddigdg           j        J j        j        d         j	        }|J |j
        }|J |j        d
u sJ ddi	t          	          
dt          g t          f         dd	fd}	fd	fd
fd	fdg}|D ]} ||           d	S )zFDisabled metadata inverted index should block filter-based operations.r  r  r  r  r  r  r  r  NFr   r  r7   c                     t          j        t                    5 } |              d d d            n# 1 swxY w Y   dt          |j                  v sJ d S r  r  r  s     r3   r  z[test_disabled_metadata_index_filters_raise_invalid_argument.<locals>._expect_disabled_error^  r  r  c                  0                                    S r  r  r  s   r3   r  zMtest_disabled_metadata_index_filters_raise_invalid_argument.<locals>.<lambda>f  r  r5   c                  `                          t          t          g dg          d          S r  r  r  s   r3   r  zMtest_disabled_metadata_index_filters_raise_invalid_argument.<locals>.<lambda>g  r  r5   c                  .                                    S r/   )r>  )r   search_requests   r3   r  zMtest_disabled_metadata_index_filters_raise_invalid_argument.<locals>.<lambda>l  s    
!!.11 r5   c                  0                                    S r  r  r  s   r3   r  zMtest_disabled_metadata_index_filters_raise_invalid_argument.<locals>.<lambda>m  r  r5   )r   r  r
   r   r  r(   r   r   r   r   r   r   r    r"   r!   )ry   r   rA   r  r  r  r  r  r   r  r  s           @@@r3   ;test_disabled_metadata_index_filters_raise_invalid_argumentr  C  s   
 XX""%>%@%@ #  F 00@PPPMJNN
%9%9%9$:;;$i01'	     ((($)*:;BL###5L###5((((&6	%BN.111N
(2s7*; 
 
 
 
 
 	54444	
 	
 	
 	
 	

 	2111177777	+J   * *	y))))* *r5   c                    t          |           \  }}t          ||j                  }d}d t          |          D             }d t          |          D             }d t          |          D             }|                    |||           t
          st          ||j        |           |                    |j                  }|j        J d|j        j	        v sJ |j        j	        d         j
        }	|	J |	j        }
|
J |
j        du sJ t          ||j                  }d	}d
 t          |          D             }d t          |          D             }d t          |          D             }|                    |||           t
          st          ||j        |           |                    |j                  }|j        J d|j        j	        v sJ |j        j	        d         j
        }|J |j        }|J |j        du sJ |                    ddi          }t          |d                   dhk    sJ |                    ddi          }t          |d                   dhk    sJ |                                 }|                    |j                  }|j        J d|j        j	        v sJ d|j        j	        v sJ dS )zJCompaction promotes unseen metadata keys into discoverable schema entries.   c                     g | ]}d | S )zdiscover-add-r:   r<   s     r3   r?   zCtest_schema_discovers_new_keys_after_compaction.<locals>.<listcomp>}  s!    
:
:
:11
:
:
:r5   c                     g | ]}d | S zdoc r:   r<   s     r3   r?   zCtest_schema_discovers_new_keys_after_compaction.<locals>.<listcomp>~  s    777777r5   c                     g | ]	}d d| i
S )discover_addtopic_r:   r<   s     r3   r?   zCtest_schema_discovers_new_keys_after_compaction.<locals>.<listcomp>  0     * * *+,!&* * *r5   r   Nr  T  c                     g | ]}d | S )zdiscover-upsert-r:   r<   s     r3   r?   zCtest_schema_discovers_new_keys_after_compaction.<locals>.<listcomp>  s!    FFFQ(Q((FFFr5   c                     g | ]}d | S )zupsert doc r:   r<   s     r3   r?   zCtest_schema_discovers_new_keys_after_compaction.<locals>.<listcomp>  s!    BBB$$$BBBr5   c                     g | ]	}d d| i
S )discover_upsertr  r:   r<   s     r3   r?   zCtest_schema_discovers_new_keys_after_compaction.<locals>.<listcomp>  s1     1 1 1./	LQLL)1 1 1r5   r  topic_42r   r   zdiscover-add-42zdiscover-upsert-42)r   r   rG   rC   r  r   r   r   r   r   r   r   r   r  rx   r  r   )ry   r   r   initial_version
batch_sizer   r   r   r  discover_add_configr   next_versionupsert_count
upsert_idsupsert_docsupsert_metadataspost_upsertdiscover_upsert_configupsert_inverted_indexresultresult_upsertreload_client	persisteds                          r3   /test_schema_discovers_new_keys_after_compactionr  t  s]    55EFFJ,VZ_EEOJ
:
:j(9(9
:
:
:C77U:%6%6777I* *05j0A0A* * *I NNsi9NEEE! L!&*/?KKK$$Z_55H?&&&X_11111"/.~>E***/E ,,, (D0000)&*/BBLLFF%2E2EFFFJBBeL.A.ABBBK1 1383F3F1 1 1 "     " I!&*/<HHH''
88K))) 2 77777(/45FGN!---2H ,,, (D0000^^>:">^??Fve}"3!44444NN*;Z)HNIIM}U#$$)=(>>>>>$>>@@M,,Z_==I'''Y-22222	 0 5555555r5   c                    t          |           \  }}t          ||j                  }d t          d          D             }d t          d          D             }d t          d          D             }|                    |||           t
          st          ||j        |           |                    dgdgdd	ig           |                    d
gddig           |	                    |j                  j
        }|J d|j        v sJ |j        d         }|j        !|j        j        |j        j        j        du sJ |                    ddi          }	t!          |	d                   dhk    sJ |                    dgdg          }
|
d         J |
d         d         }|J t#          |d         t$          t&          f          sJ dS )zGConflicting value types should not corrupt discoverable schema entries.c                     g | ]}d | S )z	conflict-r:   r<   s     r3   r?   zJtest_schema_rejects_conflicting_discoverable_key_types.<locals>.<listcomp>  s    
/
/
/q?q??
/
/
/r5   r  c                     g | ]	}d d| i
S )conflict_keyvalue_r:   r<   s     r3   r?   zJtest_schema_rejects_conflicting_discoverable_key_types.<locals>.<listcomp>  r  r5   c                     g | ]}d | S r  r:   r<   s     r3   r?   zJtest_schema_rejects_conflicting_discoverable_key_types.<locals>.<listcomp>      000000r5   r   zconflict-badzbad docr  r   z
conflict-0   r   NTvalue_10r   r   zconflict-10r   r   r   )r   r   rG   rC   r  r   r   r  r  r   r   r   r   r   r   rx   r  r   rW   r;   )ry   r   r   r  r   r   r   r   conflict_entryfetchconflict_bad_metabad_metadatas               r3   6test_schema_rejects_conflicting_discoverable_key_typesr    s.    55EFFJ,VZ_EEO
/
/E#JJ
/
/
/C* *05c

* * *I 10U3ZZ000INNsi9NEEE! L!&*/?KKK+"C()     N"C()    
 "":?33:FV[(((([0N)!7C$:BdJJJJNN.*!=N>>EuU|////"N+;k]SS[)555$[1!4L###l>2S%LAAAAAAAr5   c                    t                      }|                    dt                                 t          | |          \  }}t	          ||j                  }d t          d          D             }d t          d          D             }d t          d          D             }|                    |||           t          ||j        |           |j	        J |j	        
                                }|j         d
}	|                    |	          }
|
j	        J |
j	        
                                |k    sJ t	          ||
j                  }d t          d          D             }d t          d          D             }d t          d          D             }|
                    |||           t          ||
j        |           |                    |
j                  }|j	        J d|j	        j        v sJ |j	        j        d         j        }|J |j        }|J |j        du sJ |                    |j                  }|j	        J d|j	        j        vsJ |                    ddi          }t'          |d                   dhk    sJ |
                    ddi          }t'          |d                   dhk    sJ d	S )zPAssert forked collections inherit schema and evolve independently of the parent.
shared_keyr  r  c                     g | ]}d | S )zparent-r:   r<   s     r3   r?   zEtest_collection_fork_inherits_and_isolates_schema.<locals>.<listcomp>  s    444A-A--444r5   r  c                     g | ]}d | S )zparent doc r:   r<   s     r3   r?   zEtest_collection_fork_inherits_and_isolates_schema.<locals>.<listcomp>  s!    999$$$999r5   c                     g | ]	}d d| i
S )r  parent_r:   r<   s     r3   r?   zEtest_collection_fork_inherits_and_isolates_schema.<locals>.<listcomp>  s0     1 1 1*+}}}%1 1 1r5   r   N_forkc                     g | ]}d | S )zfork-r:   r<   s     r3   r?   zEtest_collection_fork_inherits_and_isolates_schema.<locals>.<listcomp>  s    000000r5   c                     g | ]}d | S )z	fork doc r:   r<   s     r3   r?   zEtest_collection_fork_inherits_and_isolates_schema.<locals>.<listcomp>  s    555QQ555r5   c                     g | ]
}d d| dS )parentr  )r  
child_onlyr:   r<   s     r3   r?   zEtest_collection_fork_inherits_and_isolates_schema.<locals>.<listcomp>  s3     / / /ABx|||<</ / /r5   r  T	parent_10r   r   z	parent-10r  zfork-10)r   r   r
   r   r   rG   rC   r  r   r   r   forkr  r   r   r   r   r   rx   r  )ry   r   r   r   parent_version_before_add
parent_idsparent_docsparent_metadatasparent_schema_json	fork_nameforkedfork_versionfork_ids	fork_docsfork_metadatasupdated_childchild_only_configchild_inverted_indexreloaded_parentparent_resultschild_resultss                        r3   1test_collection_fork_inherits_and_isolates_schemar    s4    XXF
L1J1L1LMMM4  J
 !7vz O O44s444J99eCjj999K1 1/4Szz1 1 1 NN"     fjo7PQQQ(((#*<<>>?)))I__Y''F=$$$=**,,0BBBBB)&&+>>L00U3ZZ000H55%**555I/ /FKCjj/ / /N MMh)~MNNNffk<@@@))&+66M+++=/44444%,1,?F(((,B+++'4////++JO<<O!---5:::::$((k/J(KKN~e$%%+6666JJlJ%?J@@M}U#$$333333r5   c                    t                                          t          t          d                              }t	          | |t          d                    \  }}|                    dgdg           |                    dgd	g
          }|d	         J t          j        |d	         d         g d          sJ t                      }t                                          dt          d|                    }t	          | |          \  }}|                    dgdgddig            |dg          d         }|                    dgdg
          }	|	d         J |	d         d         }
|
J d|
v sJ t          t          |
d                   }||k    sJ |                    t                                          t!          dt          t"          |                                        }t%          |d                   dk    sJ d|d         d         v sJ |                    dgdgddig           |                    dgdg
          }|d         J |d         d         }|J d|vsJ dS )zKSchema-provided embedding functions drive both dense and sparse embeddings.r   rM   rO  rN   r   r   zembed-1zembedding documentr   r   r   r   Nr   rP  sparse_autotext_to_embedr4  r  r  zsparse-textr  zschema embeddingr   r   r;  r   r   zsparse-numeric)r   r   r   r+   r   r  rx   rV  rW  r   r   r(   r	   r>  r    r   r   r!   rf   )ry   vector_schemavector_collectionrA   embeddedr@  sparse_schemasparse_collectionsparse_querysparse_resultsparse_metarB  sparse_searchnumeric_metanumeric_metadatas                  r3   ,test_schema_embedding_configuration_enforcedr  3  s4   
 HH)) 4KPQ4R4R4RSSS *  M 72q999  q K'(    
 !$$)|n$MMHL!---;x-a02K2K2KLLLLL466IHH))&&(
 
 
 *  M 7  q
 O'#%789    
 9012215L%))}o})UUM%111,Q/K"""K'''',M(BCCN\))))%,,cm4^3L3LMMMNN M }U#$$))))M%033333'#Q'(     %((.>-?+(WWL$000#K03''' 0000000r5   c                 ,   t                                          t                                                    dt                                }t	          | |          \  }}d t          d          D             }d t          d          D             }d t          d          D             }t          ||j                  }|                    |||	           t          st          ||j        |           |                    |j                  j        }|J d|j        v sJ |j        d         j        }	|	J |	j        }
|
J |
j        du sJ d|j        v sJ |j        d         j        }|J |j        }|J |j        du sJ |                    ddi          }t%          |d                   t%          |          k    sJ t'          j        t*                    5  |                    ddi           d
d
d
           d
S # 1 swxY w Y   d
S )zLExplicit overrides take precedence over disabled defaults and discoverables.rN   explicit_keyr  r  c                     g | ]}d | S )zprecedence-r:   r<   s     r3   r?   zStest_schema_precedence_for_overrides_discoverables_and_defaults.<locals>.<listcomp>  s!    
1
1
1
1
1
1r5   r  c                     g | ]}d | S r  r:   r<   s     r3   r?   zStest_schema_precedence_for_overrides_discoverables_and_defaults.<locals>.<listcomp>  r  r5   c                     g | ]
}d d| dS )explicit	discover_)r  discover_keyr:   r<   s     r3   r?   zStest_schema_precedence_for_overrides_discoverables_and_defaults.<locals>.<listcomp>  s6     * * * $___EE* * *r5   r   NTr  Fr  r   r   
discover_5)r   r  r
   r   r   rC   r   rG   r  r   r   r   r   r   r   r   r   rx   r  r.  r/  r  )ry   r   r   r   r   r   r   r  schema_stateexplicit_key_stringexplicit_inverted_indexdiscover_key_stringdiscover_inverted_indexexplicit_results                 r3   ?test_schema_precedence_for_overrides_discoverables_and_defaultsr%  x  s   
 		688	9	9	.1J1L1L	M	M  5  J
 2
1eCjj
1
1
1C00U3ZZ000I* *s* * *I
 -VZ_EEONNsi9NEEE! L!&*/?KKK((99@L###\.....&+N;B***1G"..."*d2222\.....&+N;B***1G"..."*e3333 nnNJ+GnHHOu%&&#c((2222	y	!	! = =nl;<<<= = = = = = = = = = = = = = = = = =s   #H		HHc                 >   t          d          }t                                          dt          d|                    }t	          | |          \  }}|                    g dg d	
           |                    g ddg          }|d         J  |g d	          }t          g d          D ]h\  }}|d         |         }	|	J d|	v sJ t          t          |	d                   }
|
j
        ||         j
        k    sJ |
j        ||         j        k    sJ idS )zQTest sparse embedding auto-generation using #document as source with no metadata.doc_no_metar2  auto_sparser   r4  r  r  doc1doc2doc3)rb  test documentshortr  r   r   N)r   r   r   r   r   r  rx   r   r(   r	   r   r   )ry   r@  r   r   rA   r  expected_batchr>   doc_idrA  actuals              r3   ;test_sparse_auto_embedding_with_document_source_no_metadatar2    s   
 5=IIIIXX""&"(
 
 
 #  F 00@PPPMJ NN$$$;;;     ^^ 8 8 8;-^PPF+*** YHHHIIN77788 9 9	6+&q)###(((( lH]$;<<~!2!:::::}q 1 8888889 9r5   c                    t          d          }t                                          dt          d|                    }t	          | |          \  }}|                    dd	gd
dgddddddg           |                    dd	gdg          }|d         J |d         d         d         dk    sJ |d         d         d         dk    sJ |d         d         d         dk    sJ |d         d         d         dk    sJ d|d         d         v sJ d|d         d         v sJ  |d
dg          }t          t          |d         d         d                   }|j	        |d         j	        k    sJ dS )zKTest sparse embedding with #document source when metadata is also provided.doc_with_metar2  
doc_sparser   r4  r  r  m1m2r   r   test*   )r   r  prodc   r   r   r   Nr   r   r  r   
r   r   r   r   r   r  rx   r(   r	   r   )ry   r@  r   r   rA   r  r/  	actual_m1s           r3   <test_sparse_auto_embedding_with_document_source_and_metadatar>    s   
 5?KKKIXX""&"(
 
 
 #  F 00@PPPMJ NN4LF#"--"--
     ^^t{m^DDF+*** +q!*-7777+q!'*b0000+q!*-7777+q!'*b0000 6+.q111116+.q11111 Y011N\6+#6q#9,#GHHIq 1 9999999r5   c                 T   t          d          }t                                          dt          d|                    }t	          | |          \  }}|                    g dg d	dd
iddiddig           |                    g ddg          }|d         J  |g d          }t          d          D ]k}|d         |         }|J d|v sJ d|v sJ t          t          |d                   }	|	j
        ||         j
        k    sJ |	j        ||         j        k    sJ ldS )z7Test sparse embedding using a metadata field as source.meta_sourcer2  content_sparsecontentr4  r  r  s1s2s3r)  sparse content onesparse content twosparse content threer   r   r   N)rG  rH  rI  r   )r   r   r   r   r   r  rx   rC   r(   r	   r   r   
ry   r@  r   r   rA   r  r/  r>   rA  r1  s
             r3   3test_sparse_auto_embedding_with_metadata_source_keyrK    s   
 5=IIIIXX""& (
 
 
 #  F 00@PPPMJ NN***,-,-./
     ^^ 2 2 2[M^JJF+*** YLLL N 1XX 9 9+&q)###8++++ H$$$$ lH-=$>??~!2!:::::}q 1 8888889 9r5   c                 <   t          d          }t                                          dt          d|                    }t	          | |          \  }}|                    g dg d	d
d
ddid
g           |                    g ddg          }|d         J  |g d	          }t          |d                   D ]B\  }}|J d|v sJ t          t          |d                   }	|	j
        ||         j
        k    sJ C|d         d         d         dk    sJ d
S )z@Test sparse embedding with mixed metadata (None, empty, filled).
mixed_metar2  mixed_sparser   r4  r  r  )n1n2n3n4)zdoc onezdoc twoz	doc threezdoc fourNexistingdatar   r   r   rq  )r   r   r   r   r   r  rx   r   r(   r	   r   rJ  s
             r3   >test_sparse_auto_embedding_with_mixed_metadata_none_and_filledrU  4  s   
 5<HHHIXX""&"(
 
 
 #  F 00@PPPMJ NN$$$AAA 	
  	 	 	 ^^ 8 8 8;-^PPF+*** YNNNOON !!455 ; ;8###)))) lH^$<==~!2!:::::: +q!*-777777r5   c                    t          d          }t                                          dt          d|                    }t	          | |          \  }}t          dgd	g
          }|                    ddgddgdd|ig           |                    ddgdg          }|d         J |d         d         }|J d|v sJ  |dg          d         }t          t
          |d                   }	|	j	        |j	        k    sJ |d         d         }
|
J t          t
          |
d                   }|j	        dgk    sJ |j
        d	gk    sJ dS )zBTest that sparse auto-embedding doesn't overwrite existing values.preserver2  preserve_sparser   r4  r  r  i  gw/^@r   	preserve1	preserve2zauto documentzmanual documentNr   r   r   r   r   )r   r   r   r   r   r	   r  rx   r(   r   r   )ry   r@  r   r   rA   existing_sparser  	auto_metaexpected_autoactual_automanual_metaactual_manuals               r3   0test_sparse_auto_embedding_skips_existing_valuesra  d  s   
 5:FFFIXX""&"(
 
 
 #  F 00@PPPMJ #C5'CCCONN+&"$560
     ^^k :[M^RRF+*** {#A&I   	))))I/003M|Y/@%ABBK-"77777 %a(K"""{3D'EFFM SE))))G9,,,,,,r5   c                 h   t          d          }t                                          dt          d|                    }t	          | |          \  }}|                    g dg d	dd
iddiddidg           |                    g ddg          }|d         J d|d         d         v sJ  |d
g          d         }t          t          |d         d         d                   }|j	        |j	        k    sJ d|d         d         vsJ d|d         d         vsJ |d         d         J dS )zJTest sparse embedding when source metadata field is missing or wrong type.missing_fieldr2  field_sparse
text_fieldr4  r  r  )f1f2f3f4)r*  r+  r,  doc4z
valid text{   other_fieldr  Nr   r   r   r   r   rq  r   r<  )ry   r@  r   r   rA   r  expectedr1  s           r3   4test_sparse_auto_embedding_with_missing_source_fieldrn    s   
 5?KKKIXX""&#(
 
 
 #  F 00@PPPMJNN$$$222<(3G$	
  	 	 	 ^^ 8 8 8;-^PPF+*** VK033333y,((+H,{ 3A 6~ FGGF>X----- !4Q!77777!4Q!77777+q!)))))r5   c                    t          d          }t                      }|                    dt                                 |                    dt	          d|                     t          | |          \  }}|                    d	d
gddgddddddg           |                    d	d
gdg          }|d         J  |ddg          }t          |d                   D ]H\  }}|J d|v sJ t          t          |d                   }	|	j        ||         j        k    sJ d|v sJ I|                    ddi          }
t          |
d                   d	hk    sJ dS )zCTest sparse auto-embedding works alongside string inverted indexes.with_string_indexr2  r   r  sparse_fieldcustom_textr4  r  multi1multi2zmain documentzanother documentzfield contentcat1)rr  r   zdifferent contentcat2r   r   r   Nr   r   )r   r   r   r
   r   r   r  rx   r   r(   r	   r   r  )ry   r@  r   r   rA   r  r/  r>   rA  actual_fieldr  s              r3   5test_sparse_auto_embedding_with_string_inverted_indexrx    s   
 5;NOOOIXXF
(**     &$(
 
 
     00@PPPMJNNx "$67+@@/VDD
     ^^8 4{m^LLF+*** Y1DEFFN !455 & &8### )))) L(>*BCC#~a'8'@@@@@ X%%%%% ~~Z$8~99HxH:------r5   c                 V   t          d          }t          d          }t                      }|                    t	          |                     |                    dt          d|	          
           t          | ||          \  }}|                    ddgddg           |                    ddgddg          }|d         J t          |d                   dk    sJ t          |d         d                   dk    sJ |d         J |d         D ]}|J d|v sJ dS )z9Test that dense and sparse auto-embeddings work together.r,   rM   
with_denser2  rO  rN   
sparse_keyr   r4  r  r  both1both2zcombined documentzanother docr  r   r   r   Nrq  r   )
r+   r   r   r   r   r   r   r  rx   rf   )ry   dense_efr@  r   r   rA   r  rA  s           r3   .test_dense_and_sparse_auto_embeddings_togetherr    s   
 '1---H4<HHHIXXF
0HMMMNNN
&"(
 
 
     0#  MJ NNg&6    
 ^^g{+   F ,+++vl#$$))))vl#A&''1,,,, +***;' ( (###x'''''( (r5   c                    t          d          }t                                          dt          d|                    }t	          | |          \  }}|                    dgd	g
           |                    dgdg
           |                    dgddg          }|d         J |d         J |d         d         dk    sJ d|d         d         v sJ  |dg          d         }t          t          |d         d         d                   }|j
        |j
        k    sJ |                    dgdg
           |                    dgdg          }|d         J d|d         d         v sJ  |dg          d         }	t          t          |d         d         d                   }
|
j
        |	j
        k    sJ dS )z=Test sparse auto-embedding with update and upsert operations.update_upsertr2  update_sparser   r4  r  r  up1zoriginal docr  zupdated docr   r   r   Nr   up2zupserted doc)r   r   r   r   r   r  r  rx   r(   r	   r   r  )ry   r@  r   r   rA   result_updaterm  r1  r  expected_upsertactual_upserts              r3   1test_sparse_auto_embedding_with_update_and_upsertr  $  s>   
 5?KKKIXX""&"(
 
 
 #  F 00@PPPMJ NNG!"     G /    
 NNwk8RNSSM%111%111%a(M9999mK8;;;;; y-))!,H,k :1 =o NOOF>X----- G!"    
 NNwNFFM%111mK8;;;;;  i 011!4O}['A!'D_'UVVM O$;;;;;;;r5   c                 :   t          d          }t                                          dt          d|                    }t	          | |          \  }}|j        }|                    dgd	g
           |                                 }|                    |          }|j	        J d|j	        j
        v sJ |                    dgdg
           |                    ddgdg          }|d         J d|d         d         v sJ d|d         d         v sJ dS )zFTest that sparse auto-embedding config persists across client reloads.persist_testr2  persist_sparser   r4  r  r  persist1zpersistent documentr  r   Npersist2znew document after reloadr   r   r   r   )r   r   r   r   r   rG   r  r   r   r   r   rx   )	ry   r@  r   r   r   r   rL  r   r  s	            r3   ;test_sparse_auto_embedding_persistence_across_client_reloadr  \  s   
 5>JJJIXX""&"(
 
 
 #  F 55EfUUUJ oONNL()     '@@BBO)88 9  
 %11129>>>>> L./     !$$$ %  F +***vk2155555vk215555555r5   c                 b   t          d          }t                                          dt          d|                    }t	          | |          \  }}d}d	 t          |          D             }d
 t          |          D             }|                    ||           |                    |dd         dg          }|d         J  ||          }	t          |d                   D ]B\  }
}|J d|v sJ t          t          |d                   }|j        |	|
         j        k    sJ CdS )z9Test sparse auto-embedding with large batch of documents.
batch_testr2  batch_sparser   r4  r  r  r   c                     g | ]}d | S )zbatch-r:   r<   s     r3   r?   zDtest_sparse_auto_embedding_with_batch_operations.<locals>.<listcomp>  s    
3
3
3A<A<<
3
3
3r5   c                     g | ]}d | S )zdocument number r:   r<   s     r3   r?   zDtest_sparse_auto_embedding_with_batch_operations.<locals>.<listcomp>  s!    CCCA'A''CCCr5   r  Nr  r   r   )r   r   r   r   r   rC   r  rx   r   r(   r	   r   )ry   r@  r   r   rA   r  r   r   r  r/  r>   rA  r1  s                r3   0test_sparse_auto_embedding_with_batch_operationsr    su   
 5<HHHIXX""&"(
 
 
 #  F 00@PPPMJ J
3
3z!2!2
3
3
3CCCz1B1BCCCINNsiN000 ^^CRC;-^@@F+*** Yy))N !455 ; ;8###))))lH^$<==~!2!::::::; ;r5   c                 Z   t          d          }t                                          dt          d|                    }t	          | |          \  }}|                    dgd	g
           |                    dgdg          }|d         J |d         d         }|J d|v sJ dS )zCTest sparse auto-embedding handles empty/None documents gracefully.
empty_testr2  empty_sparser   r4  r  r  empty1 r  r   r   Nr   )r   r   r   r   r   r  rx   )ry   r@  r   r   rA   r  rA  s          r3   /test_sparse_auto_embedding_with_empty_documentsr    s    
 5<HHHIXX""&"(
 
 
 #  F 00@PPPMJ NNJ$    
 ^^
[M^BBF+*** k"1%HX%%%%%%r5   c                 d   t          |           \  }}|j        }|J d|j        v sJ |j        d         j        }|J |j        }|J |j        du sJ |j        J |j        j        J |j        j        dk    sJ |j        j        }|J |	                                dk    sJ |j
        }|J |                    d          }|J |	                                dk    sJ |                                }	|	d         d         d         d	         }
|
d
         d         }|d         dk    sJ |d         dk    sJ |	d         d         d	         }|d
         d         }|d         dk    sJ |d         dk    sJ |	d         d         d         }|J |d
         d         }|d         dk    sJ dS )zbVerify that when no schema is provided, the collection uses DefaultEmbeddingFunction (not legacy).Nr   Tl2r\   r   r   r   r   rK   typeknownrG   r   sparse_vectorsparse_vector_indexunknown)r   r   r   r   r   r   rK   r   r   rG   r   rx   r   )ry   r   r   r   r   r   r   rK   r   	json_dataembedding_vector	ef_configdefaults_vectordefaults_ef_configdefaults_sparsesparse_ef_configs                   r3   7test_default_embedding_function_when_no_schema_providedr    sJ   
 55EFFJ F 6;&&&&\2=)))%2L###4''''***$000$,,,, 
		/B>>> 7799	!!!!%F

/00I   >>y(((( ((**I (6|D^T *+?@I V''''V	))))  
+L9.IO(23GHf%0000f%2222  
+O<=RSO&&&&x01EFF#y000000r5   c                    |                                  }|                                 dt                      j         }t	          d          }|                    ||           |                    ||          }|j        }|J d|j        v sJ |j        d         j	        }|J |j
        }|J |j        du sJ |j        J |j        j        J |j        j        dk    sJ |j        j        }|J |                                d	k    sJ |                                d
dik    sJ |                                }	|	d         d         d         d         }
|
d         d         }|d         dk    sJ |d         d	k    sJ |d         d
dik    sJ |	d         d         d         }|d         d         }|d         dk    sJ |d         d	k    sJ |d         d
dik    sJ |                    dgdg           |                    dgdg          }|d         J t)          |d         d                   dk    sJ dS )zXVerify that custom embedding function is reflected in schema when no schema is provided.custom_ef_no_schema_   rM   r   Nr   TrQ   rF   r-   r   r   r   rK   r   r  r  rG   r   test1r-  r  r   r   r   )r   r   r)   r   r+   r$  r   r   r   r   r   r   rK   r   r   rG   rJ   r   r  rx   rf   ry   r   r   	custom_efr   r   r   r   r   r  r  r  r  r  r  s                  r3   -test_custom_embedding_function_without_schemar    s    7799F
LLNNN:UWW[::O (A...I $     &&$ '  J F 6;&&&&\2=)))%2L###4''''***$000$0000 
		/B>>> 7799####==??uaj(((( ((**I (6|D^T *+?@I V''''V++++X5!*,,,,  
+L9.IO(23GHf%0000f%4444h'E1:5555 NNI"#    
 ^^	L>^BBF,+++vl#A&''1,,,,,,r5   c                    |                                  }|                                 dt                      j         }t	          d          }|                    ||t                                 |                    ||          }|j        }|J d|j	        v sJ |j	        d         j
        }|J |j        }|J |j        du sJ |j        J |j        j        J |j        j        d	k    sJ |j        j        }|J |                                d
k    sJ |                                ddik    sJ |                                }	|	d         d         d         d         }
|
d         d         }|d         dk    sJ |d         d
k    sJ |d         ddik    sJ |	d         d         d         }|d         d         }|d         dk    sJ |d         d
k    sJ |d         ddik    sJ |                    dgdg           |                    dgdg          }|d         J t+          |d         d                   dk    sJ dS )z_Verify that custom embedding function is reflected in schema when default Schema() is provided.custom_ef_default_schema_r  rM   rG   r   r   r   Nr   TrQ   rF   r-   r   r   r   rK   r   r  r  rG   r   r  r-  r  r   r   r   )r   r   r)   r   r+   r$  r   r   r   r   r   r   r   rK   r   r   rG   rJ   r   r  rx   rf   r  s                  r3   2test_custom_embedding_function_with_default_schemar  Y  s    7799F
LLNNN?%''+??O (A...I $xx     &&$ '  J F 6;&&&&\2=)))%2L###4''''***$000$0000 
		/B>>> 7799####==??uaj(((( ((**I (6|D^T *+?@I V''''V++++X5!*,,,,  
+L9.IO(23GHf%0000f%4444h'E1:5555 NNI"#    
 ^^	L>^BBF,+++vl#A&''1,,,,,,r5   c                 <   |                                  }|                                 dt                      j         }t	          d          }t	          d          }t                                          t          |                    }t          j	        t                    5 }|                    |||           ddd           n# 1 swxY w Y   t          |j                  }d	|                                v sd
|                                v sJ dS dS )zTVerify that setting different custom embedding functions in schema and config fails.conflict_ef_r,   rM   r   rO  rN   r  Nr   conflict)r   r   r)   r   r+   r   r   r   r.  r/  r  r$  rX   r  lower)ry   r   r   r   	schema_efr   r  	error_msgs           r3   ?test_conflicting_embedding_functions_in_schema_and_config_failsr    s[    7799F
LLNNN2UWW[22O (A...I'A...I XX"" I>>> #  F 
y	!	! 
X   ( 	! 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 HN##Iy((((J)//:K:K,K,K,K,K((,K,Ks   (CCCc                 N   t                      }|                    t                      d           |                    t                      t	          d                     t          | |          \  }}|j        J d|j        j        v sJ d|j        j        v sJ |                    dgdgdd	d
g           |	                    ddi          }t          |d                   dhk    sJ |	                    dd	i          }t          |d                   dhk    sJ dS )z6Test that create_index accepts both str and Key types.	field_strrK   r  	field_keyr  Nz
key-test-1test docvalue1r9  )r  r  r   r   r   )r   r   r
   r   r   r   r   r   r  rx   r  )ry   r   r   rA   
str_result
int_results         r3   test_create_index_with_key_typer    sZ    XXF
8::LLL
577S=M=MNNN/0@PPPMJ (((*+00000*+00000 NNN,!);;<     {H&=>>Jz% !!l^3333 {B&788Jz% !!l^333333r5   c                    t                      }|                    t                      d           |                    t                      t	          d                     t          | |          \  }}|j        J d|j        j        v sJ d|j        j        v sJ |                    dgdgdd	d
g           t          j
        t                    5  |                    ddi           ddd           n# 1 swxY w Y   t          j
        t                    5  |                    dd	i           ddd           dS # 1 swxY w Y   dS )z6Test that delete_index accepts both str and Key types.disabled_strr  disabled_keyr  Nzdisable-test-1r  r  r   )r  r  r   r   )r   r  r
   r   r   r   r   r   r  r.  r/  r  rx   )ry   r   r   rA   s       r3   test_delete_index_with_key_typer    s    XXF
8::OOO
577S=P=PQQQ/0@PPPMJ (((Z.33333Z.33333 NN,$+SAAB     
y	!	! 8 8ng67778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 
y	!	! 4 4nc23334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4s$   C;;C?C?EE	E	c                 ~   t                      }|                    t          t          j        t          d                               t          | |t          d                    \  }}|j        J |j        j        d         j	        }|J |j
        J |j
        j        j        J |j
        j        j        dk    sJ |j
        j        j        d	k    sJ |                    g d
g d           |                    dgdg          }|d         J t!          |d         d                   dk    sJ |                    t%                                          t)          g dd                              }t!          |d                   dk    sJ t!          |d         d                   dk    sJ dS )z<Test that VectorIndexConfig source_key accepts Key.DOCUMENT.r   rM   r4  rN   r  Nr   rQ   r   )vec-1zvec-2zvec-3)zapple fruitzbanana fruitzcar vehicler  r  r   r   r   rP  rq  )r   limitr   )r   r   r   r   DOCUMENTr+   r   r   r   r   r   rK   r   r5  r  rx   rf   r>  r    r   r   )ry   r   r   rA   embedding_configr  rC  s          r3   1test_vector_index_config_with_key_document_sourcer    s   
 XXF
 |61===
 
 
     02q999  MJ (((!(-l;F'''(444(/5AAA(/5AAAA(/:kIIII NN'''@@@     ^^	L>^BBF,+++vl#A&''1,,,, %%c 9 9 9CCCDD M }U#$$q((((}U#A&''1,,,,,,r5   c                    t          d          }t                                          dt          t          j        |                    }t          | |          \  }}|j        J |j        j        d         j	        }|J |j
        J |j
        j        j        dk    sJ |                    g d	g d
           |                    dgdg          }|d         J d|d         d         v sJ t          t           |d         d         d                   }|                    t%                                          t)          dt          t*          |          d                              }t-          |d                   dk    sJ d|d         d         v sJ t                                          dt          t	          d          |                    }	t          | |	          \  }
}|
j        J |
j        j        d         j	        }|J |j
        J |j
        j        j        dk    sJ |
                    ddgddgddiddig           |
                    ddgdg          }|d         J d|d         d         v sJ d|d         d         v sJ t          t           |d         d         d                   }|
                    t%                                          t)          dt          t*          |          d                              }t-          |d                   dk    sJ d|d         d         v sJ dS )zLTest that SparseVectorIndexConfig source_key accepts both str and Key types.	key_typesr2  sparse1r4  r  r  Nr   rC  )applebananaoranger  rD  r   r   r   rq  )r  r   r  r   sparse2re  zsparse-key-1zsparse-key-2r*  r+  zcontent onezcontent twor   r   )r   r   r   r   r   r  r   r   r   r  r  rK   r5  r  rx   r(   r	   r>  r    r   r   r!   rf   )ry   r@  schema1collection1rA   sparse1_configresult1
sparse_vecsearch_result1schema2collection2sparse2_configresult2r  search_result2s                  r3   .test_sparse_vector_index_config_with_key_typesr  6  s   
 5;GGGI hh##&|(
 
 
 $  G 11A'RRRNK))) ',Y7EN%%%-999-4?;NNNN OO///     oo4&;-o@@G;+++,Q/////lGK$8$;I$FGGJ !''citC/D/DANNNOO N ~e$%%))))>%(+++++ hh##&<(((
 
 
 $  G 11A'RRRNK))) ',Y7EN%%%-999-4?<OOOO OO^,6" -0<2OP     oo^,{m   G ;+++,Q/////,Q///// gk&:1&=i&HIIL !''citC/F/FaPPPQQ N ~e$%%))))^E215555555r5   c                     t                      } t          j        t          d          5  |                     t                      d           ddd           n# 1 swxY w Y   t          j        t          d          5  |                     t                      t          j                   ddd           dS # 1 swxY w Y   dS )zITest that create_index rejects keys starting with # (except system keys).key cannot begin with '#'r,  #custom_fieldr  Nz/Cannot create index on special key '#embedding')r   r.  r/  r?  r   r
   r   	EMBEDDINGr  s    r3   /test_schema_rejects_special_key_in_create_indexr    sd    XXF	z)D	E	E	E U U#<#>#>OTTTU U U U U U U U U U U U U U U 
K
 
 
 S S 	#<#>#>CMRRRS S S S S S S S S S S S S S S S S S#   $AA!A .B;;B?B?c                     t                      } t          j        t          d          5  |                     t                      d           ddd           n# 1 swxY w Y   t          j        t          d          5  |                     t                      t          j                   ddd           dS # 1 swxY w Y   dS )zITest that delete_index rejects keys starting with # (except system keys).r  r,  r  r  Nz.Cannot delete index on special key '#document')r   r.  r/  r?  r  r
   r   r  r  s    r3   /test_schema_rejects_special_key_in_delete_indexr    sd    XXF	z)D	E	E	E U U#<#>#>OTTTU U U U U U U U U U U U U U U 
J
 
 
 R R 	#<#>#>CLQQQR R R R R R R R R R R R R R R R R Rr  c                     t          j        t          d          5  t          d           ddd           n# 1 swxY w Y   t          j        t          d          5  t          t          j                   ddd           n# 1 swxY w Y   t          j        t          d          5  t          d           ddd           n# 1 swxY w Y   t          j        t          d          5  t          t          j                   ddd           n# 1 swxY w Y   t          j        t          d          5  t          d           ddd           dS # 1 swxY w Y   dS )z=Test that config validators reject invalid source_key values.z source_key cannot begin with '#'r,  r   r=  Nz	#metadata)r.  r/  r?  r   r   r  r   r:   r5   r3   1test_schema_rejects_invalid_source_key_in_configsr    s    
z)K	L	L	L 3 3\22223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 
z)K	L	L	L 4 4S]33334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 
z)K	L	L	L 9 9<88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 
z)K	L	L	L : :3=9999: : : : : : : : : : : : : : : 
z)K	L	L	L 8 8;77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8sU   9= =BB
B
,C		CC/DDD<EE!Ec           	      <   |                                  }|                                 dt                      j         }t	                      }ddlm}m}m}m	}  | | |d |                                          |j
        d<   t          j        t                    5 }|                    ||	           d
d
d
           n# 1 swxY w Y   t          |j                  }	d|	v s.d|	                                v sd|	                                v sJ d
S d
S d
S )zKTest that server-side validation rejects schemas with invalid special keys.server_validate_r   )
ValueTypesStringValueTypeStringInvertedIndexTyper
   Tr   rK   )r   )r   z#invalid_keyr   N#r  invalid)r   r   r)   r   r   chromadb.api.typesr  r  r  r
   r   r.  r/  r  r$  rX   r  r  )
ry   r   r   r   r  r  r  r
   r  r  s
             r3   .test_server_validates_schema_with_special_keysr    s   
 7799F
LLNNN666O XXF            #-*"9"90022# # #
 
 
# # #FK 
y	!	! FX  of EEEF F F F F F F F F F F F F F F HN##IyEY__%6%666)yGXGX:X:X:X:X66:X:Xs   #CCCc                 V   |                                  }|                                 dt                      j         }ddlm}m}m} t                      }t          j
        ddd          } | | |d|          	          
          |j        d<   t          j        t                    5 }|                    ||           ddd           n# 1 swxY w Y   t!          |j                  }	d|	                                v sd|	v sd|	                                v sJ dS dS dS )zWTest that server-side validation rejects invalid source_key in SparseVectorIndexConfig.server_source_key_r   )r  SparseVectorValueTypeSparseVectorIndexTypeNr   )r   r5  bm25Tr  )r  )r  test_sparser   r5  r  document)r   r   r)   r   r  r  r  r  r   r   model_constructr   r.  r/  r  r$  rX   r  r  )
ry   r   r   r  r  r  r   invalid_configr  r  s
             r3   @test_server_validates_invalid_source_key_in_sparse_vector_configr    s   
 7799F
LLNNN8577;88O          XXF,<  N ",++ 5 5%! ! !
 
 
" " "FK 
y	!	! FX  of EEEF F F F F F F F F F F F F F F HN##I	)))))****** 	*)**s   0CCCr   c                 `   dt                       }|                     |          }|j        }|J |j        j        J |j        j        j        J |j        j        j        j        j        J |                    dddii           |j        }|J |j        j        j        j        j        j	        dk    sJ |j
        d         j        j        j        j        j	        dk    sJ |                     |          }|j        }|J |j        j        j        j        j        j	        dk    sJ dS )	bTest that modifying a collection without initial config/schema creates and updates default schema.test_modify_no_init_r   Nr   r   r~   r   r   )r)   r$  r   r   r   r   rK   r   modifyr   r   r   r   r   r   r   updated_schemacollection_refreshedrefreshed_schemas          r3   ?test_modify_collection_no_initial_config_creates_default_schemar	  	  s]   
 7UWW66O )))??J F?%111?%2>>>?%29?KKK W.C$DEEE  &N%%%"-:AGUY[[[[[|,7DKQ_ceeeee "00AA+2'''$/<CIW[]]]]]]]r5   zSPANN is disabledc                 `   dt                       }|                     |          }|j        }|J |j        j        J |j        j        j        J |j        j        j        j        j        J |                    dddii           |j        }|J |j        j        j        j        j        j	        dk    sJ |j
        d         j        j        j        j        j	        dk    sJ |                     |          }|j        }|J |j        j        j        j        j        j	        dk    sJ dS )	r  r  r   Nr   r   r   r  r   )r)   r$  r   r   r   r   rK   r   r  r   r   r   r  s          r3   Etest_modify_collection_no_initial_config_creates_default_schema_localr  5	  s]   
 7UWW66O )))??J F?%111?%2>>>?%29>JJJ Vk3-?$@AAA  &N%%%"-:AFPTWWWWW|,7DKPZ^aaaaa "00AA+2'''$/<CHRVYYYYYYYr5   c                    dt                       }|                     |ddddi          }|j        }|J |j        j        j        j        j        J |j        j        j        j        j        j        dk    sJ |j        j        j        j        j        j	        dk    sJ |
                    ddd	ii
           |j        }|J |j        j        j        j        j        j        d	k    sJ |j        j        j        j        j        j	        dk    sJ |j        d         j        j        j        j        j        d	k    sJ |j        d         j        j        j        j        j	        dk    sJ |                     |          }|j        }|J |j        j        j        j        j        j        d	k    sJ |j        j        j        j        j        j	        dk    sJ dS )zQTest creating collection with SPANN schema within server limits and modifying it.test_modify_with_schema_r   r  2   r   r   rG   r   Nr      r  r   )r)   r$  r   r   r   r   rK   r   r   r   r  r   r   r  s          r3   0test_modify_collection_with_initial_spann_schemar  W	  s    ;::O ))!# 
 *  J F?%29?KKK?%29?MQSSSSS?%29?IROOOO W.C$DEEE  &N%%%"-:AGUY[[[[["-:AGQUWWWWW |,7DKQ_ceeeee|,7DKQ[_aaaaa "00AA+2'''$/<CIW[]]]]]$/<CISWYYYYYYYr5   c                 d   dt                       }|                     |ddddi          }|                    ddddi	           |j        }|J |j        j        j        j        j        j	        dk    sJ |j        j        j        j        j        j
        dk    sJ |j        j        j        j        j        j        dk    sJ |j        d         j        j        j        j        j	        dk    sJ |j        d         j        j        j        j        j
        dk    sJ |                     |          }|j        }|J |j        j        j        j        j        j	        dk    sJ |j        j        j        j        j        j
        dk    sJ |j        j        j        j        j        j        dk    sJ d
S )zCTest that modifying multiple SPANN fields updates schema correctly.test_modify_schema_multi_r   @   r   r  r     r  r  Nr~   r   )r)   r$  r  r   r   r   r   rK   r   r   r   r   r   r   )r   r   r   r   r  r  s         r3   ;test_modify_collection_updates_schema_spann_multiple_fieldsr  	  s   
 <%'';;O ))"3 G GH *  J !$  
     F?%29?MQTTTTT?%29?ISPPPP ?%29?LPRRRRR ;|$/<CIW[^^^^^;|$/<CISWZZZZZ "00AA+2'''$/<CIW[^^^^^$/<CISWZZZZZ$/<CIVZ\\\\\\\r5   c                 ^   dt                       }|                     |dddii          }|j        }|J |j        j        J |j        j        J |j        j        J |j        j        J |j        j        J |j	        d         J |
                    dddii	           |j        }|J |j        j        J |j        j        j        J |j        j        j        J |j        j        J |j        j        j        J |j        j        J |j        j        j        J |j        j        J |j        j        j        J |j        j        J |j        j        j        J |j	        d         J |j	        d         j        J |j	        d         j        j        J |j	        d         j        j        j        d
u sJ |j        j        j        j        j        j        dk    sJ |                     |          }|j        }|J |j        j        j        j        j        j        dk    sJ |j        j        J |j        j        j        J |j        j        j        J |j        j        J |j        j        J |j        j        J |j        j        J |j	        d         J dS )zJTest that modifying configuration doesn't affect other schema value types.test_modify_schema_preserve_r   r   r  r  Nr   r  r  T)r)   r$  r   r   r   r   r   r   r  r   r  r   r  r   r   r  r  r   r   r   rK   r   r   r   )r   r   r   initial_schemar  r  r  s          r3   4test_modify_collection_preserves_other_schema_fieldsr  	  sX    ?UWW>>O ))" 56 *  J  &N%%%")555",888".:::"*666"0<<<{+777 W.D$EFFF  &N%%% ")555")?KKK")3???",888",?KKK".:::".COOO"*666"*>JJJ"0<<<"0DPPP {+777{+2>>>{+2<HHH{+2<DLLLL "-:AGUY\\\\\ "00AA+2''' $/<CIW[^^^^^ $+777$+AMMM$+5AAA$.:::$0<<<$,888$2>>> -99999r5   c                     t                                          t          t                                          } t	          t                      di |                                 dddddd	
  
        }t          t          t          t                                |d
          }|j        J |j        J |j                            d          J |                    dg          }|J t          j        |d         g d          sJ dS )z=Test that embeddings are using the schema embedding function.rO  rN   schema_only_collectionNr,   tenantdatabaser   )
idrG   configuration_jsonserialized_schemarA  	dimensionr  r  versionlog_position)r   modelr   r   rb  )rQ  rR  rS  rT  )r   r   r   r+   CollectionModelr)   r   r   r(   r   object_embedding_functionr   rx   rU  rV  rW  )r   collection_modelr   r   s       r3   +test_embeds_using_schema_embedding_functionr+  	  s/   XX"" 4K4M4MNNN #  F '77% 2244   "Ivxx((  J )111#///#''(<==EEE""M?33J!!!;z!}&:&:&:;;;;;;;r5   c                       e Zd ZdZddefdZdee         dee         fdZdee         dee         fdZ	e
defd	            Zdeeef         fd
Ze
deeef         dd fd            ZdS )TestSparseEmbeddingFunctionzESparse embedding function for testing search API with string queries.r  r]   c                     || _         d S r/   ru   rb   s     r3   r4   z$TestSparseEmbeddingFunction.__init__ 
  r   r5   r6   r7   c                 4    d t          |          D             S )Nc           
      t    g | ]5\  }}t          |gt          t          |          |z             g           6S r   r   r   s      r3   r?   z8TestSparseEmbeddingFunction.__call__.<locals>.<listcomp>$
  r   r5   r   r   s     r3   rD   z$TestSparseEmbeddingFunction.__call__#
  r   r5   c                 4    d t          |          D             S )Nc           
      z    g | ]8\  }}t          |gt          t          |          |z   d z             g          9S )r   r   r   r   s      r3   r?   z;TestSparseEmbeddingFunction.embed_query.<locals>.<listcomp>*
  sU     
 
 
T #c$ii#o6I0J0J/KLLL
 
 
r5   r   r   s     r3   rp   z'TestSparseEmbeddingFunction.embed_query)
  r   r5   c                      dS )Ntest_sparse_efr:   r:   r5   r3   rG   z TestSparseEmbeddingFunction.name/
  s    r5   c                     d| j         iS rt   ru   rI   s    r3   rJ   z&TestSparseEmbeddingFunction.get_config3
  rv   r5   rK   c                 H    t          |                     dd                    S )Nr]   r  )r-  rx   rN   s    r3   rO   z-TestSparseEmbeddingFunction.build_from_config6
  s    *6::g}+M+MNNNr5   N)r  )rS   rT   rU   rV   rX   r4   r$   r	   rD   rp   rY   rG   r#   r!   rJ   rO   r:   r5   r3   r-  r-  
  s       OO c    
d3i 
D,> 
 
 
 

c 
tL/A 
 
 
 
  #       \ &DcN & & & & O$sCx. O5R O O O \O O Or5   r-  c                    t          d          }t                                          dt          t          j        |                    }t          | |          \  }}|                    ddgd	d
g           |                                }|J |d         J t          |d                   dk    sJ t                                          t          dd	                    }|                    |          }|d         J t          |d                   dk    sJ t          |d         d                   dk    sJ d|d         d         v sJ dS )zMTest that search API uses embedded_searches when string queries are embedded.search_testr2  rq  r4  r  r  r*  r+  rb  r-  r  Nr   rq  r;  r   r   r   )r-  r   r   r   r   r  r   r  rx   rf   r    r   r   r>  )ry   r@  r   r   rA   r  r>  resultss           r3   =test_search_api_uses_embedded_searches_with_sparse_embeddingsr:  ;
  s    ,-@@@IXX""&|(
 
 
 #  F 00@PPPMJNNV /2    
 ^^F+***vk"##q((((XX]]3>GGGHHF''G5>%%%wu~!####wu~a !!A%%%%WU^A&&&&&&&r5   c                 v   t                      }|                    t                      d           t          | |          \  }}|j        J |j        j        d         j        }|J |j        J |j        j        du sJ |	                    g dg ddd	idd
idd	ig           t          j        t                    5 }|                    ddi           ddd           n# 1 swxY w Y   dt          |j                                                  v sJ t          j        t                    5 }|                    dgdddi           ddd           n# 1 swxY w Y   dt          |j                                                  v sJ t          j        t                    5 }|                    ddi           ddd           n# 1 swxY w Y   dt          |j                                                  v sJ |                    dd	i          }t'          |d                   ddhk    sJ dS )zWDisabling FTS via schema should block where_document queries on get, query, and delete.r   r  r  NF)	fts-off-1z	fts-off-2	fts-off-3)zalpha beta gammazdelta epsilon zetazeta theta iotar   group_agroup_br   	$containsr   where_documentftszsome query textr   rp  )query_textsr  rB  etar   r   r<  r=  )r   r  r   r   r   r   r   r  r   r  r.  r/  r   rx   rX   r  r  r   r  r  )ry   r   r   rA   doc_overrider  r  s          r3   /test_fts_disabled_blocks_where_document_queriesrG  e
  s,    XXF
~//[AAA/0@PPPMJ ((($)+6=L###!---!)U2222 NN333NNN###
     
+	,	, >{G&<===> > > > > > > > > > > > > > >C''--//////	+	,	, 
*+'1 	 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 C''--//////	+	,	, ?+u)=>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?C''--////// ~~Z$;~<<HxK#=======s6   C&&C*-C*4EE #E *GGGc                    t          |           \  }}|j        J |j        j        d         j        }|J |j        J |j        j        du sJ |                    ddgddg           |                    d	d
i          }t          |d                   dhk    sJ |                    d	di          }t          |d                   dhk    sJ dS )zNDefault schema should have FTS enabled and where_document queries should work.Nr   Tzfts-on-1zfts-on-2zthe quick brown foxzlazy dog sleepsr  r@  foxrA  r   lazy)	r   r   r   r   r  r   r  rx   r  )ry   r   rA   rF  itemss        r3   0test_fts_enabled_by_default_where_document_worksrL  
  s    00@AAMJ ((($)+6=L###!---!)T1111NN$(*;<    
 NN;*>N??EuU|,,,,NN;*?N@@EuU|,,,,,,r5   c                 2   t                      }|                    t                      d           t          | |          \  }}|j        }|                    dgdg           |                                 }|                    |          }|j        J |j        j	        d         j
        }|J |j        J |j        j        d	u sJ t          j        t                    5  |                    d
di           ddd           dS # 1 swxY w Y   dS )z9FTS-disabled schema should persist across client reloads.r   r  r  zpersist-fts-1zpersistent doc for fts testr  r   NFr@  
persistentrA  )r   r  r   r   rG   r  r   r   r   r   r   r  r   r.  r/  r   rx   )ry   r   r   r   r   rL  r   rF  s           r3   $test_fts_disabled_schema_persistencerO  
  s    XXF
~//[AAA45EfUUUJ oONN01     '@@BBO)88o8NN %111&-2;?FL###!---!)U2222 
+	,	, L L\/JKKKL L L L L L L L L L L L L L L L L Ls   &DDDc                    t                      }|                    t                      d           t          | |          \  }}|                    ddgddg           |                    ddgd	g
          }|d	         J |d	         d         dk    sJ |d	         d         dk    sJ |                                dk    sJ dS )zFDocuments should still be stored and retrievable when FTS is disabled.r   r  r  zstored-1zstored-2zdocument one contentzdocument two contentr  r   r   Nr   r   rq  )r   r  r   r   r  rx   count)ry   r   r   rA   r  s        r3   (test_fts_disabled_documents_still_storedrR  
  s     XXF
~//[AAA/0@PPPMJNN$)+AB     ^^Z 8;-^PPF+***+q!%;;;;;+q!%;;;;; """"""r5   c                 T   t                      }|                    t                      d           t          | |          \  }}|j        }|                    dgdg           t          j        t                    5  |	                    ddi	           d
d
d
           n# 1 swxY w Y   |
                    |           |                    |          }|                    dgdg           |	                    ddi	          }t          |d                   dhk    sJ d
S )zVDeleting a FTS-disabled collection and recreating with FTS enabled should restore FTS.r   r  r  zfts-toggle-1zsearchable textr  r@  
searchablerA  Nr   zfts-toggle-2zsearchable text againr   )r   r  r   r   rG   r  r.  r/  r   rx   r#  r   r  )ry   schema_disabledr   r   col_namer  rK  s          r3   6test_fts_disabled_then_new_collection_with_fts_enabledrW  
  s   
 hhO  (8(8k JJJ4  J HNN$%    
 
+	,	, C C{L&ABBBC C C C C C C C C C C C C C C (+++..H.==HLL*+    
 LLl(CLDDEuU| 0000000s   ?B$$B(+B(c                    t                      }|                    t                      d           t          | |          \  }}|                    ddgddg           t          j        t                    5 }|                    t          t          j                            d	          
                     ddd           n# 1 swxY w Y   dt          |j                                                  v sJ dS )zDSearch API should also reject document filters when FTS is disabled.r   r  r  zsearch-fts-1zsearch-fts-2zalpha contentzbeta contentr  r   r   NrC  )r   r  r   r   r  r.  r/  r   r>  r    r   r  containsrX   r  r  )ry   r   r   rA   r  s        r3   3test_fts_disabled_search_api_blocks_document_filterrZ    s8   
 XXF
~//[AAA/0@PPPMJNN^,"N3    
 
+	,	, 
..w77888	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 C''--////////s   :ACCC)ry   r   r7   N)NN)r7   N){chromadb.apir   r   r  r   r   r   r   r	   r
   r   r   r   r   r   r   r   &chromadb.execution.expression.operatorr   chromadb.test.conftestr   r   r   r   r   -chromadb.test.utils.wait_for_version_increaser   r   "chromadb.utils.embedding_functionsr   r   chromadb.api.models.Collectionr   $chromadb.api.models.CollectionCommonr   chromadb.errorsr   chromadb.execution.expressionr   r    chromadb.typesr'  typingr!   r"   r#   r$   r%   r&   r'   r(   uuidr)   numpyrV  r.  rX   r+   r[   r   r   r   r   r   r  r   r(  markskipifr0  rE  rM  r^  rj  rm  r  r  r  r  r  r  r  r  r%  r2  r>  rK  rU  ra  rn  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r  r  r  r  r+  r-  r:  rG  rL  rO  rR  rW  rZ  r:   r5   r3   <module>rj     sA   - - - - - - - -                              7 6 6 6 6 6                            6 5 5 5 5 5 A A A A A A 0 0 0 0 0 0 5 5 5 5 5 5 5 5 8 8 8 8 8 8 L L L L L L L L L L L L L L L L L L L L                /S	:   2 P P P P P'8c'C P P P<7:'7:	7: 7: 7: 7:tn0'n0	n0 n0 n0 n0b $W W W W W+B49+M W W $#W8  $;? 'V !!23!78 :y !	   :s tCy    :KQ'KQ	KQ KQ KQ KQ\'5''5	'5 '5 '5 '5TS'S	S S S S8 ..7PQQ	E'	E		E 	E 	E RQ	E *3MNN65'65	65 65 65 ON65r&0'&0	&0 &0 &0 &0R>'>	> > > >: *3MNN/'/	/ / / ON/6 *3MNNK'K	K K K ONK: *3MNNA'A	A A A ONA851'51	51 51 51 51p+*'+*	+* +* +* +*\ *3MNN-*'-*	-* -* -* ON-*`B6'B6	B6 B6 B6 B6J.B'.B	.B .B .B .Bb *3MNND4'D4	D4 D4 D4 ON D4N *3MNNA1'A1	A1 A1 A1 ONA1H0='0=	0= 0= 0= 0=f *3MNN%9'%9	%9 %9 %9 ON%9P *3MNN*:'*:	*: *: *: ON*:Z *3MNN.9'.9	.9 .9 .9 ON.9b *3MNN,8',8	,8 ,8 ,8 ON,8^ *3MNN,-',-	,- ,- ,- ON,-^ *3MNN'*''*	'* '* '* ON'*T *3MNN3.'3.	3. 3. 3. ON3.l *3MNN*('*(	*( *( *( ON*(Z *3MNN4<'4<	4< 4< 4< ON4<n *3MNN-6'-6	-6 -6 -6 ON-6` *3MNN#;'#;	#; #; #; ON#;L *3MNN&'&	& & & ON&>61'61	61 61 61 61rI-'I-	I- I- I- I-XJ-'J-	J- J- J- J-ZL'L	L L L L@ *3MNN4'4	4 4 4 ON4@ *3MNN4'4	4 4 4 ON4> *3MNN+-'+-	+- +- +- ON+-\ *3MNNP6'P6	P6 P6 P6 ONP6fS S S SR R R R8 8 8 8( *3MNN%'%	% % % ON%P *3MNN,',	, , , ON,^ *3MNN^^	^ ^ ^ ON^B ..7JKKZZ	Z Z Z LKZB *3MNN)ZY )Z4 )Z )Z )Z ON)ZX *3MNN)])]	)] )] )] ON)]X *3MNNE: E:t E: E: E: ONE:P< < < <D $O O O O O"9$s)"D O O $#O< *3MNN!'%!'	!' !' !' ON!'R.>'.>	.> .> .> .>b-'-	- - - -2L'L	L L L L>#'#	# # # #01'1	1 1 1 1D *3MNN0'0	0 0 0 ON0 0 0r5   