
    _j/                     :   d dl mZ d dlmZmZmZmZmZmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& d dl'Z'd dl(m)Z)m*Z*m+Z+ d dl,Z,d dl-Z. e,j/        e0          Z1dZ2 G d de          Z3dS )    )override)OptionalSequenceDictSetListcast)UUID)VectorReader)Consumer)SystemSettings)Batch)
HnswParams)OpenTelemetryClientOpenTelemetryGranularitytrace_method)
	LogRecordRequestVersionContextVectorEmbeddingRecordVectorQueryVectorQueryResultSeqIdSegmentMetadata	OperationVector)InvalidDimensionExceptionN)ReadWriteLock
ReadRWLockWriteRWLocki  c            
       2    e Zd ZU eed<   eed<   ee         ed<   ee         ed<   eed<   eed<   ee	j
                 ed<   ee         ed<   eed	<   eed
<   eed<   eeef         ed<   eeef         ed<   eeef         ed<   eed<   dedefdZeededee         fd                        Z edej                  ed2 fd                        Z edej                  ed2 fd                        Z edej                  e	 d3dedeee                  dee         fd                        Z  edej                  ed e!deee"                  fd!                        Z#edefd"            Z$ededefd#            Z% ed$ej                  d%eddfd&            Z& ed'ej                  d(ed)eddfd*            Z' ed+ej                  d,e(ddfd-            Z) ed.ej                  d/ee*         ddfd0            Z+ed2d1            Z, xZ-S )4LocalHnswSegment_id	_consumer_collection_subscription	_settings_params_index_dimensionality_total_elements_added_max_seq_id_lock_id_to_label_label_to_id_id_to_seq_id_opentelemtry_clientsystemsegmentc                    |                     t                    | _        |d         | _        |d         | _        d | _        |j        | _        t          |d         pi           | _	        d | _
        d | _        d| _        | j                                        | _        i | _        i | _        i | _        t%                      | _        |                    t*                    | _        d S )Nid
collectionmetadatar   )instancer   r%   r$   r&   r'   settingsr(   r   r)   r*   r+   r,   	min_seqidr-   r1   r/   r0   r   r.   requirer   r2   )selfr3   r4   s      d/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/chromadb/segment/impl/vector/local_hnsw.py__init__zLocalHnswSegment.__init__=   s    224="<0!!'*"5";<<#%&">3355"__
$*NN3F$G$G!!!    r8   returnc                 .    t          j        |           }|S N)r   extract)r8   segment_metadatas     r>   propagate_collection_metadataz.LocalHnswSegment.propagate_collection_metadataQ   s     &-h77r@   zLocalHnswSegment.startNc                     t                                                       | j        rB|                                 }| j                            | j        | j        |          | _        d S d S )N)start)superrH   r&   	max_seqidr%   	subscribe_write_recordsr'   )r=   seq_id	__class__s     r>   rH   zLocalHnswSegment.startX   sl     	 	^^%%F!%!9!9 $"5V ": " "D	 	r@   zLocalHnswSegment.stopc                     t                                                       | j        r!| j                            | j                   d S d S rC   )rI   stopr'   r%   unsubscribe)r=   rN   s    r>   rP   zLocalHnswSegment.stopb   sJ     	 	;N&&t'9:::::	; 	;r@   zLocalHnswSegment.get_vectorsrequest_version_contextidsc                    |'t          | j                                                  }n0g }|D ]+}|| j        v r |                    | j        |                    ,g }| j        t          t          t                   t          j
        | j                            |                              }t          ||          D ]6\  }}| j        |         }|                    t          ||                     7|S )N)r6   	embedding)listr0   keysr/   appendr*   r	   r   r   nparray	get_itemszipr   )	r=   rR   rS   labelsr6   resultsvectorslabelvectors	            r>   get_vectorszLocalHnswSegment.get_vectorsi   s     ;$+002233FFF 9 9***MM$"3B"7888;" "(4;+@+@+H+H"I"I G "%VW!5!5 O Ov&u-4fMMMNNNNr@   zLocalHnswSegment.query_vectorsqueryc           
      ^     j         ,d t          t          |d                             D             S |d         }t           j                  }||k    r%t                              d| d| d|            |}t                      |d         }|0 fd|D             t                    |k     rt                    }d	t          d
t          ffd}|d         }t           j
                  5   j                             t          j        |t          j                  ||r|nd           \  }}g }	t          t          |                    D ]}
g }t          ||
         ||
                   D ]\  }} j        |         }|d         r4t          j         j                             |g          d                   }nd }|                    t'          ||                                |                     |	                    |           |	cd d d            S # 1 swxY w Y   d S )Nc                     g | ]}g S  rf   ).0_s     r>   
<listcomp>z2LocalHnswSegment.query_vectors.<locals>.<listcomp>   s    ===1B===r@   r_   kzNumber of requested results z- is greater than number of elements in index z, updating n_results = allowed_idsc                 <    h | ]}|j         v j         |         S rf   )r/   )rg   r6   r=   s     r>   	<setcomp>z1LocalHnswSegment.query_vectors.<locals>.<setcomp>   s.    UUUR4CT=T=Td'+=T=T=Tr@   r`   rA   c                     | v S rC   rf   )r`   r]   s    r>   filter_functionz7LocalHnswSegment.query_vectors.<locals>.filter_function   s    F?"r@   )dtype)rj   filterinclude_embeddingsr   )r6   distancerU   )r*   rangelenr/   loggerwarningsetintboolr    r.   	knn_queryrY   rZ   float32r\   r0   r[   rX   r   item)r=   rc   rj   sizerS   ro   query_vectorsresult_labels	distancesall_resultsresult_ir^   r`   rs   r6   rU   r]   s   `               @r>   r   zLocalHnswSegment.query_vectors   s   
 ;==c%	*:&;&; < <====#J4$%%t88NN Cq  C  C_c  C  C  }A  C  C   A55M"?UUUUcUUUF6{{QKK	#3 	#4 	# 	# 	# 	# 	# 	# i(
## !	 !	'+{'<'<bj999*-74 (= ( ($M9 :<K!#m"4"455 , ,35'*!(+Yx-@( (  OE8 *51B12 )$&H K115'::1=% %		 %)	NN)!%-]]__&/      ""7++++C!	 !	 !	 !	 !	 !	 !	 !	 !	 !	 !	 !	 !	 !	 !	 !	 !	 !	s   5D H""H&)H&c                     | j         S rC   )r-   r=   s    r>   rJ   zLocalHnswSegment.max_seqid   s    r@   c                 *    t          | j                  S rC   )ru   r/   )r=   rR   s     r>   countzLocalHnswSegment.count   s    4$%%%r@   zLocalHnswSegment._init_indexdimensionalityc                 @   t          j        | j        j        |          }|                    t
          | j        j        | j        j                   |                    | j        j	                   |
                    | j        j                   || _        || _        d S )N)spacedim)max_elementsef_constructionM)hnswlibIndexr)   r   
init_indexDEFAULT_CAPACITYconstruction_efr   set_ef	search_efset_num_threadsnum_threadsr*   r+   )r=   r   indexs      r>   _init_indexzLocalHnswSegment._init_index   s     ,$.
 
 
 	) L8ln 	 	
 	
 	

 	T\+,,,dl6777-r@   zLocalHnswSegment._ensure_indexnr   c                    | j         s|| _        |                     |           n*|| j        k    rt          d| dd| j         dz             t	          t
          j        | j                   }| j        |z   |                                k    rNt          | j        |z   | j
        j        z            }|                    t          |t                               dS dS )zCCreate or resize the index as necessary to accomodate N new recordszDimensionality of (z) does not match indexzdimensionality ()N)r*   r+   r   r   r	   r   r   r,   get_max_elementsry   r)   resize_factorresize_indexmaxr   )r=   r   r   r   new_sizes        r>   _ensure_indexzLocalHnswSegment._ensure_index   s     { 	#&D S!!!!d***/E#EEE@)=@@@A  
 W]DK00&*e.D.D.F.FFF+a/4<3MM H s8-=>>?????	 GFr@   zLocalHnswSegment._apply_batchbatchc                 ^   |                                 }|                                }|                    |          }dgt          |          z  }t          |          dk    rt	          t
          j        | j                  }t          t          |                    D ]N}||         }|| j	        vr| j	        |         }	|
                    |	           | j	        |= | j        |	= | j        |= Ot          |          dk    r+|                     |j        t          |d                              | j        dz   }
t          t          |                    D ]2}||         | j	        vr|
||<   |
dz  }
| j	        ||                  ||<   3t	          t
          j        | j                  }|                    ||           t#          |          D ]H\  }}|                    |          d         | j        |<   ||         | j	        |<   || j        ||         <   I| xj        |j        z  c_        dS dS )z4Apply a batch of changes, as atomically as possible.r      
log_offsetN)get_deleted_idsget_written_idsget_written_vectorsru   r	   r   r   r*   rt   r/   mark_deletedr0   r1   r   	add_countr,   	add_items	enumerate
get_record)r=   r   deleted_idswritten_idsvectors_to_writelabels_to_writer   ir6   r`   
next_labels              r>   _apply_batchzLocalHnswSegment._apply_batch   sN    ++--++-- 44[AA#$4 5 55{a44E3{++,, 
+ 
+ ^T...)"-""5)))%b)%e,&r**{au4DQ4G0H0HIII3a7J3{++,, K Kq>):::)3OA&!OJJ)-):;q>)JOA&&44E OO,o>>> #;// ; ;2).)9)9")=)=l)K"2&(7(:!"%8:!/!"455 &&%/9&&&&/  r@   zLocalHnswSegment._write_recordsrecordsc                    | j         st          d          t          | j                  5  t	                      }|D ]r}t          | j        |d                   | _        |d         d         }|d         d         }| j                            |d          }|t          j
        k    r6|r|                    |           t                              d|            |t          j        k    rQ|d         d         A||                    |           t                              d	|d         d                     |t          j        k    r9|s|                    |d
           +t                              d|            J|t          j        k    r|                    ||du           t|                     |           ddd           dS # 1 swxY w Y   dS )z&Add a batch of embeddings to the indexz*Cannot add embeddings to stopped componentr   recordr6   	operationNz$Delete of nonexisting embedding ID: rU   z$Update of nonexisting embedding ID: FzAdd of existing embedding ID: )_runningRuntimeErrorr!   r.   r   r   r-   r/   getr   DELETEapplyrv   rw   UPDATEADDUPSERTr   )r=   r   r   r   r6   opr`   s          r>   rL   zLocalHnswSegment._write_records"  s5    } 	MKLLL $$ 	% 	%GGE! ; ;#&t'79M#N#N H%d+H%k2)--b$77))) TF++++'Rb'R'RSSSS9+++h'4@ ,!KK////"NN _vhGWX\G] _ _   9=((  NFE2222'L'L'LMMMM9+++KKT(9:::e$$$?	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%s   FGGGc                     t                      rC   )NotImplementedErrorr   s    r>   deletezLocalHnswSegment.deleteJ  s    !###r@   )rA   NrC   ).__name__
__module____qualname__r
   __annotations__r   r   r   r   r   r   ry   r   r   r   strr   r   r   r?   staticmethodr   r   rF   r   r   ALLrH   rP   r   r   r   rb   r   r   r   rJ   r   r   r   r   r   r   rL   r   __classcell__)rN   s   @r>   r#   r#   %   sM        	III$D>!!!W]####c]"""sCx.   sCx.    U
####----Hv H H H H H(    Xh=O       X \ 
 \*,D,HII     X JI \)+C+GHH; ; ; ; ; X IH;
 \02J2NOO (, !6 hsm$ 
'	(	   X PO2 \24L4PQQ< <	(,-	.< < < X RQ<|  5       X  &-B &s & & & X& \02J2NOO.# .$ . . . PO." \24L4PQQ@s @ @ @ @ @ RQ@( \13K3OPP,:% ,:D ,: ,: ,: QP,:\ \35M5QRR%%hy&9 %%d %% %% %% SR%%N $ $ $ X$ $ $ $ $r@   r#   )4	overridesr   typingr   r   r   r   r   r	   uuidr
   chromadb.segmentr   chromadb.ingestr   chromadb.configr   r   "chromadb.segment.impl.vector.batchr   (chromadb.segment.impl.vector.hnsw_paramsr    chromadb.telemetry.opentelemetryr   r   r   chromadb.typesr   r   r   r   r   r   r   r   r   r   chromadb.errorsr   r   chromadb.utils.read_write_lockr   r    r!   loggingnumpyrY   	getLoggerr   rv   r   r#   rf   r@   r>   <module>r      s	         < < < < < < < < < < < < < < < <       ) ) ) ) ) ) $ $ $ $ $ $ , , , , , , , , 4 4 4 4 4 4 ? ? ? ? ? ?         
                        6 5 5 5 5 5  Q Q Q Q Q Q Q Q Q Q     		8	$	$ g$ g$ g$ g$ g$| g$ g$ g$ g$ g$r@   